From b999f53d4bf5d44586ecf028876e8bc20b5fd2ce Mon Sep 17 00:00:00 2001 From: Marcin Krol Date: Wed, 27 Jul 2016 22:47:38 +0000 Subject: [PATCH] - PLD merge, fully (re)TLDized --- .gitignore | 9 + Makefile | 27 ++- PLD_Builder/request_handler_server.py | 192 ---------------- {PLD_Builder => TLD_Builder}/__init__.py | 0 {PLD_Builder => TLD_Builder}/acl.py | 7 + {PLD_Builder => TLD_Builder}/bqueue.py | 15 +- {PLD_Builder => TLD_Builder}/build.py | 6 +- {PLD_Builder => TLD_Builder}/buildlogs.py | 0 {PLD_Builder => TLD_Builder}/chroot.py | 0 {PLD_Builder => TLD_Builder}/cia-foot.xml | 0 {PLD_Builder => TLD_Builder}/cia-head.xml | 2 +- {PLD_Builder => TLD_Builder}/config.py | 14 +- {PLD_Builder => TLD_Builder}/deps.py | 0 {PLD_Builder => TLD_Builder}/file_sender.py | 2 +- {PLD_Builder => TLD_Builder}/ftp.py | 0 {PLD_Builder => TLD_Builder}/get_br.py | 0 {PLD_Builder => TLD_Builder}/gpg.py | 4 + {PLD_Builder => TLD_Builder}/install.py | 22 +- {PLD_Builder => TLD_Builder}/load_balancer.py | 0 {PLD_Builder => TLD_Builder}/lock.py | 0 {PLD_Builder => TLD_Builder}/log.py | 0 {PLD_Builder => TLD_Builder}/loop.py | 1 - {PLD_Builder => TLD_Builder}/mailer.py | 10 +- {PLD_Builder => TLD_Builder}/maintainer.py | 1 - {PLD_Builder => TLD_Builder}/notify.py | 0 {PLD_Builder => TLD_Builder}/notifyq.py | 0 {PLD_Builder => TLD_Builder}/path.py | 3 +- {PLD_Builder => TLD_Builder}/pipeutil.py | 0 {PLD_Builder => TLD_Builder}/poldek.py | 0 {PLD_Builder => TLD_Builder}/report.py | 48 ++-- {PLD_Builder => TLD_Builder}/request.py | 154 +++++++++++-- .../request_fetcher.py | 8 +- .../request_handler.py | 30 ++- .../request_handler_server | 9 +- TLD_Builder/request_handler_server.py | 119 ++++++++++ {PLD_Builder => TLD_Builder}/rpm_builder.py | 86 +++++--- {PLD_Builder => TLD_Builder}/srpm_builder.py | 18 +- {PLD_Builder => TLD_Builder}/status.py | 0 {PLD_Builder => TLD_Builder}/stopwatch.py | 0 {PLD_Builder => TLD_Builder}/util.py | 9 +- {PLD_Builder => TLD_Builder}/wrap.py | 2 +- admin/fresh-queue.sh | 4 +- admin/install-chroot.sh | 38 ++-- bin/file-sender.sh | 9 +- bin/load-balancer.sh | 9 +- bin/maintainer.sh | 9 +- bin/request-fetcher.sh | 9 +- bin/request-handler.sh | 9 +- bin/rpm-builder.sh | 9 +- bin/src-builder.sh | 9 +- client/make-request.sh | 205 +++++++++++------- client/smtpwrapper.py | 2 +- config/acl.conf | 6 +- config/builder.conf | 47 ++-- doc/README | 2 +- doc/README.bin-builder | 3 +- etc/crontab | 8 +- etc/pld-builder.sysconfig | 3 - etc/poldek.conf | 2 +- etc/procmailrc | 4 +- etc/rpm.macros | 2 +- etc/{pld-builder.init => tld-builder.init} | 26 +-- xml/queue.xslt | 2 +- 63 files changed, 702 insertions(+), 513 deletions(-) create mode 100644 .gitignore delete mode 100644 PLD_Builder/request_handler_server.py rename {PLD_Builder => TLD_Builder}/__init__.py (100%) rename {PLD_Builder => TLD_Builder}/acl.py (95%) rename {PLD_Builder => TLD_Builder}/bqueue.py (89%) rename {PLD_Builder => TLD_Builder}/build.py (96%) rename {PLD_Builder => TLD_Builder}/buildlogs.py (100%) rename {PLD_Builder => TLD_Builder}/chroot.py (100%) rename {PLD_Builder => TLD_Builder}/cia-foot.xml (100%) rename {PLD_Builder => TLD_Builder}/cia-head.xml (81%) rename {PLD_Builder => TLD_Builder}/config.py (83%) rename {PLD_Builder => TLD_Builder}/deps.py (100%) rename {PLD_Builder => TLD_Builder}/file_sender.py (99%) rename {PLD_Builder => TLD_Builder}/ftp.py (100%) rename {PLD_Builder => TLD_Builder}/get_br.py (100%) rename {PLD_Builder => TLD_Builder}/gpg.py (96%) rename {PLD_Builder => TLD_Builder}/install.py (90%) rename {PLD_Builder => TLD_Builder}/load_balancer.py (100%) rename {PLD_Builder => TLD_Builder}/lock.py (100%) rename {PLD_Builder => TLD_Builder}/log.py (100%) rename {PLD_Builder => TLD_Builder}/loop.py (99%) rename {PLD_Builder => TLD_Builder}/mailer.py (90%) rename {PLD_Builder => TLD_Builder}/maintainer.py (99%) rename {PLD_Builder => TLD_Builder}/notify.py (100%) rename {PLD_Builder => TLD_Builder}/notifyq.py (100%) rename {PLD_Builder => TLD_Builder}/path.py (90%) rename {PLD_Builder => TLD_Builder}/pipeutil.py (100%) rename {PLD_Builder => TLD_Builder}/poldek.py (100%) rename {PLD_Builder => TLD_Builder}/report.py (74%) rename {PLD_Builder => TLD_Builder}/request.py (75%) rename {PLD_Builder => TLD_Builder}/request_fetcher.py (94%) rename {PLD_Builder => TLD_Builder}/request_handler.py (86%) rename {PLD_Builder => TLD_Builder}/request_handler_server (60%) create mode 100644 TLD_Builder/request_handler_server.py rename {PLD_Builder => TLD_Builder}/rpm_builder.py (84%) rename {PLD_Builder => TLD_Builder}/srpm_builder.py (90%) rename {PLD_Builder => TLD_Builder}/status.py (100%) rename {PLD_Builder => TLD_Builder}/stopwatch.py (100%) rename {PLD_Builder => TLD_Builder}/util.py (86%) rename {PLD_Builder => TLD_Builder}/wrap.py (97%) delete mode 100644 etc/pld-builder.sysconfig rename etc/{pld-builder.init => tld-builder.init} (70%) mode change 100644 => 100755 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..25ac96d --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +config/acl.conf +config/builder.conf +config/rsync-passwords +config/*.tar +*.pyc +lock +log +spool +www diff --git a/Makefile b/Makefile index e82dc21..dc98a49 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,21 @@ -PACKAGE := pld-builder -VERSION := 0.5 +PACKAGE := tld-builder +VERSION := 0.6 SNAP := $(shell date +%Y%m%d) -# for make dist -CVSROOT := :pserver:cvs@cvs.pld-linux.org:/cvsroot -CVSMODULE := pld-builder.new -CVSTAG := HEAD +all: compile -all: +compile: python -c "import compileall; compileall.compile_dir('.')" clean: find -name '*.pyc' | xargs rm -f + rm -f *.tar.bz2 -dist: - rm -rf $(PACKAGE)-$(VERSION).$(SNAP) - mkdir -p $(PACKAGE)-$(VERSION).$(SNAP) - cvs -d $(CVSROOT) export -d $(PACKAGE)-$(VERSION).$(SNAP) -r $(CVSTAG) $(CVSMODULE) - tar -cjf $(PACKAGE)-$(VERSION).$(SNAP).tar.bz2 $(PACKAGE)-$(VERSION).$(SNAP) - rm -rf $(PACKAGE)-$(VERSION).$(SNAP) - test -x ./dropin && ./dropin $(PACKAGE)-$(VERSION).$(SNAP).tar.bz2 +dist: $(PACKAGE)-$(VERSION).$(SNAP).tar.bz2 + test ! -x ./dropin || ./dropin $< + +%.tar.bz2: %.tar + bzip2 -9 $< + +%.tar: + git archive --prefix=$(patsubst %.tar,%,$@)/ HEAD -o $@ diff --git a/PLD_Builder/request_handler_server.py b/PLD_Builder/request_handler_server.py deleted file mode 100644 index 5c591f0..0000000 --- a/PLD_Builder/request_handler_server.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/python - -import socket -import string -import cgi -import time -import log -import sys -import traceback -import os -from config import config, init_conf - -from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer - -import request_handler -import path - -class MyHandler(BaseHTTPRequestHandler): - - def do_GET(self): - self.send_error(401); - - def do_POST(self): - global rootnode - try: - length = int(self.headers.getheader('content-length')) - ctype, pdict = cgi.parse_header(self.headers.getheader('content-type')) - if ctype != 'application/x-www-form-urlencoded': - log.error("request_handler_server: [%s]: 401 Unauthorized" % self.client_address[0]) - self.send_error(401) - self.end_headers() - return - - query = self.rfile.read(length) - - filename = self.headers.getheader('x-filename') - - if not request_handler.handle_request_main(query, filename = filename): - error = log.last_log(); - log.error("request_handler_server: [%s]: handle_request_main(..., %s) failed" % (self.client_address[0], filename)) - self.send_error(500, "%s: request failed. %s" % (filename, error)) - self.end_headers() - return - - self.send_response(200) - self.end_headers() - - except Exception, e: - self.send_error(500, "%s: %s" % (filename, e)) - self.end_headers() - log.error("request_handler_server: [%s]: exception: %s\n%s" % (self.client_address[0], e, traceback.format_exc())) - raise - pass - -def write_css(): - css_file = path.www_dir + "/style.css" - # skip if file exists and code is not newer - if os.path.exists(css_file) and os.stat(__file__).st_mtime < os.stat(css_file).st_mtime: - return - - # css from www.pld-linux.org wiki theme, using css usage firebug plugin to cleanup - css = """ -html { - background-color: white; - color: #5e5e5e; - font-family: Tahoma, Arial, Lucida Grande, sans-serif; - font-size: 0.75em; - line-height: 1.25em; -} - -a { - text-decoration: underline; - color: #006; -} - -a:hover { - color: #006; -} - -pre { - background: #FFF8EB; - border: 1pt solid #FFE2AB; - font-family: courier, monospace; - padding: 0.5em; - white-space: pre-wrap; - word-wrap: break-word; -} - -@media screen, projection { - html { - background-color: #f3efe3; - } - - body { - position: relative; - } - - div { - background-color: white; - margin: 10px 0px; - padding: 2px; - } - div > a { - font-weight: bold; - color: #5e5e5e; - } - div > a:hover { - color: #5e5e5e; - } - div.upgrade { - background-color: #e4f1cf; - } - div:target { - background-color: #ffffcc; - color: black; - } -} -@media print { - a { - background-color: inherit; - color: inherit; - } -} - -@media projection { - html { line-height: 1.8em; } - body, b, a, p { font-size: 22pt; } -} -""" - old_umask = os.umask(0022) - f = open(css_file, "w") - f.write(css) - f.close() - os.umask(old_umask) - -def write_js(): - js_file = path.www_dir + "/script.js" - # skip if file exists and code is not newer - if os.path.exists(js_file) and os.stat(__file__).st_mtime < os.stat(js_file).st_mtime: - return - - js = """ -// update date stamps to reflect viewers timezone -function update_tz(t) { - var el, off, dt, - collection = document.getElementsByTagName('span'); - for (off in collection) { - el = collection[off]; - if (el.id == 'tz') { - dt = new Date(el.innerHTML).toString(); - // strip timezone name, it is usually wrong when not initialized - // from TZ env, but reverse calculated from os data - dt = dt.replace(/\s+\(.+\)/, ""); - // strip "GMT" - dt = dt.replace(/GMT/, ""); - el.innerHTML = dt; - } - } -} -window.onload = update_tz; -""" - old_umask = os.umask(0022) - f = open(js_file, "w") - f.write(js) - f.close() - os.umask(old_umask) - -def main(): - write_css(); - write_js(); - socket.setdefaulttimeout(30) - try: - init_conf() - host = "" - port = config.request_handler_server_port - - try: - server = HTTPServer((host, port), MyHandler) - except Exception, e: - log.notice("request_handler_server: can't start server on [%s:%d]: %s" % (host, port, e)) - print >> sys.stderr, "ERROR: Can't start server on [%s:%d]: %s" % (host, port, e) - sys.exit(1) - - log.notice('request_handler_server: started on [%s:%d]...' % (host, port)) - server.serve_forever() - except KeyboardInterrupt: - log.notice('request_handler_server: ^C received, shutting down server') - server.socket.close() - -if __name__ == '__main__': - main() - diff --git a/PLD_Builder/__init__.py b/TLD_Builder/__init__.py similarity index 100% rename from PLD_Builder/__init__.py rename to TLD_Builder/__init__.py diff --git a/PLD_Builder/acl.py b/TLD_Builder/acl.py similarity index 95% rename from PLD_Builder/acl.py rename to TLD_Builder/acl.py index a769d06..5f06219 100644 --- a/PLD_Builder/acl.py +++ b/TLD_Builder/acl.py @@ -18,6 +18,7 @@ class User: self.privs = [] self.gpg_emails = [] self.mailto = "" + self.change_requester = False if p.has_option(login, "gpg_emails"): self.gpg_emails = string.split(p.get(login, "gpg_emails")) @@ -30,6 +31,9 @@ class User: if len(self.gpg_emails) > 0: self.mailto = self.gpg_emails[0] + if p.has_option(login, "change_requester"): + self.change_requester = True + if p.has_option(login, "privs"): for p in string.split(p.get(login, "privs")): l = string.split(p, ":") @@ -124,6 +128,9 @@ class ACL_Conf: return self.users[e] return None + def user_by_login(self, l): + return self.users[l] + def user(self, l): if not self.users.has_key(l): log.panic("no such user: %s" % l) diff --git a/PLD_Builder/bqueue.py b/TLD_Builder/bqueue.py similarity index 89% rename from PLD_Builder/bqueue.py rename to TLD_Builder/bqueue.py index 2a4d759..f95f0f3 100644 --- a/PLD_Builder/bqueue.py +++ b/TLD_Builder/bqueue.py @@ -9,7 +9,7 @@ import fcntl import string import tempfile -# PLD_Builder: +# TLD_Builder: import gpg import request import util @@ -40,18 +40,25 @@ class B_Queue: f.write(""" - + TLD builder queue + + -\n""" + + +
+""" ) self.requests.reverse() for r in self.requests: r.dump_html(f) self.requests.reverse() - f.write("\n") + f.write("
\n") f.flush() os.fsync(f.fileno()) f.close() diff --git a/PLD_Builder/build.py b/TLD_Builder/build.py similarity index 96% rename from PLD_Builder/build.py rename to TLD_Builder/build.py index e996731..31689e7 100644 --- a/PLD_Builder/build.py +++ b/TLD_Builder/build.py @@ -46,7 +46,7 @@ def run_command(batch): else: user = "root" if "as-builder" in batch.command_flags: - user = "builder" + user = None return chroot.run(command, logfile = batch.logfile, user = user) def build_all(r, build_fnc): @@ -82,7 +82,7 @@ def build_all(r, build_fnc): log.notice("running %s OK" % batch.command) notify.add_batch(batch, "OK") batch.build_time = stopwatch.stop() - report.add_pld_builder_info(batch) + report.add_tld_builder_info(batch) buildlogs.add(batch.logfile, failed = batch.build_failed, id=r.id) else: log.notice("not running command, not for me.") @@ -106,7 +106,7 @@ def build_all(r, build_fnc): log.notice("building %s OK" % (batch.spec)) notify.add_batch(batch, "OK") batch.build_time = stopwatch.stop() - report.add_pld_builder_info(batch) + report.add_tld_builder_info(batch) buildlogs.add(batch.logfile, failed = batch.build_failed, id=r.id) else: batch.build_failed = 1 diff --git a/PLD_Builder/buildlogs.py b/TLD_Builder/buildlogs.py similarity index 100% rename from PLD_Builder/buildlogs.py rename to TLD_Builder/buildlogs.py diff --git a/PLD_Builder/chroot.py b/TLD_Builder/chroot.py similarity index 100% rename from PLD_Builder/chroot.py rename to TLD_Builder/chroot.py diff --git a/PLD_Builder/cia-foot.xml b/TLD_Builder/cia-foot.xml similarity index 100% rename from PLD_Builder/cia-foot.xml rename to TLD_Builder/cia-foot.xml diff --git a/PLD_Builder/cia-head.xml b/TLD_Builder/cia-head.xml similarity index 81% rename from PLD_Builder/cia-head.xml rename to TLD_Builder/cia-head.xml index fefeda5..afad52a 100644 --- a/PLD_Builder/cia-head.xml +++ b/TLD_Builder/cia-head.xml @@ -2,7 +2,7 @@ TLD Linux Builder client for CIA 0.001 - http://git.tld-linux.org/?p=pld-builder.new.git;a=blob;f=PLD_Builder/report.py + http://git.tld-linux.org/?p=tld-builder.new.git;a=blob;f=TLD_Builder/report.py tld diff --git a/PLD_Builder/config.py b/TLD_Builder/config.py similarity index 83% rename from PLD_Builder/config.py rename to TLD_Builder/config.py index 2e78cd8..5e3bd9e 100644 --- a/PLD_Builder/config.py +++ b/TLD_Builder/config.py @@ -37,8 +37,10 @@ class Builder_Conf: def read(self, builder): p = ConfigParser.ConfigParser() - def get(o, d = None): - if p.has_option(builder, o): + def get(o, d = None, sec=None): + if p.has_option(sec, o): + return string.strip(p.get(sec, o)) + elif p.has_option(builder, o): return string.strip(p.get(builder, o)) elif p.has_option("all", o): return string.strip(p.get("all", o)) @@ -70,7 +72,12 @@ class Builder_Conf: self.max_keep_time = int(get("max_keep_time", 168))*60*60 self.bot_email = get("bot_email", "") self.control_url = get("control_url") - self.request_handler_server_port = int(get("request_handler_server_port", 1234)) + self.request_handler_server_port = int(get("port", d=1234, sec="request-server")) + self.request_handler_server_ssl = get("ssl", d="False", sec="request-server") + if self.request_handler_server_ssl: + self.request_handler_server_ssl_port = int(get("port", d=1235, sec="request-server")) + self.request_handler_server_ssl_key = get("ssl_key", d="", sec="request-server") + self.request_handler_server_ssl_cert = get("ssl_cert", d="", sec="request-server") self.builder_list = get("builder_list", "") self.gen_upinfo = get("gen_upinfo", "yes") if self.gen_upinfo == 'no': @@ -90,6 +97,7 @@ class Builder_Conf: self.arch = get("arch") self.chroot = get("chroot") self.buildlogs_url = get("buildlogs_url", "/dev/null") + self.buildlogs = get("buildlogs", "http://buildlogs.tld-linux.org/") self.ftp_url = get("ftp_url") self.notify_url = get("notify_url") self.test_ftp_url = get("test_ftp_url", "/dev/null") diff --git a/PLD_Builder/deps.py b/TLD_Builder/deps.py similarity index 100% rename from PLD_Builder/deps.py rename to TLD_Builder/deps.py diff --git a/PLD_Builder/file_sender.py b/TLD_Builder/file_sender.py similarity index 99% rename from PLD_Builder/file_sender.py rename to TLD_Builder/file_sender.py index 1df7ea1..0f652db 100644 --- a/PLD_Builder/file_sender.py +++ b/TLD_Builder/file_sender.py @@ -119,7 +119,7 @@ def send_file(src, target): m = re.match('ssh\+rsync://([^@:]+@[^/:]+)(:|)(.*)', target) if m: return not rsync_ssh_file(src, m.group(1) + ":" + m.group(3)) - m = re.match('http://.*', target) + m = re.match('(http|https)://.*', target) if m: return not post_file(src, target) log.alert("unsupported protocol: %s" % target) diff --git a/PLD_Builder/ftp.py b/TLD_Builder/ftp.py similarity index 100% rename from PLD_Builder/ftp.py rename to TLD_Builder/ftp.py diff --git a/PLD_Builder/get_br.py b/TLD_Builder/get_br.py similarity index 100% rename from PLD_Builder/get_br.py rename to TLD_Builder/get_br.py diff --git a/PLD_Builder/gpg.py b/TLD_Builder/gpg.py similarity index 96% rename from PLD_Builder/gpg.py rename to TLD_Builder/gpg.py index 5c5dbec..aeb8ebd 100644 --- a/PLD_Builder/gpg.py +++ b/TLD_Builder/gpg.py @@ -78,6 +78,7 @@ def sign(buf): d_stdout = None d_stderr = None cmd = ['/usr/bin/gpg', '--batch', '--no-tty', '--clearsign'] + # TODO: check for gpg return code! gpg_run = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) try: d_stdout, d_stderr = gpg_run.communicate(buf.encode('utf-8')) @@ -85,4 +86,7 @@ def sign(buf): log.error("gnupg signing failed, does gpg binary exist? : %s" % e) raise + if len(d_stderr): + log.error("gpg: %s" % d_stderr) + return d_stdout diff --git a/PLD_Builder/install.py b/TLD_Builder/install.py similarity index 90% rename from PLD_Builder/install.py rename to TLD_Builder/install.py index 91cc889..0131e19 100644 --- a/PLD_Builder/install.py +++ b/TLD_Builder/install.py @@ -13,6 +13,7 @@ hold = [ 'poldek', 'rpm-build', 'pdksh', + 'mksh', 'coreutils' ] @@ -100,12 +101,11 @@ def uninstall(conflicting, b): def uninstall_self_conflict(b): b.log_line("checking BuildConflict-ing packages") - packagename = b.spec[:-5] - tmpdir = os.environ.get('HOME') + "/rpm/BUILD/%s/tmp" % packagename - f = chroot.popen("set -e; TMPDIR=%(tmpdir)s rpmbuild -bp --nobuild --short-circuit --define 'prep exit 0' %(rpmdefs)s rpm/packages/%(package)s/%(spec)s 2>&1" % { - 'tmpdir': tmpdir, + f = chroot.popen("set -e; TMPDIR=%(tmpdir)s " \ + "rpmbuild -bp --nobuild --short-circuit --define 'prep exit 0' %(rpmdefs)s %(topdir)s/%(spec)s 2>&1" % { + 'tmpdir': b.tmpdir(), 'rpmdefs' : b.rpmbuild_opts(), - 'package' : packagename, + 'topdir' : b.get_topdir(), 'spec': b.spec, }) # java-sun >= 1.5 conflicts with soprano-2.1.67-1.src @@ -128,13 +128,11 @@ def install_br(r, b): # ignore internal rpm dependencies, see lib/rpmns.c for list ignore_br = re.compile(r'^\s*(rpmlib|cpuinfo|getconf|uname|soname|user|group|mounted|diskspace|digest|gnupg|macro|envvar|running|sanitycheck|vcheck|signature|verify|exists|executable|readable|writable)\(.*') - packagename = b.spec[:-5] - tmpdir = os.environ.get('HOME') + "/rpm/BUILD/%s/tmp" % packagename - chroot.run("install -m 700 -d %s" % tmpdir) - cmd = "set -e; TMPDIR=%(tmpdir)s rpmbuild --nobuild %(rpmdefs)s rpm/packages/%(package)s/%(spec)s 2>&1" % { + tmpdir = b.tmpdir() + cmd = "set -e; TMPDIR=%(tmpdir)s rpmbuild --nobuild %(rpmdefs)s %(topdir)s/%(spec)s 2>&1" % { 'tmpdir': tmpdir, + 'topdir' : b.get_topdir(), 'rpmdefs' : b.rpmbuild_opts(), - 'package' : packagename, 'spec': b.spec, } f = chroot.popen(cmd) @@ -163,7 +161,7 @@ def install_br(r, b): chroot.run("poldek --up --upa", user = "root", logfile = b.logfile) # check conflicts in BRed packages b.log_line("checking conflicting packages in BRed packages") - f = chroot.popen("poldek --test --test --noask --caplookup -Q -v --upgrade %s" % br, user = "root") + f = chroot.popen("poldek --test --test --noask --caplookup -Q -v %s --upgrade %s" % (b.ignores(), br), user = "root") # phonon-devel-4.3.1-1.i686 conflicts with qt4-phonon-devel-4.5.0-6.i686 # jdbc-stdext >= 2.0 is required by installed java-struts-1.3.10-1.noarch # jmx is needed by (installed) java-commons-modeler-2.0-1.noarch @@ -193,7 +191,7 @@ def install_br(r, b): br = string.strip(nbr) b.log_line("installing BR: %s" % br) - res = chroot.run("poldek --noask --caplookup -Q -v --upgrade %s" % br, + res = chroot.run("set -x; poldek --noask --caplookup -Q -v %s --upgrade %s" % (b.ignores(), br), user = "root", logfile = b.logfile) if res != 0: diff --git a/PLD_Builder/load_balancer.py b/TLD_Builder/load_balancer.py similarity index 100% rename from PLD_Builder/load_balancer.py rename to TLD_Builder/load_balancer.py diff --git a/PLD_Builder/lock.py b/TLD_Builder/lock.py similarity index 100% rename from PLD_Builder/lock.py rename to TLD_Builder/lock.py diff --git a/PLD_Builder/log.py b/TLD_Builder/log.py similarity index 100% rename from PLD_Builder/log.py rename to TLD_Builder/log.py diff --git a/PLD_Builder/loop.py b/TLD_Builder/loop.py similarity index 99% rename from PLD_Builder/loop.py rename to TLD_Builder/loop.py index bc5be9a..5ec5306 100644 --- a/PLD_Builder/loop.py +++ b/TLD_Builder/loop.py @@ -28,4 +28,3 @@ def run_loop(fnc, secs = 5, max = 60): took = time.time() - last if took < secs: time.sleep(secs - took) - diff --git a/PLD_Builder/mailer.py b/TLD_Builder/mailer.py similarity index 90% rename from PLD_Builder/mailer.py rename to TLD_Builder/mailer.py index e0b3ae6..f1459fa 100644 --- a/PLD_Builder/mailer.py +++ b/TLD_Builder/mailer.py @@ -32,6 +32,10 @@ class Message: if subject != None: self.set_header("Subject", subject) + def remove_header(self, n): + if n in self.headers: + del self.headers[n] + def write_line(self, l): self.body.write(recode("%s\n" % l)) @@ -59,11 +63,11 @@ class Message: def set_std_headers(self): self.headers["Date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()) - self.headers["Message-ID"] = "" \ + self.headers["Message-ID"] = "" \ % (time.time(), os.getpid(), os.uname()[1]) - self.headers["From"] = "PLD %s builder <%s>" \ + self.headers["From"] = "TLD %s builder <%s>" \ % (config.builder, config.email) - self.headers["X-PLD-Builder"] = config.builder + self.headers["X-TLD-Builder"] = config.builder def write_to(self, f): for k, v in self.headers.items(): diff --git a/PLD_Builder/maintainer.py b/TLD_Builder/maintainer.py similarity index 99% rename from PLD_Builder/maintainer.py rename to TLD_Builder/maintainer.py index 0ef6608..5ba3deb 100644 --- a/PLD_Builder/maintainer.py +++ b/TLD_Builder/maintainer.py @@ -77,4 +77,3 @@ if __name__ == '__main__': continue else: handle_bin() - diff --git a/PLD_Builder/notify.py b/TLD_Builder/notify.py similarity index 100% rename from PLD_Builder/notify.py rename to TLD_Builder/notify.py diff --git a/PLD_Builder/notifyq.py b/TLD_Builder/notifyq.py similarity index 100% rename from PLD_Builder/notifyq.py rename to TLD_Builder/notifyq.py diff --git a/PLD_Builder/path.py b/TLD_Builder/path.py similarity index 90% rename from PLD_Builder/path.py rename to TLD_Builder/path.py index 1558ab9..24776d7 100644 --- a/PLD_Builder/path.py +++ b/TLD_Builder/path.py @@ -4,7 +4,7 @@ import os root_dir = os.getenv('BUILDERPATH') if root_dir is None: - root_dir = os.path.expanduser('~/pld-builder.new') + root_dir = os.path.expanduser('~/tld-builder.new') conf_dir = root_dir + "/config" spool_dir = root_dir + "/spool" lock_dir = root_dir + "/lock" @@ -13,6 +13,7 @@ www_dir = root_dir + "/www" acl_conf = conf_dir + "/acl.conf" builder_conf = conf_dir + "/builder.conf" rsync_password_file = conf_dir + "/rsync-passwords" +blacklist_file = conf_dir + "/package-blacklist" # spool/ queue_file = spool_dir + "/queue" diff --git a/PLD_Builder/pipeutil.py b/TLD_Builder/pipeutil.py similarity index 100% rename from PLD_Builder/pipeutil.py rename to TLD_Builder/pipeutil.py diff --git a/PLD_Builder/poldek.py b/TLD_Builder/poldek.py similarity index 100% rename from PLD_Builder/poldek.py rename to TLD_Builder/poldek.py diff --git a/PLD_Builder/report.py b/TLD_Builder/report.py similarity index 74% rename from PLD_Builder/report.py rename to TLD_Builder/report.py index a0fe7be..c6e6294 100644 --- a/PLD_Builder/report.py +++ b/TLD_Builder/report.py @@ -18,26 +18,26 @@ def unpackaged_files(b): copy_mode = 1 out.append(l) elif copy_mode: - if l[0] != ' ': + if l[0] != ' ' and l[0] != '\t': copy_mode = 0 else: out.append(l) return out -def add_pld_builder_info(b): +def add_tld_builder_info(b): l = open(b.logfile, "a") - l.write("Begin-PLD-Builder-Info\n") + l.write("Begin-TLD-Builder-Info\n") l.write("Build-Time: %s\n\n" % b.build_time) st = ftp.status() if st != "": l.write("Files queued for ftp:\n%s\n" % st) ftp.clear_status() l.writelines(unpackaged_files(b)) - l.write("End-PLD-Builder-Info\n") + l.write("End-TLD-Builder-Info\n") def info_from_log(b, target): - beg = "Begin-PLD-Builder-Info\n" - end = "End-PLD-Builder-Info\n" + beg = "Begin-TLD-Builder-Info\n" + end = "End-TLD-Builder-Info\n" f = open(b.logfile) copy_mode = 0 need_header = 1 @@ -72,15 +72,8 @@ def send_report(r, is_src = False): subject += ' '.join((s_failed, s_ok)).strip() m = mailer.Message() - m.set_headers(to = r.requester_email, - cc = config.builder_list, - subject = subject[0:100]) - if is_src: - m.set_header("Message-ID", "<%s@pld.src.builder>" % r.id) - else: - m.set_header("References", "<%s@pld.src.builder>" % r.id) - m.set_header("In-Reply-To", "<%s@pld.src.builder>" % r.id) + m.write("Request by: %s\n\n" % r.requester_email) for b in r.batches: if b.build_failed and b.logfile == None: info = b.skip_reason @@ -100,6 +93,25 @@ def send_report(r, is_src = False): m.append_log(b.logfile) m.write("\n\n") + m.set_headers(to = r.requester_email, + subject = subject[0:100]) + if is_src: + m.set_header("Message-ID", "" % r.id) + else: + m.set_header("References", "" % r.id) + m.set_header("In-Reply-To", "" % r.id) + m.send() + + m.remove_header("To") + m.remove_header("Cc") + m.set_header("To", config.builder_list) + # reset Message-ID + m.set_std_headers() + if is_src: + m.set_header("Message-ID", "<%s@tld.src.builder>" % r.id) + else: + m.set_header("References", "<%s@tld.src.builder>" % r.id) + m.set_header("In-Reply-To", "<%s@tld.src.builder>" % r.id) m.send() def send_cia_report(r, is_src = False): @@ -112,12 +124,12 @@ def send_cia_report(r, is_src = False): m.set_headers(to = config.bot_email, subject = subject) - m.set_header("Message-ID", "<%s@pld.src.builder>" % r.id) + m.set_header("Message-ID", "<%s@tld.src.builder>" % r.id) m.set_header("X-mailer", "$Id$") - m.set_header("X-builder", "PLD") + m.set_header("X-builder", "TLD") # get header of xml message from file - f = open(path.root_dir + '/PLD_Builder/cia-head.xml') + f = open(path.root_dir + '/TLD_Builder/cia-head.xml') m.write(f.read()) f.close() @@ -131,7 +143,7 @@ def send_cia_report(r, is_src = False): m.write('\n') # get footer of xml message from file - f = open(path.root_dir + '/PLD_Builder/cia-foot.xml') + f = open(path.root_dir + '/TLD_Builder/cia-foot.xml') m.write(f.read()) f.close() diff --git a/PLD_Builder/request.py b/TLD_Builder/request.py similarity index 75% rename from PLD_Builder/request.py rename to TLD_Builder/request.py index 9396394..27bc426 100644 --- a/PLD_Builder/request.py +++ b/TLD_Builder/request.py @@ -1,6 +1,7 @@ # vi: encoding=utf-8 ts=8 sts=4 sw=4 et from xml.dom.minidom import * +from datetime import datetime import string import time import xml.sax.saxutils @@ -8,6 +9,8 @@ import fnmatch import os import urllib import cgi +import pytz +import tempfile import util import log @@ -36,6 +39,12 @@ def attr(e, a, default = None): def escape(s): return xml.sax.saxutils.escape(s) +# return date in iso8601 format +def iso8601(ts, timezone='UTC'): + tz = pytz.timezone(timezone) + dt = datetime.fromtimestamp(ts, tz) + return dt.isoformat() + def is_blank(e): return e.nodeType == Element.TEXT_NODE and string.strip(e.nodeValue) == "" @@ -96,22 +105,25 @@ class Group: def dump_html(self, f): f.write( - "
\n" - "%(no)d. %(time)s from %(requester)s " + "
\n" + "%(no)d. " + " " + "from %(requester)s " "%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s\n" % { 'no': self.no, 'id': '%(id)s' % {'id': self.id}, 'time': escape(time.strftime("%a %b %d %Y %H:%M:%S %z", time.localtime(self.time))), + 'datetime': escape(iso8601(self.time)), 'requester': escape(self.requester), 'priority': self.priority, 'max_jobs': self.max_jobs, 'flags': string.join(self.flags) }) - f.write("
    \n") + f.write("
      \n") for b in self.batches: b.dump_html(f, self.id) - f.write("
\n") + f.write("\n") f.write("
\n") def write_to(self, f): @@ -143,6 +155,7 @@ class Batch: self.builders_status_time = {} self.builders_status_buildtime = {} self.kernel = "" + self.defines = {} self.target = [] self.branch = "" self.src_rpm = "" @@ -155,6 +168,17 @@ class Batch: self.b_id = attr(e, "id") self.depends_on = string.split(attr(e, "depends-on")) self.upgraded = True + + self.parse_xml(e) + + self.__topdir = None + + def get_topdir(self): + if not self.__topdir: + self.__topdir = tempfile.mkdtemp(prefix='B.', dir='/tmp') + return self.__topdir + + def parse_xml(self, e): for c in e.childNodes: if is_blank(c): continue @@ -174,6 +198,9 @@ class Batch: self.info = text(c) elif c.nodeName == "kernel": self.kernel = text(c) + elif c.nodeName == "define": + define = attr(c, "name") + self.defines[define] = text(c) elif c.nodeName == "target": self.target.append(text(c)) elif c.nodeName == "skip": @@ -193,6 +220,21 @@ class Batch: else: log.panic("xml: evil batch child (%s)" % c.nodeName) + def get_package_name(self): + if len(self.spec) <= 5: + return None + return self.spec[:-5] + + def tmpdir(self): + """ + return tmpdir for this batch job building + """ + # it's better to have TMPDIR and BUILD dir on same partition: + # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2 + # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory + path = os.path.join(self.get_topdir(), 'BUILD', 'tmp') + return path + def is_done(self): ok = 1 for b in self.builders: @@ -205,6 +247,7 @@ class Batch: f.write(" batch: %s/%s\n" % (self.src_rpm, self.spec)) f.write(" info: %s\n" % self.info) f.write(" kernel: %s\n" % self.kernel) + f.write(" defines: %s\n" % self.defines_string()) f.write(" target: %s\n" % self.target_string()) f.write(" branch: %s\n" % self.branch) f.write(" bconds: %s\n" % self.bconds_string()) @@ -221,23 +264,23 @@ class Batch: if self.is_command(): desc = "SH: 
%s
flags: [%s]" % (self.command, ' '.join(self.command_flags)) else: - cmd = "/usr/bin/git ls-remote --heads git://git.tld-linux.org/packages/%s 1>/dev/null 2>&1" % (self.spec[:-5]) - r = call(cmd, shell=True) - if r == 0: - dist = "tld" - else: - dist = "pld" - package_url = "http://git.%(dist)s-linux.org/?p=packages/%(package)s.git;a=blob;f=%(spec)s;hb=%(branch)s" % { - 'dist': dist, - 'spec': self.spec, - 'branch': self.branch, - 'package': self.spec[:-5], + cmd = "/usr/bin/git ls-remote --heads git://git.tld-linux.org/packages/%s 1>/dev/null 2>&1" % (self.spec[:-5]) + r = call(cmd, shell=True) + if r == 0: + dist = "tld" + else: + dist = "pld" + package_url = "http://git.%(dist)s-linux.org/?p=packages/%(package)s.git;a=blob;f=%(spec)s;hb=%(branch)s" % { + 'dist': dist, + 'spec': urllib.quote(self.spec), + 'branch': urllib.quote(self.branch), + 'package': urllib.quote(self.spec[:-5]), } - desc = "%(src_rpm)s (%(spec)s -r %(branch)s%(bconds)s)" % { + desc = "%(src_rpm)s (%(spec)s -r %(branch)s%(rpmopts)s)" % { 'src_rpm': self.src_rpm, 'spec': self.spec, 'branch': self.branch, - 'bconds': self.bconds_string() + self.kernel_string() + self.target_string(), + 'rpmopts': self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string(), 'package_url': package_url, } f.write("%s [" % desc) @@ -269,8 +312,8 @@ class Batch: bld = lin_ar.split('-') tree_name = '-'.join(bld[:-1]) tree_arch = '-'.join(bld[-1:]) - link_pre = "" \ - % (urllib.quote(tree_name), urllib.quote(tree_arch), urllib.quote(bl_name), urllib.quote(rid)) + link_pre = "" \ + % (urllib.quote(tree_name), urllib.quote(tree_arch), urllib.quote(bl_name), urllib.quote(rid)) link_post = "" def ftime(s): @@ -298,13 +341,62 @@ class Batch: """ return all rpmbuild options related to this build """ - bconds = self.bconds_string() + self.kernel_string() + self.target_string() + rpmopts = self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string() rpmdefs = \ - "--define '_topdir %(echo $HOME/rpm)' " \ - "--define '_specdir %{_topdir}/packages/%{name}' " \ + "--define '_topdir %s' " % self.get_topdir() + \ + "--define '_specdir %{_topdir}' " \ "--define '_sourcedir %{_specdir}' " \ - "--define '_builddir %{_topdir}/BUILD/%{name}' " - return rpmdefs + bconds + "--define '_rpmdir %{_topdir}/RPMS' " \ + "--define '_builddir %{_topdir}/BUILD' " + return rpmdefs + rpmopts + + def php_ignores(self): + # transform php package name (52) to version (5.2) + def php_name_to_ver(v): + return '.'.join(list(v)) + + # transform php version (5.2) to package name (52) + def php_ver_to_name(v): + return v.replace('.', '') + + # available php versions in distro + php_versions = ['4', '5.2', '5.3', '5.4', '5.5', '5.6', '7.0'] + + # current version if -D php_suffix is present + php_version = php_name_to_ver(self.defines['php_suffix']) + + # remove current php version + try: + php_versions.remove(php_version) + except ValueError: + log.notice("Attempt to remove inexistent key '%s' from %s" % (php_version, php_versions)) + pass + + # map them to poldek ignores + # always ignore hhvm + res = ['hhvm-*'] + for v in map(php_ver_to_name, php_versions): + res.append("php%s-*" % v) + + return res + + # build ignore package list + # currently only php ignore is filled based on build context + def ignores(self): + ignores = [] + + # add php version based ignores + if self.defines.has_key('php_suffix'): + ignores.extend(self.php_ignores()) + + # return empty string if the list is empty + if len(ignores) == 0: + return "" + + def add_ignore(s): + return "--ignore=%s" % s + + return " ".join(map(add_ignore, ignores)) def kernel_string(self): r = "" @@ -326,6 +418,18 @@ class Batch: r = r + " --without " + b return r + def defines_string(self): + r = "" + for key,value in self.defines.items(): + r += " --define '%s %s'" % (key, value) + return r + + def defines_xml(self): + r = "" + for key,value in self.defines.items(): + r += "%s\n" % (escape(key), escape(value)) + return r + def default_target(self, arch): self.target.append("%s-tld-linux" % arch) @@ -349,6 +453,8 @@ class Batch: f.write(" %s\n" % escape(b)) for b in self.bconds_without: f.write(" %s\n" % escape(b)) + if self.defines: + f.write(" %s\n" % self.defines_xml()) for b in self.builders: if self.builders_status_buildtime.has_key(b): t = self.builders_status_buildtime[b] diff --git a/PLD_Builder/request_fetcher.py b/TLD_Builder/request_fetcher.py similarity index 94% rename from PLD_Builder/request_fetcher.py rename to TLD_Builder/request_fetcher.py index a677a71..9877cde 100644 --- a/PLD_Builder/request_fetcher.py +++ b/TLD_Builder/request_fetcher.py @@ -18,6 +18,7 @@ import gpg import request import loop import socket +import struct from acl import acl from bqueue import B_Queue from config import config, init_conf @@ -74,7 +75,12 @@ def fetch_queue(control_url): f.close() sio.seek(0) f = gzip.GzipFile(fileobj = sio) - (signers, body) = gpg.verify_sig(f.read()) + try: + fdata = f.read() + except struct.error, e: + log.alert("corrupted fetched queue.gz file") + sys.exit(1) + (signers, body) = gpg.verify_sig(fdata) u = acl.user_by_email(signers) if u == None: log.alert("queue.gz not signed with signature of valid user: %s" % signers) diff --git a/PLD_Builder/request_handler.py b/TLD_Builder/request_handler.py similarity index 86% rename from PLD_Builder/request_handler.py rename to TLD_Builder/request_handler.py index 2c38e83..033cf39 100644 --- a/PLD_Builder/request_handler.py +++ b/TLD_Builder/request_handler.py @@ -16,9 +16,11 @@ import util import wrap import status from acl import acl +from blacklist import blacklist from lock import lock from bqueue import B_Queue from config import config, init_conf +from mailer import Message def check_double_id(id): id_nl = id + "\n" @@ -45,7 +47,8 @@ def handle_group(r, user): else: spec = "None.spec" log.error("%s: %s" % (spec, msg)) - m = user.message_to() + m = Message() + m.set_headers(to = r.requester_email, cc = config.builder_list) m.set_headers(subject = "building %s failed" % spec) m.write_line(msg) m.send() @@ -55,6 +58,15 @@ def handle_group(r, user): lockf.close() return + try: + if (user.change_requester and r.requester): + user = acl.user_by_login(r.requester) + except KeyError: + r.requester += '/' + user.get_login() + else: + r.requester = user.get_login() + r.requester_email = user.mail_to() + for batch in r.batches: if not user.can_do("src", config.builder, batch.branch): @@ -112,10 +124,21 @@ def handle_group(r, user): % (user.get_login(), pkg, bld, batch.branch)) lockf.close() return + if not "test-build" in r.flags and not user.can_do("ready", bld, batch.branch): + fail_mail("user %s is not allowed to send ready builds (ready:%s:%s)" \ + % (user.get_login(), bld, batch.branch)) + lockf.close() + return + + pkg = batch.spec + if pkg.endswith(".spec"): + pkg = pkg[:-5] + if not "test-build" in r.flags and blacklist.package(pkg): + fail_mail("package '%s' is blacklisted, only test-builds allowed" % pkg) + lockf.close() + return r.priority = user.check_priority(r.priority,config.builder) - r.requester = user.get_login() - r.requester_email = user.mail_to() r.time = time.time() log.notice("queued %s from %s" % (r.id, user.get_login())) q = B_Queue(path.queue_file) @@ -190,6 +213,7 @@ def handle_request(req, filename = None): def handle_request_main(req, filename = None): acl.try_reload() + blacklist.try_reload() init_conf("src") status.push("handling email request") ret = handle_request(req, filename = filename) diff --git a/PLD_Builder/request_handler_server b/TLD_Builder/request_handler_server similarity index 60% rename from PLD_Builder/request_handler_server rename to TLD_Builder/request_handler_server index 4abb47d..0c1f265 100755 --- a/PLD_Builder/request_handler_server +++ b/TLD_Builder/request_handler_server @@ -24,16 +24,17 @@ if [ $daemon = 0 -a $attach = 0 ]; then fi if [ "$daemon" = 1 ]; then - echo "Starting request_handler_server" + echo "Starting request_handler_server(s)" rm -f $sock cd $(dirname $sock) - exec dtach -n $(basename $sock) -r none python request_handler_server.py + tmux -S $sock new-session -d -s "request_handler_server" -n "http" "python request_handler_server.py" && \ + exec tmux -S $sock new-window -d -t "request_handler_server" -n "https" "python request_handler_server.py ssl" exit 1 fi # attach to session if [ "$attach" = 1 ]; then - echo "Attaching to request_handler_server" - exec dtach -a $sock + echo "Attaching to request_handler_server(s)" + exec tmux -S $sock attach exit 1 fi diff --git a/TLD_Builder/request_handler_server.py b/TLD_Builder/request_handler_server.py new file mode 100644 index 0000000..ece2f96 --- /dev/null +++ b/TLD_Builder/request_handler_server.py @@ -0,0 +1,119 @@ +#!/usr/bin/python + +import socket +import string +import cgi +import time +import log +import ssl +import sys +import traceback +import os +from config import config, init_conf + +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer + +import request_handler +import path + +class MyHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_error(401); + + def do_POST(self): + global rootnode + try: + length = int(self.headers.getheader('content-length')) + filename = self.headers.getheader('x-filename') + if not length: + log.error("request_handler_server: [%s]: 401 Unauthorized" % self.client_address[0]) + self.send_error(401) + self.end_headers() + return + + query = self.rfile.read(length) + + if not request_handler.handle_request_main(query, filename = filename): + error = log.last_log(); + log.error("request_handler_server: [%s]: handle_request_main(..., %s) failed" % (self.client_address[0], filename)) + self.send_error(500, "%s: request failed. %s" % (filename, error)) + self.end_headers() + return + + self.send_response(200) + self.end_headers() + + except Exception, e: + self.send_error(500, "%s: %s" % (filename, e)) + self.end_headers() + log.error("request_handler_server: [%s]: exception: %s\n%s" % (self.client_address[0], e, traceback.format_exc())) + raise + pass + +def write_css(): + css_src = os.path.join(os.path.dirname(__file__), 'style.css') + css_file = path.www_dir + "/style.css" + # skip if file exists and code is not newer + if os.path.exists(css_file) and os.stat(css_src).st_mtime < os.stat(css_file).st_mtime: + return + + old_umask = os.umask(0022) + r = open(css_src, 'r') + f = open(css_file, "w") + f.write(r.read()) + f.close() + r.close() + os.umask(old_umask) + +def write_js(): + js_src = os.path.join(os.path.dirname(__file__), 'script.js') + js_file = path.www_dir + '/script.js' + # skip if file exists and code is not newer + if os.path.exists(js_file) and os.stat(js_src).st_mtime < os.stat(js_file).st_mtime: + return + + old_umask = os.umask(0022) + r = open(js_src, 'r') + f = open(js_file, 'w') + f.write(r.read()) + f.close() + r.close() + os.umask(old_umask) + +def main(srv_ssl=False): + write_css(); + write_js(); + socket.setdefaulttimeout(30) + try: + init_conf() + host = "" + port = config.request_handler_server_port + if srv_ssl: + port = config.request_handler_server_ssl_port + + try: + server = HTTPServer((host, port), MyHandler) + if srv_ssl: + server.socket = ssl.wrap_socket (server.socket, + keyfile = config.request_handler_server_ssl_key, + certfile = config.request_handler_server_ssl_cert, + ca_certs = "/etc/certs/ca-certificates.crt", + server_side=True) + except Exception, e: + log.notice("request_handler_server: can't start server on [%s:%d], ssl=%s: %s" % (host, port, str(srv_ssl), e)) + print >> sys.stderr, "ERROR: Can't start server on [%s:%d], ssl=%s: %s" % (host, port, str(srv_ssl), e) + sys.exit(1) + + log.notice('request_handler_server: started on [%s:%d], ssl=%s...' % (host, port, str(srv_ssl))) + server.serve_forever() + except KeyboardInterrupt: + log.notice('request_handler_server: ^C received, shutting down server') + server.socket.close() + +if __name__ == '__main__': + srv_ssl = False + if len(sys.argv) == 2 and sys.argv[1] == "ssl": + srv_ssl = True + + main(srv_ssl) diff --git a/PLD_Builder/rpm_builder.py b/TLD_Builder/rpm_builder.py similarity index 84% rename from PLD_Builder/rpm_builder.py rename to TLD_Builder/rpm_builder.py index e42c085..12cc265 100644 --- a/PLD_Builder/rpm_builder.py +++ b/TLD_Builder/rpm_builder.py @@ -106,17 +106,22 @@ def fetch_src(r, b): b.log_line(msg) return False except urllib2.URLError, error: - # see errno.h - try: - errno = error.errno - except AttributeError: - # python 2.4 - errno = error.reason[0] + errno = 0 + if isinstance(error.args[0], IOError): + errno = error.args[0].errno if errno in [-3, 60, 61, 110, 111]: b.log_line("unable to connect to %s... trying again" % (src_url)) continue else: + try: + print "error.errno: %s" % str(error.errno) + except Exception, e: + print "error.errno: exception %s" % e + try: + print "error.reason %s" % str(error.reason) + except Exception, e: + print "error.reason exception %s" % e raise o = chroot.popen("cat > %s" % b.src_rpm, mode = "w") @@ -135,7 +140,7 @@ def fetch_src(r, b): else: b.log_line("fetched %d bytes, %.1f K/s" % (bytes, bytes / 1024.0 / t)) -def prepare_env(): +def prepare_env(logfile = None): chroot.run(""" test ! -f /proc/uptime && mount /proc 2>/dev/null test ! -c /dev/full && rm -f /dev/full && mknod -m 666 /dev/full c 1 7 @@ -145,7 +150,7 @@ def prepare_env(): test ! -c /dev/zero && rm -f /dev/zero && mknod -m 666 /dev/zero c 1 5 # need entry for "/" in mtab, for diskspace() to work in rpm - [ -z $(awk '$2 == "/" {print $1}' /etc/mtab) ] && mount -f -t rootfs rootfs / + [ -z $(awk '$2 == "/" {print $1; exit}' /etc/mtab) ] && mount -f -t rootfs rootfs / # make neccessary files readable for builder user # TODO: see if they really aren't readable for builder @@ -156,16 +161,16 @@ def prepare_env(): # try to limit network access for builder account /bin/setfacl -m u:builder:--- /etc/resolv.conf - """, 'root') + """, 'root', logfile = logfile) def build_rpm(r, b): - if len(b.spec) <= 5: + packagename = b.get_package_name() + if not packagename: # should not really get here b.log_line("error: No .spec not given of malformed: '%s'" % b.spec) res = "FAIL_INTERNAL" return res - packagename = b.spec[:-5] status.push("building %s (%s)" % (b.spec, packagename)) b.log_line("request from: %s" % r.requester) @@ -178,38 +183,33 @@ def build_rpm(r, b): fetch_src(r, b) b.log_line("installing srpm: %s" % b.src_rpm) res = chroot.run(""" - # b.id %(bid)s set -ex; - install -d rpm/packages/%(package)s rpm/BUILD/%(package)s; - rpm -Uhv %(rpmdefs)s %(src_rpm)s; + install -d %(topdir)s/{BUILD,RPMS}; + LC_ALL=en_US.UTF-8 rpm -qp --changelog %(src_rpm)s; + rpm -Uhv --nodeps %(rpmdefs)s %(src_rpm)s; rm -f %(src_rpm)s; """ % { - 'bid' : b.b_id, - 'package' : packagename, + 'topdir' : b.get_topdir(), 'rpmdefs' : b.rpmbuild_opts(), 'src_rpm' : b.src_rpm }, logfile = b.logfile) b.files = [] - # it's better to have TMPDIR and BUILD dir on same partition: - # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2 - # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory - tmpdir = os.environ.get('HOME') + "/rpm/BUILD/%s/tmp" % packagename + tmpdir = b.tmpdir() if res: b.log_line("error: installing src rpm failed") res = "FAIL_SRPM_INSTALL" else: prepare_env() - chroot.run("install -m 700 -d %s" % tmpdir) - + chroot.run("set -x; install -m 700 -d %s" % tmpdir, logfile=b.logfile) b.default_target(config.arch) # check for build arch before filling BR cmd = "set -ex; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \ - "rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' rpm/packages/%(package)s/%(spec)s" % { + "rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' %(topdir)s/%(spec)s" % { 'tmpdir': tmpdir, 'nice' : config.nice, + 'topdir' : b.get_topdir(), 'rpmdefs' : b.rpmbuild_opts(), - 'package' : packagename, 'spec': b.spec, } res = chroot.run(cmd, logfile = b.logfile) @@ -227,12 +227,12 @@ def build_rpm(r, b): if r.max_jobs > 0: max_jobs = max(min(config.max_jobs, r.max_jobs), 1) cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \ - "rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' %(rpmdefs)s rpm/packages/%(package)s/%(spec)s" % { + "rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' --define '_make_opts -Otarget' --define '_tld_builder 1' %(rpmdefs)s %(topdir)s/%(spec)s" % { 'r_id' : r.id, 'tmpdir': tmpdir, 'nice' : config.nice, 'rpmdefs' : b.rpmbuild_opts(), - 'package' : packagename, + 'topdir' : b.get_topdir(), 'max_jobs' : max_jobs, 'spec': b.spec, } @@ -243,7 +243,7 @@ def build_rpm(r, b): b.log_line("ended at: %s, done in %s" % (time.asctime(), datetime.timedelta(0, end_time - begin_time))) if res: res = "FAIL" - files = util.collect_files(b.logfile) + files = util.collect_files(b.logfile, basedir = b.get_topdir()) if len(files) > 0: r.chroot_files.extend(files) else: @@ -255,14 +255,14 @@ def build_rpm(r, b): res = "FAIL_%s" % last_section.upper() b.files = files + # cleanup tmp and build files chroot.run(""" set -ex; - rpmbuild %(rpmdefs)s --nodeps --nobuild --clean --rmspec --rmsource rpm/packages/%(package)s/%(spec)s - rm -rf %(tmpdir)s; - chmod -R u+rwX rpm/BUILD/%(package)s; - rm -rf rpm/BUILD/%(package)s; - """ % - {'tmpdir' : tmpdir, 'spec': b.spec, 'package' : packagename, 'rpmdefs' : b.rpmbuild_opts()}, logfile = b.logfile) + chmod -R u+rwX %(topdir)s/BUILD; + rm -rf %(topdir)s/{tmp,BUILD} + """ % { + 'topdir' : b.get_topdir(), + }, logfile = b.logfile) def ll(l): util.append_to(b.logfile, l) @@ -279,18 +279,26 @@ def build_rpm(r, b): ) else: ll("test-build: not copying to " + rpm_cache_dir) - ll("Begin-PLD-Builder-Info") + ll("Begin-TLD-Builder-Info") if "upgrade" in r.flags: b.upgraded = install.upgrade_from_batch(r, b) else: ll("not upgrading") - ll("End-PLD-Builder-Info") + ll("End-TLD-Builder-Info") for f in b.files: local = r.tmp_dir + os.path.basename(f) chroot.cp(f, outfile = local, rm = True) ftp.add(local) + # cleanup all remains from this build + chroot.run(""" + set -ex; + rm -rf %(topdir)s; + """ % { + 'topdir' : b.get_topdir(), + }, logfile = b.logfile) + def uploadinfo(b): c="file:SRPMS:%s\n" % b.src_rpm for f in b.files: @@ -367,6 +375,14 @@ def main_for(builder): f.close() l.close() else: + # be able to avoid locking with very low priority + if req.priority > -1000: + # don't kill server + check_load() + # allow only one build in given builder at once + if not lock.lock("building-high-priority-rpm-for-%s" % config.builder, non_block = 1): + return + msg = "HIGH PRIORITY: " msg += "handling request %s (%d) for %s from %s, priority %s" \ diff --git a/PLD_Builder/srpm_builder.py b/TLD_Builder/srpm_builder.py similarity index 90% rename from PLD_Builder/srpm_builder.py rename to TLD_Builder/srpm_builder.py index 42e3b6b..39ac93d 100644 --- a/PLD_Builder/srpm_builder.py +++ b/TLD_Builder/srpm_builder.py @@ -90,7 +90,7 @@ def transfer_file(r, b): ftp.add(fname, "uploadinfo") def build_srpm(r, b): - if len(b.spec) == 0: + if len(b.spec) <= len('.spec'): # should not really get here util.append_to(b.logfile, "error: No .spec given but build src.rpm wanted") return "FAIL" @@ -98,14 +98,14 @@ def build_srpm(r, b): status.push("building %s" % b.spec) b.src_rpm = "" - builder_opts = "-nu -nm --nodeps --http" - if ("test-build" in r.flags) or b.branch and b.branch.startswith(config.tag_prefixes[0]): - tag_test="" + builder_opts = "-nu -nm --nodeps --http --define \'_tld_builder 1\'" + if ("test-build" in r.flags): + tag_test="" else: - tag_test=" -Tp %s -tt" % (config.tag_prefixes[0],) - cmd = ("cd rpm/packages; nice -n %s ./builder %s -bs %s -r %s %s %s %s 2>&1" % + tag_test=" -Tp %s -tt" % (config.tag_prefixes[0],) + cmd = ("cd rpm/packages; nice -n %s ./builder %s -bs %s -r %s %s %s %s %s 2>&1" % (config.nice, builder_opts, b.bconds_string(), b.branch, - tag_test, b.kernel_string(), b.spec)) + tag_test, b.kernel_string(), b.defines_string(), b.spec)) util.append_to(b.logfile, "request from: %s" % r.requester) util.append_to(b.logfile, "started at: %s" % time.asctime()) util.append_to(b.logfile, "building SRPM using: %s\n" % cmd) @@ -126,8 +126,8 @@ def build_srpm(r, b): if res == 0 and not "test-build" in r.flags: for pref in config.tag_prefixes: util.append_to(b.logfile, "Tagging with prefix: %s" % pref) - res = chroot.run("cd rpm/packages; ./builder -r %s -Tp %s -Tv --pkgrev %s" % \ - (b.branch, pref, b.spec), logfile = b.logfile) + res = chroot.run("cd rpm/packages; ./builder -bs -r %s -Tp %s -Tv --pkgrev %s" % \ + (b.bconds_string(), b.branch, pref, b.defines_string(), b.spec), logfile = b.logfile) if res == 0: transfer_file(r, b) diff --git a/PLD_Builder/status.py b/TLD_Builder/status.py similarity index 100% rename from PLD_Builder/status.py rename to TLD_Builder/status.py diff --git a/PLD_Builder/stopwatch.py b/TLD_Builder/stopwatch.py similarity index 100% rename from PLD_Builder/stopwatch.py rename to TLD_Builder/stopwatch.py diff --git a/PLD_Builder/util.py b/TLD_Builder/util.py similarity index 86% rename from PLD_Builder/util.py rename to TLD_Builder/util.py index 316a8f1..b623115 100644 --- a/PLD_Builder/util.py +++ b/TLD_Builder/util.py @@ -49,11 +49,14 @@ def clean_tmp(dir): # FIXME: use python os.system("rm -f %s/* 2>/dev/null; rmdir %s 2>/dev/null" % (dir, dir)) -def collect_files(log): +def collect_files(log, basedir = "/home"): f = open(log, 'r') - rx = re.compile(r"^Wrote: (/home.*\.rpm)$") + rx = re.compile(r"^Wrote: (%s.*\.rpm)$" % basedir) + proc = re.compile(r"^Processing (files):.*$") files = [] - for l in f.xreadlines(): + for l in reversed(list(f.xreadlines())): + if proc.match(l): + break m = rx.search(l) if m: files.append(m.group(1)) diff --git a/PLD_Builder/wrap.py b/TLD_Builder/wrap.py similarity index 97% rename from PLD_Builder/wrap.py rename to TLD_Builder/wrap.py index b9809a3..0c9df39 100644 --- a/PLD_Builder/wrap.py +++ b/TLD_Builder/wrap.py @@ -27,7 +27,7 @@ except: To: %s Cc: %s, %s Date: %s -X-PLD-Builder: fatal error report +X-TLD-Builder: fatal error report %s diff --git a/admin/fresh-queue.sh b/admin/fresh-queue.sh index 5d80086..c971c50 100755 --- a/admin/fresh-queue.sh +++ b/admin/fresh-queue.sh @@ -2,9 +2,9 @@ umask 077 -CONFIG=$HOME/.pldbuilderrc +CONFIG=$HOME/.tldbuilderrc [ -f "$CONFIG" ] && . $CONFIG -[ -n "$BUILDERPATH" ] || BUILDERPATH="$HOME/pld-builder.new/" +[ -n "$BUILDERPATH" ] || BUILDERPATH="$HOME/tld-builder.new/" export BUILDERPATH if [ -d "$BUILDERPATH" ]; then diff --git a/admin/install-chroot.sh b/admin/install-chroot.sh index f8ec492..306780d 100755 --- a/admin/install-chroot.sh +++ b/admin/install-chroot.sh @@ -1,7 +1,7 @@ #!/bin/sh -DIST="th" -DISTTAG="PLD 3.0 (Th)" +DIST="ti" +DISTTAG="TLD Linux" die () { echo "$0: $*" 1>&2 @@ -21,9 +21,9 @@ EOF } default_config () { - builder_pkgs="rpm-build poldek pwdutils net-tools which rpm-perlprov rpm-php-pearprov rpm-pythonprov bash vim" + builder_pkgs="rpm-build poldek pwdutils net-tools which rpm-perlprov rpm-php-pearprov rpm-pythonprov bash vim util-linux" builder_uid=`id -u` - dist_url="ftp://ftp.$DIST.pld-linux.org" + dist_url="ftp://ftp.tld-linux.org" case "$chroot_type" in src ) @@ -67,13 +67,13 @@ EOF [source] name=main type=pndir -path=$dist_url/dists/$DIST/PLD/$arch/RPMS/ +path=$dist_url/TLD/stable/main/$arch/RPMS/ pri=6 [source] name=main type=pndir -path=$dist_url/dists/$DIST/PLD/noarch/RPMS/ +path=$dist_url/TLD/stable/main/noarch/RPMS/ pri=6 EOF @@ -104,13 +104,13 @@ set -x rm -rf rpm mkdir rpm cd rpm -git clone $git_server/rpm-build-tools rpm-build-tools +git clone $git_server/packages/rpm-build-tools rpm-build-tools ./rpm-build-tools/builder.sh --init-rpm-dir -echo "%packager PLD bug tracking system ( http://bugs.pld-linux.org/ )">~/.rpmmacros -echo "%vendor PLD">>~/.rpmmacros +echo "%packager TLD bug tracking system ( http://bugs.tld-linux.org/ )">~/.rpmmacros +echo "%vendor TLD">>~/.rpmmacros echo "%distribution $DISTTAG">>~/.rpmmacros git config --global user.name $git_user -git config --global user.email ${git_user}@pld-linux.org +git config --global user.email ${git_user}@tld-linux.org EOF chb "sh" < install-specs rm install-specs @@ -124,8 +124,8 @@ rm -rf rpm mkdir rpm cd rpm mkdir SPECS SOURCES SRPMS RPMS BUILD -echo "%packager PLD bug tracking system ( http://bugs.pld-linux.org/ )">~/.rpmmacros -echo "%vendor PLD">>~/.rpmmacros +echo "%packager TLD bug tracking system ( http://bugs.tld-linux.org/ )">~/.rpmmacros +echo "%vendor TLD">>~/.rpmmacros echo "%distribution $DISTTAG">>~/.rpmmacros EOF chb "sh" < install-bt @@ -151,7 +151,7 @@ cachedir = $chroot_dir/spools/poldek keep_downloads = no EOF -cat > install-$chroot_name.sh < install-$chroot_type.sh < /etc/resolv.conf" < /etc/resolv.conf chr "cat > /etc/mtab" < /dev/null chr "mkdir -p /spools/ready/" < /dev/null diff --git a/bin/file-sender.sh b/bin/file-sender.sh index 5babb4c..3f75f49 100755 --- a/bin/file-sender.sh +++ b/bin/file-sender.sh @@ -3,10 +3,13 @@ umask 022 export LC_CTYPE=en_US.iso-8859-1 -CONFIG=$HOME/.pldbuilderrc +CONFIG=$HOME/.tldbuilderrc [ -f "$CONFIG" ] && . $CONFIG -[ -n "$BUILDERPATH" ] || BUILDERPATH="$HOME/pld-builder.new/" +if [ -z "$BUILDERPATH" ]; then + dir=$(dirname "$0") + BUILDERPATH="$(cd "$dir"/..; pwd)" +fi export BUILDERPATH cd $BUILDERPATH -exec python PLD_Builder/file_sender.py +exec python TLD_Builder/file_sender.py diff --git a/bin/load-balancer.sh b/bin/load-balancer.sh index 055306a..269b9fa 100755 --- a/bin/load-balancer.sh +++ b/bin/load-balancer.sh @@ -3,10 +3,13 @@ umask 022 export LC_CTYPE=en_US.iso-8859-1 -CONFIG=$HOME/.pldbuilderrc +CONFIG=$HOME/.tldbuilderrc [ -f "$CONFIG" ] && . $CONFIG -[ -n "$BUILDERPATH" ] || BUILDERPATH="$HOME/pld-builder.new/" +if [ -z "$BUILDERPATH" ]; then + dir=$(dirname "$0") + BUILDERPATH="$(cd "$dir"/..; pwd)" +fi export BUILDERPATH cd $BUILDERPATH -exec python PLD_Builder/load_balancer.py +exec python TLD_Builder/load_balancer.py diff --git a/bin/maintainer.sh b/bin/maintainer.sh index e490b02..7eecd21 100755 --- a/bin/maintainer.sh +++ b/bin/maintainer.sh @@ -3,10 +3,13 @@ umask 022 export LC_CTYPE=en_US.iso-8859-1 -CONFIG=$HOME/.pldbuilderrc +CONFIG=$HOME/.tldbuilderrc [ -f "$CONFIG" ] && . $CONFIG -[ -n "$BUILDERPATH" ] || BUILDERPATH="$HOME/pld-builder.new/" +if [ -z "$BUILDERPATH" ]; then + dir=$(dirname "$0") + BUILDERPATH="$(cd "$dir"/..; pwd)" +fi export BUILDERPATH cd $BUILDERPATH -exec python PLD_Builder/maintainer.py +exec python TLD_Builder/maintainer.py diff --git a/bin/request-fetcher.sh b/bin/request-fetcher.sh index 1b446f7..6efc48b 100755 --- a/bin/request-fetcher.sh +++ b/bin/request-fetcher.sh @@ -3,10 +3,13 @@ umask 022 export LC_CTYPE=en_US.iso-8859-1 -CONFIG=$HOME/.pldbuilderrc +CONFIG=$HOME/.tldbuilderrc [ -f "$CONFIG" ] && . $CONFIG -[ -n "$BUILDERPATH" ] || BUILDERPATH="$HOME/pld-builder.new/" +if [ -z "$BUILDERPATH" ]; then + dir=$(dirname "$0") + BUILDERPATH="$(cd "$dir"/..; pwd)" +fi export BUILDERPATH cd $BUILDERPATH -exec python PLD_Builder/request_fetcher.py +exec python TLD_Builder/request_fetcher.py diff --git a/bin/request-handler.sh b/bin/request-handler.sh index 8c2ab85..e0aa6e2 100755 --- a/bin/request-handler.sh +++ b/bin/request-handler.sh @@ -2,15 +2,18 @@ umask 022 export LC_CTYPE=en_US.iso-8859-1 -CONFIG=$HOME/.pldbuilderrc +CONFIG=$HOME/.tldbuilderrc [ -f "$CONFIG" ] && . $CONFIG -[ -n "$BUILDERPATH" ] || BUILDERPATH="$HOME/pld-builder.new/" +if [ -z "$BUILDERPATH" ]; then + dir=$(dirname "$0") + BUILDERPATH="$(cd "$dir"/..; pwd)" +fi export BUILDERPATH if lockfile -r3 $HOME/.builder_request_handler.lock 2>/dev/null; then trap "rm -f $HOME/.builder_request_handler.lock" 1 2 3 13 15 cd $BUILDERPATH - python PLD_Builder/request_handler.py + python TLD_Builder/request_handler.py rm -f $HOME/.builder_request_handler.lock else return 1 diff --git a/bin/rpm-builder.sh b/bin/rpm-builder.sh index 7e678e2..5f52b1c 100755 --- a/bin/rpm-builder.sh +++ b/bin/rpm-builder.sh @@ -2,10 +2,13 @@ umask 022 export LC_CTYPE=en_US.iso-8859-1 -CONFIG=$HOME/.pldbuilderrc +CONFIG=$HOME/.tldbuilderrc [ -f "$CONFIG" ] && . $CONFIG -[ -n "$BUILDERPATH" ] || BUILDERPATH="$HOME/pld-builder.new/" +if [ -z "$BUILDERPATH" ]; then + dir=$(dirname "$0") + BUILDERPATH="$(cd "$dir"/..; pwd)" +fi export BUILDERPATH cd $BUILDERPATH -exec python PLD_Builder/rpm_builder.py $1 +exec python TLD_Builder/rpm_builder.py $1 diff --git a/bin/src-builder.sh b/bin/src-builder.sh index c17e36b..8114070 100755 --- a/bin/src-builder.sh +++ b/bin/src-builder.sh @@ -2,10 +2,13 @@ umask 022 export LC_CTYPE=en_US.iso-8859-1 -CONFIG=$HOME/.pldbuilderrc +CONFIG=$HOME/.tldbuilderrc [ -f "$CONFIG" ] && . $CONFIG -[ -n "$BUILDERPATH" ] || BUILDERPATH="$HOME/pld-builder.new/" +if [ -z "$BUILDERPATH" ]; then + dir=$(dirname "$0") + BUILDERPATH="$(cd "$dir"/..; pwd)" +fi export BUILDERPATH cd $BUILDERPATH -exec python PLD_Builder/srpm_builder.py +exec python TLD_Builder/srpm_builder.py diff --git a/client/make-request.sh b/client/make-request.sh index d73c08a..a14bd7c 100755 --- a/client/make-request.sh +++ b/client/make-request.sh @@ -1,4 +1,6 @@ #!/bin/sh +# vim:noet:ts=4:sw=4 +VERSION=1.89 # prevent "*" from being expanded in builders var set -f @@ -16,6 +18,8 @@ url= no_depend=no verbose=no autotag=no +requester_override= +relup=no if [ -x /usr/bin/python ]; then send_mode="python" @@ -34,14 +38,14 @@ if [ ! -f "$USER_CFG" ]; then echo "Creating config file $USER_CFG. You *must* edit it." cat > $USER_CFG <> sys.stdout, "Request queued via HTTP." esac } +# htmlspecialchars: escape <, > and & +hsc() { + local input=$1 + echo -E "$input" | sed -e 's,&,\&,g;s,<,\<,g;s,>,\>,g' +} + # simple df_fetcher, based on packages/fetchsrc_request # TODO: tcp (smtp) mode # TODO: adjust for ~/.requestrc config @@ -115,7 +133,7 @@ df_fetch() { local specs="$@" # Sending by - local MAILER='/usr/sbin/sendmail' + local MAILER='/usr/lib/sendmail' # MAILER='/usr/bin/msmtp' # Sending via local VIA="SENDMAIL" @@ -126,23 +144,24 @@ df_fetch() { # VIA_ARGS='-a gmail' # # DISTFILES EMAIL - local DMAIL="distfiles@pld-linux.org" + local DMAIL="df@tld-linux.org" local HOST=$(hostname -f) local LOGIN=${requester%@*} + local SPEC BRANCH for spec in $specs; do - local SPEC=$(echo "$spec" | sed -e 's|:.*||') - local BRANCH=$(echo "$spec" | sed -e 's|.*:||') + SPEC=$(echo "$spec" | sed -e 's|:.*||') + SPEC=${SPEC%.spec} + BRANCH=$(echo "$spec" | sed -e 's|.*:||') echo >&2 "Distfiles Request: $SPEC:$BRANCH via $MAILER ${VIA_ARGS:+ ($VIA_ARGS)}" cat <<-EOF | "$MAILER" -t -i $VIA_ARGS To: $DMAIL From: $LOGIN <$LOGIN@$HOST> Subject: fetchsrc_request notify - X-CVS-Module: SPECS X-distfiles-request: yes X-Login: $LOGIN - X-Spec: $SPEC + X-Package: $SPEC X-Branch: $BRANCH X-Flags: force-reply @@ -160,19 +179,21 @@ autotag() { s=${s%:*} # ensure package ends with .spec s=${s%.spec}.spec - out=$(cvs status -v $s | awk "!/Sticky/&&/auto-$dist-/{if (!a++) print \$1}") + git fetch --tags + out=$(git for-each-ref --count=1 --sort=-authordate refs/tags/auto/$dist \ + --format='%(refname:short)') echo "$s:$out" done } # get autotag for specs -# WARNING: This may checkout some files from CVS +# WARNING: This may checkout some files from VCS get_autotag() { local pkg spec rpmdir rpmdir=$(rpm -E %_topdir) - cd $rpmdir for pkg in "$@"; do + cd $rpmdir # strip branches pkg=${pkg%:*} # strip .spec extension @@ -185,11 +206,17 @@ get_autotag() { # just print it out, to fallback to base pkg name echo "$pkg" else - autotag $pkg/$pkg.spec + cd $pkg + autotag $pkg.spec fi done } +relup() { + local script=$(dirname $(rpm -E %_topdir))/rpm-build-tools/relup.sh + $script -u -i "$@" +} + usage() { cat <$requester_override" + fi + msg "Queue-ID: $id" echo # job to depend on local depend= - local b i=1 + local b i=1 val local name branch builders_xml for b in $builders; do @@ -694,7 +736,7 @@ gen_req() { echo -E >&2 "* Command: $command" echo " " echo " " - echo -E "$command" | sed -e 's,&,\&,g;s,<,\<,g;s,>,\>,g' + hsc "$command" echo "" echo " " echo "$builders_xml" @@ -725,6 +767,17 @@ gen_req() { echo " $branch" echo " ${kernel:+$kernel}" echo " ${target:+$target}" + + oIFS=$IFS; IFS=$cr + for b in $define; do + [ -z "$b" ] && continue + value=${b#*=} + b=${b%%=*} + echo " $(hsc "$value")" + msg "- define $b=$value" + done + IFS=$oIFS + echo " " echo for b in $with; do @@ -750,7 +803,7 @@ gen_req() { echo -E >&2 "* Post-Command: $post_command" echo " " echo " " - echo -E "$post_command" | sed -e 's,&,\&,g;s,<,\<,g;s,>,\>,g' + hsc "$post_command" echo "" echo " " echo "$builders_xml" @@ -776,8 +829,8 @@ gen_email () { To: $builder_email Subject: build request Message-Id: <$id@$(hostname)> - X-New-PLD-Builder: request - X-Requester-Version: \$Id$ + X-New-TLD-Builder: request + X-Requester-Version: $VERSION EOF diff --git a/client/smtpwrapper.py b/client/smtpwrapper.py index ad4ff7b..2827f65 100755 --- a/client/smtpwrapper.py +++ b/client/smtpwrapper.py @@ -11,6 +11,6 @@ msg = sys.stdin.read() server = smtplib.SMTP(smtp_host) # server.set_debuglevel(1) -server.sendmail("matkor@pld-linux.org","builder-ac@pld-linux.org", msg) # Adresses should be taken from .requestrc +server.sendmail("ti-admins@tld-linux.org","builder-src@tld-linux.org", msg) # Adresses should be taken from .requestrc server.quit() diff --git a/config/acl.conf b/config/acl.conf index 81256cc..a24a6bd 100644 --- a/config/acl.conf +++ b/config/acl.conf @@ -1,5 +1,5 @@ # -# Access Control Lists for PLD builder +# Access Control Lists for TLD builder # # Format: # @@ -36,7 +36,7 @@ # sense). Lowest priority allowe will be 3. # # [foo] -# gpg_emails = foo@pld-linux.org Foo.Bar@blah.com +# gpg_emails = foo@tld-linux.org Foo.Bar@blah.com # mailto foo-builder@blah.com # privs = src:src !binary:security-* binary:* 3:* # @@ -50,6 +50,6 @@ gpg_emails = srpms_builder@roke.freak privs = sign_queue:* notify:* [malekith] -gpg_emails = malekith@pld-linux.org +gpg_emails = malekith@tld-linux.org mailto = malekith@roke.freak privs = src:roke-src binary:roke-*:AC-branch diff --git a/config/builder.conf b/config/builder.conf index 98d3b8f..aefdabf 100644 --- a/config/builder.conf +++ b/config/builder.conf @@ -1,19 +1,19 @@ [all] # Builder email (put in From: field when this builder is sending mails). -email = builder-th@pld-linux.org +email = builder-src@tld-linux.org # Admin's email (she gets mails about errors and such). -admin_email = th-admin@pld-linux.org +admin_email = ti-admins@tld-linux.org # To be placed in Cc: of status and error reports (a mailing list usually). builder_list = # List of binary builders. This thing must be present on both src and bin # builders. -binary_builders = th-athlon +binary_builders = ti-i686 ti-x86_64 # Name of the src builder (this field is ignored on binary builders). -src_builder = th-src +src_builder = ti-src # Logs always go to spool/log, if you want syslog in addition, # specify facility here (user or local0-7 is the best choice probably). @@ -28,14 +28,14 @@ sudo_chroot_wrapper = max_keep_time = 168 # TCP port number for request_handler_server.py. ignored on bin builders -request_handler_server_port = 1234 +request_handler_server_port = 1231 -# Send '*.uploadinfo' files along with built rpms (used by pld-ftp-admin), but +# Send '*.uploadinfo' files along with built rpms (used by tld-ftp-admin), but # probably not usefull for small, 'personal' builders. [yes/no] gen_upinfo = yes # CIA bot (see cia.navi.cx). -#bot_email = cia@pld-linux.org +#bot_email = cia@tld-linux.org # ------ Binary builders config: # How many jobs can be run at once (how many concurrent builds). @@ -51,39 +51,26 @@ max_jobs = 1 #nice = 0 # Where to look for queue.gz and counter (published through www by src builder). -control_url = http://src.th.pld-linux.org/ +control_url = http://kraz.tld-linux.org/~builderti/ # Src builder's email for sending status notifications (build ok/failed). -#notify_email = builder-th-src@pld-linux.org +#notify_email = builder-src@tld-linux.org # Where to cache rpm's from binary builders locally rpm_cache_dir = /spools/ready # ------ SRC builder config: # List of name-version-release tag prefixes. -#tag_prefixes = auto- auto-ac- auto-th- -tag_prefixes = auto-th- +tag_prefixes = auto/ti/ # ------ Configs for particular builders: -[th-src] +[ti-src] arch = i686 -chroot = /home/pld/builderth/chroots/chroot-src/ -buildlogs_url = rsync://blogs-th@buildlogs.pld-linux.org/pld-buildlogs-th-SRPMS/ -ftp_url = scp://pldth@ep09.pld-linux.org:ftp/.incoming/SRPMS/ -notify_url = http://src.th.pld-linux.org:1234/ -test_ftp_url = scp://pldth@ep09.pld-linux.org:ftp/.test-builds/SRPMS/ -rpmqa_url = scp://pldth@ep09.pld-linux.org:ftp/.stat/builder/th/ +chroot = /home/tld/builderti/chroots/chroot-src/ +buildlogs_url = rsync://buildlogs-ti@buildlogs.tld-linux.org/buildlogs-ti-src/ +ftp_url = scp://ftpti@kraz.tld-linux.org:ftp/.incoming/SRPMS/ +notify_url = http://kraz.tld-linux.org:1231/ +test_ftp_url = scp://ftpti@kraz.tld-linux.org:ftp/.test-builds/SRPMS/RPMS/ +rpmqa_url = scp://ftpti@kraz.tld-linux.org:ftp/.stat/builder/ti/ rpmqa_filename = rpmqa-SRPMS.txt - -[th-athlon] -arch = athlon -chroot = /home/users/builderth/chroot-athlon/ -buildlogs_url = /dev/null -notify_url = http://src.th.pld-linux.org:1234/ -ftp_url = scp://pldth@ep09.pld-linux.org:ftp/.incoming/athlon/ -test_ftp_url = /dev/null -rpmqa_url = /dev/null -rpmqa_filename = nothing - - diff --git a/doc/README b/doc/README index cc9fb47..07ab398 100644 --- a/doc/README +++ b/doc/README @@ -2,7 +2,7 @@ Roadmap: Makefile -- nothing interesting. - PLD_Builder/ -- all python stuff live here. + TLD_Builder/ -- all python stuff live here. admin/ -- scripts for chroot admin. diff --git a/doc/README.bin-builder b/doc/README.bin-builder index 89a7b77..2699cbc 100644 --- a/doc/README.bin-builder +++ b/doc/README.bin-builder @@ -2,7 +2,7 @@ new bin builder setup packages and chroot ~~~~~~~~~~~~~~~~~~~ -1. install pld-builder from th repoeistory on target host +1. install pld-builder from th repoistory on target host 2. create chroot /srv/chroot (you can symlink to real dest for the sake of copy-paste from here) @@ -47,6 +47,7 @@ gpg keys 1. import src builder key to bin builder so it can download queue.gz src-builder$ gpg --export builder-th-src@pld-linux.org --armor > th-src.asc +bin-builder$ su - builder bin-builder$ gpg --import < th-src.asc 2. generate new key for bin builder and import it to src builder so it can diff --git a/etc/crontab b/etc/crontab index 89fd9d2..a7ed0e5 100644 --- a/etc/crontab +++ b/etc/crontab @@ -2,8 +2,8 @@ SHELL=/bin/sh PATH=/sbin:/bin:/usr/sbin:/usr/bin MAILTO=root -#* * * * * builder exec nice -n 19 /usr/share/pld-builder/bin/request-fetcher.sh -#* * * * * builder exec nice -n 19 /usr/share/pld-builder/bin/load-balancer.sh -#* * * * * builder exec nice -n 19 /usr/share/pld-builder/bin/file-sender.sh +#* * * * * builder exec nice -n 19 /usr/share/tld-builder/bin/request-fetcher.sh +#* * * * * builder exec nice -n 19 /usr/share/tld-builder/bin/load-balancer.sh +#* * * * * builder exec nice -n 19 /usr/share/tld-builder/bin/file-sender.sh -#0 0 * * * root chroot /srv/chroot nice -n 19 tmpwatch -m 240 /var/cache/pld-builder/ready +#0 0 * * * root chroot /srv/chroot nice -n 19 tmpwatch -m 240 /var/cache/tld-builder/ready diff --git a/etc/pld-builder.sysconfig b/etc/pld-builder.sysconfig deleted file mode 100644 index 8702c2f..0000000 --- a/etc/pld-builder.sysconfig +++ /dev/null @@ -1,3 +0,0 @@ -# define chroots whose /proc to mount on startup - -#CHROOTS=/srv/builders/pld/{ac/{athlon,i586},actw/{athlon,i586},th/{athlon,i486}} diff --git a/etc/poldek.conf b/etc/poldek.conf index 19111d3..818d2b5 100644 --- a/etc/poldek.conf +++ b/etc/poldek.conf @@ -3,5 +3,5 @@ name = ready pri = -1 type = dir -path = /var/cache/pld-builder/ready/ +path = /var/cache/tld-builder/ready/ autoup = no diff --git a/etc/procmailrc b/etc/procmailrc index ce50ac0..3f8c387 100644 --- a/etc/procmailrc +++ b/etc/procmailrc @@ -4,8 +4,8 @@ LOGFILE=procmail.log #mail.copy :0 -* ^X-New-PLD-Builder: -| /usr/share/pld-builder/bin/request-handler.sh +* ^X-New-TLD-Builder: +| /usr/share/tld-builder/bin/request-handler.sh :0 * ^FROM_MAILER diff --git a/etc/rpm.macros b/etc/rpm.macros index 0639204..de5f098 100644 --- a/etc/rpm.macros +++ b/etc/rpm.macros @@ -1,4 +1,4 @@ -# rpm macros for pld builder chroot +# rpm macros for tld builder chroot # A colon separated list of desired locales to be installed; # "all" means install all locale specific files. diff --git a/etc/pld-builder.init b/etc/tld-builder.init old mode 100644 new mode 100755 similarity index 70% rename from etc/pld-builder.init rename to etc/tld-builder.init index 8672b25..c184b28 --- a/etc/pld-builder.init +++ b/etc/tld-builder.init @@ -1,10 +1,10 @@ #!/bin/sh # -# pld-builder perform adminstrator's tasks for PLD Linux Builder +# tld-builder perform adminstrator's tasks for TLD Linux Builder # # chkconfig: 345 99 01 # -# description: perform adminstrator's tasks for PLD Linux Builder +# description: perform adminstrator's tasks for TLD Linux Builder # # $Id$ @@ -15,7 +15,7 @@ CHROOTS= # Get service config - may override defaults -[ -f /etc/sysconfig/pld-builder ] && . /etc/sysconfig/pld-builder +[ -f /etc/sysconfig/tld-builder ] && . /etc/sysconfig/tld-builder mount_chroots() { # mount /proc in chroots @@ -59,8 +59,8 @@ chroots_status() { start() { # Check if the service is already running? - if [ -f /var/lock/subsys/pld-builder ]; then - msg_already_running "PLD Linux Builder" + if [ -f /var/lock/subsys/tld-builder ]; then + msg_already_running "TLD Linux Builder" return fi @@ -68,30 +68,30 @@ start() { # no chroots configured. return and be silent return fi - msg_starting "PLD Linux Builder" + msg_starting "TLD Linux Builder" busy; echo mount_chroots RETVAL=$? - [ $RETVAL -eq 0 ] && touch /var/lock/subsys/pld-builder + [ $RETVAL -eq 0 ] && touch /var/lock/subsys/tld-builder } stop() { - if [ ! -f /var/lock/subsys/pld-builder ]; then - msg_not_running "PLD Linux Builder" + if [ ! -f /var/lock/subsys/tld-builder ]; then + msg_not_running "TLD Linux Builder" return fi # Stop daemons. - msg_stopping "PLD Linux Builder" + msg_stopping "TLD Linux Builder" busy; echo umount_chroots - rm -f /var/lock/subsys/pld-builder >/dev/null 2>&1 + rm -f /var/lock/subsys/tld-builder >/dev/null 2>&1 RETVAL=0 } condrestart() { - if [ ! -f /var/lock/subsys/pld-builder ]; then - msg_not_running "PLD Linux Builder" + if [ ! -f /var/lock/subsys/tld-builder ]; then + msg_not_running "TLD Linux Builder" RETVAL=$1 return fi diff --git a/xml/queue.xslt b/xml/queue.xslt index 82d1dc6..d8d578b 100644 --- a/xml/queue.xslt +++ b/xml/queue.xslt @@ -5,7 +5,7 @@ 2) proper dates with http://www.djkaty.com/drupal/xsl-date-time --> -PLD builder queue +TLD builder queue

. from , ,
-- 2.44.0