From df9f6d1aa763bace4cd73b189b2f8be4f99394ae Mon Sep 17 00:00:00 2001 From: Marcin Krol Date: Thu, 17 Apr 2025 19:55:35 +0200 Subject: [PATCH] - merged fixes from PLD --- TLD_Builder/acl.py | 2 +- TLD_Builder/blacklist.py | 2 +- TLD_Builder/chroot.py | 2 ++ TLD_Builder/config.py | 6 ++++-- TLD_Builder/file_sender.py | 8 ++++---- TLD_Builder/gpg.py | 11 ++++++++++- TLD_Builder/install.py | 18 +++++++++++------- TLD_Builder/request.py | 2 +- TLD_Builder/request_handler_server.py | 11 +++++------ TLD_Builder/rpm_builder.py | 4 ++-- TLD_Builder/style.css | 13 +++++++------ client/make-request.sh | 24 ++++++++++++++++-------- 12 files changed, 64 insertions(+), 39 deletions(-) diff --git a/TLD_Builder/acl.py b/TLD_Builder/acl.py index 88ed536..75c3c7b 100644 --- a/TLD_Builder/acl.py +++ b/TLD_Builder/acl.py @@ -107,7 +107,7 @@ class ACL_Conf: self.current_user = None status.push("reading acl.conf") p = ConfigParser.ConfigParser() - p.readfp(open(path.acl_conf)) + p.read_file(open(path.acl_conf)) self.users = {} for login in p.sections(): if login in self.users: diff --git a/TLD_Builder/blacklist.py b/TLD_Builder/blacklist.py index 3f55238..1455ffe 100644 --- a/TLD_Builder/blacklist.py +++ b/TLD_Builder/blacklist.py @@ -34,7 +34,7 @@ class Blacklist_File: if re.match(r"^#.*", p): continue self.blacklist.add(p) - log.notice("blacklist added: %s" % l) + log.notice("blacklist added: %s" % p) f.close() status.pop() diff --git a/TLD_Builder/chroot.py b/TLD_Builder/chroot.py index 6e7976a..e214ae1 100644 --- a/TLD_Builder/chroot.py +++ b/TLD_Builder/chroot.py @@ -44,6 +44,8 @@ def run(cmd, user = "builder", logfile = None, logstdout = None): if logstdout != None: for l in f: logstdout.write(l) + else: + f.read() r = f.close() if r == None: return 0 diff --git a/TLD_Builder/config.py b/TLD_Builder/config.py index d8c42c4..f185119 100644 --- a/TLD_Builder/config.py +++ b/TLD_Builder/config.py @@ -51,7 +51,7 @@ class Builder_Conf: else: log.panic("cannot find %s::%s" % (builder, o)) - p.readfp(open(path.builder_conf)) + p.read_file(open(path.builder_conf)) self.admin_email = get("admin_email") self.email = self.admin_email @@ -109,6 +109,8 @@ class Builder_Conf: self.rpm_cache_dir = get("rpm_cache_dir", "/spools/ready") self.builder_user = get("builder_user", "builder") self.sudo_chroot_wrapper = get("sudo_chroot_wrapper", "") + # Available php versions in the distro + self.php_versions = get("php_versions", "4 5.2 5.3 5.4 5.5 5.6 7.0 7.1 7.2 7.3 7.4 8.0 8.1 8.2").split(" ") self.nice = get("nice", "0") f = get("syslog", "") @@ -129,7 +131,7 @@ def init_conf(builder=None): pass else: ssl._create_default_https_context = _create_unverified_https_context - os.environ['LC_ALL'] = "C" + os.environ['LC_ALL'] = "C.UTF8" status.push("reading builder config") log.builder = builder if not builder: diff --git a/TLD_Builder/file_sender.py b/TLD_Builder/file_sender.py index f5ef280..1415804 100644 --- a/TLD_Builder/file_sender.py +++ b/TLD_Builder/file_sender.py @@ -108,18 +108,18 @@ def send_file(src, target): global problems try: log.notice("sending %s to %s (size %d bytes)" % (src, target, os.stat(src).st_size)) - m = re.match('rsync://([^/]+)/.*', target) + m = re.match(r'rsync://([^/]+)/.*', target) if m: return not rsync_file(src, target, host = m.group(1)) if target != "" and target[0] == '/': return not copy_file(src, target) - m = re.match('scp://([^@:]+@[^/:]+)(:|)(.*)', target) + m = re.match(r'scp://([^@:]+@[^/:]+)(:|)(.*)', target) if m: return not scp_file(src, m.group(1) + ":" + m.group(3)) - m = re.match('ssh\+rsync://([^@:]+@[^/:]+)(:|)(.*)', target) + m = re.match(r'ssh\+rsync://([^@:]+@[^/:]+)(:|)(.*)', target) if m: return not rsync_ssh_file(src, m.group(1) + ":" + m.group(3)) - m = re.match('(http|https)://.*', target) + m = re.match(r'(http|https)://.*', target) if m: return not post_file(src, target) log.alert("unsupported protocol: %s" % target) diff --git a/TLD_Builder/gpg.py b/TLD_Builder/gpg.py index 942cfb5..f9a5b2b 100644 --- a/TLD_Builder/gpg.py +++ b/TLD_Builder/gpg.py @@ -87,6 +87,15 @@ def sign(buf): raise if len(d_stderr): - log.error("gpg: %s" % util.to_str(d_stderr)) + gerr = util.to_str(d_stderr) + log.error("gpg: %s" % gerr) + + if gerr.find('gpgconf --kill all') >= 0: + try: + cmd = [ 'gpgconf', '--kill', 'all' ] + subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, close_fds=True) + except OSError as e: + log.error("gpgconf --kill all failed: %s" % e) + return d_stdout diff --git a/TLD_Builder/install.py b/TLD_Builder/install.py index c8eb543..d18c207 100644 --- a/TLD_Builder/install.py +++ b/TLD_Builder/install.py @@ -118,22 +118,24 @@ def uninstall_self_conflict(b): if is_rpmorg(): rpmcommand = "rpmbuild --nobuild -br" else: - rpmcommand = "rpmbuild -bp --nobuild --short-circuit --define 'prep exit 0'" - f = chroot.popen("set -e; TMPDIR=%(tmpdir)s %(rpmcommand)s %(rpmdefs)s %(topdir)s/%(spec)s 2>&1" % { + rpmcommand = "rpmbuild --nobuild" + cmd = "set -e; TMPDIR=%(tmpdir)s %(rpmcommand)s %(rpmdefs)s %(topdir)s/%(spec)s 2>&1" % { 'rpmcommand': rpmcommand, 'tmpdir': b.tmpdir(), - 'rpmdefs' : b.rpmbuild_opts(), 'topdir' : b.get_topdir(), + 'rpmdefs' : b.rpmbuild_opts(), 'spec': b.spec, - }, encoding = "utf-8") + } + f = chroot.popen(cmd, encoding = "utf-8") # java-sun >= 1.5 conflicts with soprano-2.1.67-1.src # java-sun conflicts with soprano-2.1.67-1.src - rx = re.compile(r"\s+(?P[\w-]+)\s+.*conflicts with [^\s]+-[^-]+-[^-]+\.src($| .*)") + # plplot conflicts with plplot-5.15.0-4.x86_64 + rx = re.compile(r"\s+(?P[\w-]+)\s+.*conflicts with [^\s]+-[^-]+-[^-]+\.[^.\s]+($| .*)") conflicting = {} for l in f: + b.log_line("rpm: %s" % l.rstrip()) m = rx.search(l) if m: - b.log_line("rpmbuild: %s" % l.rstrip()) conflicting[m.group('name')] = 1 f.close() if len(conflicting) and not uninstall(conflicting, b): @@ -188,7 +190,9 @@ def install_br(r, b): # phonon-devel-4.3.1-1.i686 conflicts with qt4-phonon-devel-4.5.0-6.i686 # jdbc-stdext >= 2.0 is required by installed java-struts-1.3.10-1.noarch # jmx is needed by (installed) java-commons-modeler-2.0-1.noarch - rx = re.compile(r".*(conflicts with|is required by|is needed by)( installed| \(installed\)|) (?P[^\s]+)-[^-]+-[^-]+($| .*)") + # poldek: file /usr/include/php/main/php_config.h from install of php82-devel-4:8.2.17-3.x86_64 conflicts with file from package php83-devel-4:8.3.7-1.x86_64 + # poldek: php80-devel < 4:8.1 is obsoleted by (installed) php83-devel-4:8.3.7-1.x86_64 + rx = re.compile(r".*(conflicts with|conflicts with file from package|is required by|is obsoleted by|is needed by)( installed| \(installed\)|) (?P[^\s]+)-[^-]+-[^-]+($| .*)") conflicting = {} for l in f: b.log_line("poldek: %s" % l.rstrip()) diff --git a/TLD_Builder/request.py b/TLD_Builder/request.py index da02c69..eeddc30 100644 --- a/TLD_Builder/request.py +++ b/TLD_Builder/request.py @@ -419,7 +419,7 @@ class Batch: def php_ignores(self, php_version): # available php versions in distro - php_versions = ['7.2', '7.3', '7.4', '8.0'] + php_versions = config.php_versions.copy() # remove current php version try: diff --git a/TLD_Builder/request_handler_server.py b/TLD_Builder/request_handler_server.py index d97d76f..0a92950 100644 --- a/TLD_Builder/request_handler_server.py +++ b/TLD_Builder/request_handler_server.py @@ -2,7 +2,6 @@ import socket import string -import cgi import time import log import ssl @@ -79,11 +78,11 @@ def main(srv_ssl=False): try: server = ThreadingHTTPServer((host, port), MyHandler) if srv_ssl: - server.socket = ssl.wrap_socket (server.socket, - keyfile = path.conf_dir + "/" + config.request_handler_server_ssl_key, - certfile = path.conf_dir + "/" + config.request_handler_server_ssl_cert, - ca_certs = "/etc/certs/ca-certificates.crt", - server_side=True) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certfile=path.conf_dir + "/" + config.request_handler_server_ssl_cert, + keyfile=path.conf_dir + "/" + config.request_handler_server_ssl_key) + context.load_verify_locations(cafile="/etc/certs/ca-certificates.crt") + server.socket = context.wrap_socket(server.socket, server_side=True) except Exception as e: log.notice("request_handler_server: can't start server on [%s:%d], ssl=%s: %s" % (host, port, str(srv_ssl), e)) print >> sys.stderr, "ERROR: Can't start server on [%s:%d], ssl=%s: %s" % (host, port, str(srv_ssl), e) diff --git a/TLD_Builder/rpm_builder.py b/TLD_Builder/rpm_builder.py index dc06fff..a7a10b7 100644 --- a/TLD_Builder/rpm_builder.py +++ b/TLD_Builder/rpm_builder.py @@ -155,7 +155,7 @@ def prepare_env(logfile = None): test ! -c /dev/zero && rm -f /dev/zero && mknod -m 666 /dev/zero c 1 5 # need entry for "/" in mtab, for diskspace() to work in rpm - [ -z $(awk '$2 == "/" {print $1; exit}' /etc/mtab) ] && mount -f -t rootfs rootfs / + [ -z $(awk '$2 == "/" {print $1; exit}' /etc/mtab) ] && echo "Missing rootfs entry in /etc/mtab" # make neccessary files readable for builder user # TODO: see if they really aren't readable for builder @@ -236,7 +236,7 @@ def build_rpm(r, b): if r.max_jobs > 0: max_jobs = max(min(config.max_jobs, r.max_jobs), 1) cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \ - "rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' --define '_make_opts -Otarget' --define '_tld_builder 1' %(rpmdefs)s %(topdir)s/%(spec)s" % { + "rpmbuild -bb --define '__jobs %(max_jobs)d' --define '_smp_mflags -j%(max_jobs)d' --define '_make_opts -Otarget' --define '_tld_builder 1' --define '_pld_builder 1' %(rpmdefs)s %(topdir)s/%(spec)s" % { 'r_id' : r.id, 'tmpdir': tmpdir, 'nice' : config.nice, diff --git a/TLD_Builder/style.css b/TLD_Builder/style.css index 82e2150..b5fde19 100644 --- a/TLD_Builder/style.css +++ b/TLD_Builder/style.css @@ -37,10 +37,8 @@ pre { } div.request { - background-color: white; - margin: 10px 0px; - padding: 2px; - } + background-color: #b2eb91; + } div > a { font-weight: bold; color: #5e5e5e; @@ -49,10 +47,13 @@ pre { color: #5e5e5e; } div.upgrade { - background-color: #e4f1cf; + background-color: #81c784; + } + div.test-build { + background-color: #f8d7da; } div:target { - background-color: #ffffcc; + background-color: #fff3cd; color: black; } diff --git a/client/make-request.sh b/client/make-request.sh index 06e3445..878faca 100755 --- a/client/make-request.sh +++ b/client/make-request.sh @@ -1,6 +1,6 @@ #!/bin/sh # vim:noet:ts=4:sw=4 -VERSION=1.89 +VERSION=1.90 # prevent "*" from being expanded in builders var set -f @@ -20,8 +20,9 @@ verbose=no pkgrevision=no requester_override= relup=no +python=/usr/bin/python3 -if [ -x /usr/bin/python3 ]; then +if [ -x $python ]; then send_mode="python" else echo "No python3 present, aborting" @@ -90,7 +91,7 @@ send_request() { fi msg "Sending using HTTP mode to $url" - cat - | python3 -c ' + cat - | $python -c ' import sys, socket, ssl from urllib import request @@ -477,7 +478,7 @@ esac # need to do this after dist selection if [ "$skip" ]; then - skip=$(skip="$skip" control_url="$control_url" python3 -c ' + skip=$(skip="$skip" control_url="$control_url" $python -c ' import sys import gzip import re @@ -732,8 +733,9 @@ gen_req() { gen_email () { # make request first, so the STDERR/STDOUT streams won't be mixed - local tmp req - tmp=$(mktemp) + local tmpd tmp req + tmpd=$(mktemp -d) + tmp=$tmpd/req gen_req > $tmp if [ "$verbose" = "yes" ]; then @@ -750,8 +752,14 @@ gen_email () { EOF - gpg --clearsign --default-key $default_key $gpg_opts --output=- $tmp - rm -f $tmp + gpg --clearsign --default-key $default_key $gpg_opts --output=$tmpd/sign $tmp + + if [ "$verbose" = "yes" ]; then + cat >&2 $tmpd/sign + fi + + cat $tmpd/sign + rm -rf $tmpd } gen_email | send_request -- 2.49.0