X-Git-Url: https://git.tld-linux.org/?p=tld-builder.git;a=blobdiff_plain;f=TLD_Builder%2Frpm_builder.py;fp=PLD_Builder%2Frpm_builder.py;h=12cc265ca49fd24c70796cbaf10db6fa7a1bf948;hp=e42c085fee48f830bf81c25cee82d547a39753ba;hb=b999f53d4bf5d44586ecf028876e8bc20b5fd2ce;hpb=37463eaa22f48f5fecbb90e69ef67a69e0bf9788 diff --git a/PLD_Builder/rpm_builder.py b/TLD_Builder/rpm_builder.py similarity index 84% rename from PLD_Builder/rpm_builder.py rename to TLD_Builder/rpm_builder.py index e42c085..12cc265 100644 --- a/PLD_Builder/rpm_builder.py +++ b/TLD_Builder/rpm_builder.py @@ -106,17 +106,22 @@ def fetch_src(r, b): b.log_line(msg) return False except urllib2.URLError, error: - # see errno.h - try: - errno = error.errno - except AttributeError: - # python 2.4 - errno = error.reason[0] + errno = 0 + if isinstance(error.args[0], IOError): + errno = error.args[0].errno if errno in [-3, 60, 61, 110, 111]: b.log_line("unable to connect to %s... trying again" % (src_url)) continue else: + try: + print "error.errno: %s" % str(error.errno) + except Exception, e: + print "error.errno: exception %s" % e + try: + print "error.reason %s" % str(error.reason) + except Exception, e: + print "error.reason exception %s" % e raise o = chroot.popen("cat > %s" % b.src_rpm, mode = "w") @@ -135,7 +140,7 @@ def fetch_src(r, b): else: b.log_line("fetched %d bytes, %.1f K/s" % (bytes, bytes / 1024.0 / t)) -def prepare_env(): +def prepare_env(logfile = None): chroot.run(""" test ! -f /proc/uptime && mount /proc 2>/dev/null test ! -c /dev/full && rm -f /dev/full && mknod -m 666 /dev/full c 1 7 @@ -145,7 +150,7 @@ def prepare_env(): test ! -c /dev/zero && rm -f /dev/zero && mknod -m 666 /dev/zero c 1 5 # need entry for "/" in mtab, for diskspace() to work in rpm - [ -z $(awk '$2 == "/" {print $1}' /etc/mtab) ] && mount -f -t rootfs rootfs / + [ -z $(awk '$2 == "/" {print $1; exit}' /etc/mtab) ] && mount -f -t rootfs rootfs / # make neccessary files readable for builder user # TODO: see if they really aren't readable for builder @@ -156,16 +161,16 @@ def prepare_env(): # try to limit network access for builder account /bin/setfacl -m u:builder:--- /etc/resolv.conf - """, 'root') + """, 'root', logfile = logfile) def build_rpm(r, b): - if len(b.spec) <= 5: + packagename = b.get_package_name() + if not packagename: # should not really get here b.log_line("error: No .spec not given of malformed: '%s'" % b.spec) res = "FAIL_INTERNAL" return res - packagename = b.spec[:-5] status.push("building %s (%s)" % (b.spec, packagename)) b.log_line("request from: %s" % r.requester) @@ -178,38 +183,33 @@ def build_rpm(r, b): fetch_src(r, b) b.log_line("installing srpm: %s" % b.src_rpm) res = chroot.run(""" - # b.id %(bid)s set -ex; - install -d rpm/packages/%(package)s rpm/BUILD/%(package)s; - rpm -Uhv %(rpmdefs)s %(src_rpm)s; + install -d %(topdir)s/{BUILD,RPMS}; + LC_ALL=en_US.UTF-8 rpm -qp --changelog %(src_rpm)s; + rpm -Uhv --nodeps %(rpmdefs)s %(src_rpm)s; rm -f %(src_rpm)s; """ % { - 'bid' : b.b_id, - 'package' : packagename, + 'topdir' : b.get_topdir(), 'rpmdefs' : b.rpmbuild_opts(), 'src_rpm' : b.src_rpm }, logfile = b.logfile) b.files = [] - # it's better to have TMPDIR and BUILD dir on same partition: - # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2 - # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory - tmpdir = os.environ.get('HOME') + "/rpm/BUILD/%s/tmp" % packagename + tmpdir = b.tmpdir() if res: b.log_line("error: installing src rpm failed") res = "FAIL_SRPM_INSTALL" else: prepare_env() - chroot.run("install -m 700 -d %s" % tmpdir) - + chroot.run("set -x; install -m 700 -d %s" % tmpdir, logfile=b.logfile) b.default_target(config.arch) # check for build arch before filling BR cmd = "set -ex; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \ - "rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' rpm/packages/%(package)s/%(spec)s" % { + "rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' %(topdir)s/%(spec)s" % { 'tmpdir': tmpdir, 'nice' : config.nice, + 'topdir' : b.get_topdir(), 'rpmdefs' : b.rpmbuild_opts(), - 'package' : packagename, 'spec': b.spec, } res = chroot.run(cmd, logfile = b.logfile) @@ -227,12 +227,12 @@ def build_rpm(r, b): if r.max_jobs > 0: max_jobs = max(min(config.max_jobs, r.max_jobs), 1) cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \ - "rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' %(rpmdefs)s rpm/packages/%(package)s/%(spec)s" % { + "rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' --define '_make_opts -Otarget' --define '_tld_builder 1' %(rpmdefs)s %(topdir)s/%(spec)s" % { 'r_id' : r.id, 'tmpdir': tmpdir, 'nice' : config.nice, 'rpmdefs' : b.rpmbuild_opts(), - 'package' : packagename, + 'topdir' : b.get_topdir(), 'max_jobs' : max_jobs, 'spec': b.spec, } @@ -243,7 +243,7 @@ def build_rpm(r, b): b.log_line("ended at: %s, done in %s" % (time.asctime(), datetime.timedelta(0, end_time - begin_time))) if res: res = "FAIL" - files = util.collect_files(b.logfile) + files = util.collect_files(b.logfile, basedir = b.get_topdir()) if len(files) > 0: r.chroot_files.extend(files) else: @@ -255,14 +255,14 @@ def build_rpm(r, b): res = "FAIL_%s" % last_section.upper() b.files = files + # cleanup tmp and build files chroot.run(""" set -ex; - rpmbuild %(rpmdefs)s --nodeps --nobuild --clean --rmspec --rmsource rpm/packages/%(package)s/%(spec)s - rm -rf %(tmpdir)s; - chmod -R u+rwX rpm/BUILD/%(package)s; - rm -rf rpm/BUILD/%(package)s; - """ % - {'tmpdir' : tmpdir, 'spec': b.spec, 'package' : packagename, 'rpmdefs' : b.rpmbuild_opts()}, logfile = b.logfile) + chmod -R u+rwX %(topdir)s/BUILD; + rm -rf %(topdir)s/{tmp,BUILD} + """ % { + 'topdir' : b.get_topdir(), + }, logfile = b.logfile) def ll(l): util.append_to(b.logfile, l) @@ -279,18 +279,26 @@ def build_rpm(r, b): ) else: ll("test-build: not copying to " + rpm_cache_dir) - ll("Begin-PLD-Builder-Info") + ll("Begin-TLD-Builder-Info") if "upgrade" in r.flags: b.upgraded = install.upgrade_from_batch(r, b) else: ll("not upgrading") - ll("End-PLD-Builder-Info") + ll("End-TLD-Builder-Info") for f in b.files: local = r.tmp_dir + os.path.basename(f) chroot.cp(f, outfile = local, rm = True) ftp.add(local) + # cleanup all remains from this build + chroot.run(""" + set -ex; + rm -rf %(topdir)s; + """ % { + 'topdir' : b.get_topdir(), + }, logfile = b.logfile) + def uploadinfo(b): c="file:SRPMS:%s\n" % b.src_rpm for f in b.files: @@ -367,6 +375,14 @@ def main_for(builder): f.close() l.close() else: + # be able to avoid locking with very low priority + if req.priority > -1000: + # don't kill server + check_load() + # allow only one build in given builder at once + if not lock.lock("building-high-priority-rpm-for-%s" % config.builder, non_block = 1): + return + msg = "HIGH PRIORITY: " msg += "handling request %s (%d) for %s from %s, priority %s" \