1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
3 from __future__ import print_function
13 from config import config, init_conf
14 from bqueue import B_Queue
30 # *HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*
33 socket.myorigsocket=socket.socket
35 def mysocket(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
36 s=socket.myorigsocket(family, type, proto)
37 s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
40 socket.socket=mysocket
41 # *HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*HACK*
43 # this code is duplicated in srpm_builder, but we
44 # might want to handle some cases differently here
47 if r1.kind != 'group' or r2.kind != 'group':
48 raise Exception("non-group requests")
49 pri_diff = util.cmp(r1.priority, r2.priority)
51 return util.cmp(r1.time, r2.time)
54 q.requests.sort(key=util.cmp_to_key(mycmp))
58 def check_skip_build(r, b):
59 src_url = config.control_url + "/srpms/" + r.id + "/skipme"
61 b.log_line("checking if we should skip the build")
64 headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
65 req = urllib.request.Request(url=src_url, headers=headers)
66 f = urllib.request.urlopen(req)
68 except urllib.error.HTTPError as error:
70 except urllib.error.URLError as error:
74 except AttributeError:
76 errno = error.reason[0]
78 if errno in [-3, 60, 61, 110, 111]:
79 b.log_line("unable to connect... trying again")
88 src_url = config.control_url + "/srpms/" + r.id + "/" + urllib.parse.quote(b.src_rpm)
89 b.log_line("fetching %s" % src_url)
94 headers = { 'Cache-Control': 'no-cache', 'Pragma': 'no-cache' }
95 req = urllib.request.Request(url=src_url, headers=headers)
96 f = urllib.request.urlopen(req)
98 except urllib.error.HTTPError as error:
99 # fail in a way where cron job will retry
100 msg = "unable to fetch url %s, http code: %d" % (src_url, error.code)
102 queue_time = time.time() - r.time
104 if error.code != 404 or (queue_time >= 0 and queue_time < (6 * 60 * 60)):
107 msg = "in queue for more than 6 hours, download failing"
110 except urllib.error.URLError as error:
112 if isinstance(error.args[0], IOError):
113 errno = error.args[0].errno
115 if errno in [-3, 60, 61, 110, 111]:
116 b.log_line("unable to connect to %s... trying again" % (src_url))
120 print("error.errno: %s" % str(error.errno))
121 except Exception as e:
122 print("error.errno: exception %s" % e)
124 print("error.reason %s" % str(error.reason))
125 except Exception as e:
126 print("error.reason exception %s" % e)
129 o = chroot.popen("cat > %s" % b.src_rpm, mode = "w")
132 shutil.copyfileobj(f, o)
134 b.log_line("error: unable to write to `%s': %s" % (b.src_rpm, e))
137 bytes = f.headers['content-length']
140 t = time.time() - start
142 b.log_line("fetched %d bytes" % bytes)
144 b.log_line("fetched %d bytes, %.1f K/s" % (bytes, bytes / 1024.0 / t))
146 def prepare_env(logfile = None):
148 test ! -f /proc/uptime && mount /proc 2>/dev/null
149 test ! -c /dev/full && rm -f /dev/full && mknod -m 666 /dev/full c 1 7
150 test ! -c /dev/null && rm -f /dev/null && mknod -m 666 /dev/null c 1 3
151 test ! -c /dev/random && rm -f /dev/random && mknod -m 644 /dev/random c 1 8
152 test ! -c /dev/urandom && rm -f /dev/urandom && mknod -m 644 /dev/urandom c 1 9
153 test ! -c /dev/zero && rm -f /dev/zero && mknod -m 666 /dev/zero c 1 5
155 # need entry for "/" in mtab, for diskspace() to work in rpm
156 [ -z $(awk '$2 == "/" {print $1; exit}' /etc/mtab) ] && mount -f -t rootfs rootfs /
158 # make neccessary files readable for builder user
159 # TODO: see if they really aren't readable for builder
160 for db in Packages Name Basenames Providename Pubkeys; do
162 test -f $db && chmod a+r $db
165 # try to limit network access for builder account
166 /bin/setfacl -m u:builder:--- /etc/resolv.conf
167 """, 'root', logfile = logfile)
170 packagename = b.get_package_name()
172 # should not really get here
173 b.log_line("error: No .spec not given of malformed: '%s'" % b.spec)
174 res = "FAIL_INTERNAL"
177 status.push("building %s (%s)" % (b.spec, packagename))
178 b.log_line("request from: %s" % r.requester)
180 if check_skip_build(r, b):
181 b.log_line("build skipped due to src builder request")
182 res = "SKIP_REQUESTED"
185 b.log_line("started at: %s" % time.asctime())
187 b.log_line("killing old processes on a builder")
188 chroot.run("/bin/kill --verbose -9 -1", logfile = b.logfile)
190 b.log_line("cleaning up /tmp")
191 chroot.run("rm -rf /tmp/B.*", logfile = b.logfile)
194 b.log_line("installing srpm: %s" % b.src_rpm)
197 install -d %(topdir)s/{BUILD,RPMS};
198 LC_ALL=en_US.UTF-8 rpm -qp --changelog %(src_rpm)s;
199 rpm -Uhv --nodeps %(rpmdefs)s %(src_rpm)s;
202 'topdir' : b.get_topdir(),
203 'rpmdefs' : b.rpmbuild_opts(),
204 'src_rpm' : b.src_rpm
205 }, logfile = b.logfile)
210 b.log_line("error: installing src rpm failed")
211 res = "FAIL_SRPM_INSTALL"
214 chroot.run("set -x; install -m 700 -d %s" % tmpdir, logfile=b.logfile)
215 b.default_target(config.arch)
216 # check for build arch before filling BR
217 cmd = "set -ex; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
218 "rpmbuild -bp --short-circuit --nodeps %(rpmdefs)s --define 'prep exit 0' %(topdir)s/%(spec)s" % {
220 'nice' : config.nice,
221 'topdir' : b.get_topdir(),
222 'rpmdefs' : b.rpmbuild_opts(),
225 res = chroot.run(cmd, logfile = b.logfile)
228 b.log_line("error: build arch check (%s) failed" % cmd)
231 if ("no-install-br" not in r.flags) and not install.uninstall_self_conflict(b):
232 res = "FAIL_DEPS_UNINSTALL"
233 if ("no-install-br" not in r.flags) and not install.install_br(r, b):
234 res = "FAIL_DEPS_INSTALL"
236 max_jobs = max(min(int(os.sysconf('SC_NPROCESSORS_ONLN') + 1), config.max_jobs), 1)
238 max_jobs = max(min(config.max_jobs, r.max_jobs), 1)
239 cmd = "set -ex; : build-id: %(r_id)s; TMPDIR=%(tmpdir)s exec nice -n %(nice)s " \
240 "rpmbuild -bb --define '_smp_mflags -j%(max_jobs)d' --define '_make_opts -Otarget' --define '_tld_builder 1' %(rpmdefs)s %(topdir)s/%(spec)s" % {
243 'nice' : config.nice,
244 'rpmdefs' : b.rpmbuild_opts(),
245 'topdir' : b.get_topdir(),
246 'max_jobs' : max_jobs,
249 b.log_line("building RPM using: %s" % cmd)
250 begin_time = time.time()
251 res = chroot.run(cmd, logfile = b.logfile)
252 end_time = time.time()
253 b.log_line("ended at: %s, done in %s" % (time.asctime(), datetime.timedelta(0, end_time - begin_time)))
256 files = util.collect_files(b.logfile, basedir = b.get_topdir())
258 r.chroot_files.extend(files)
260 b.log_line("error: No files produced.")
261 last_section = util.find_last_section(b.logfile)
262 if last_section == None:
265 res = "FAIL_%s" % last_section.upper()
268 # cleanup tmp and build files
271 chmod -R u+rwX %(topdir)s/BUILD;
272 rm -rf %(topdir)s/{tmp,BUILD}
274 'topdir' : b.get_topdir(),
275 }, logfile = b.logfile)
278 util.append_to(b.logfile, l)
281 rpm_cache_dir = config.rpm_cache_dir
282 if "test-build" not in r.flags:
283 # NOTE: copying to cache dir doesn't mean that build failed, so ignore result
284 b.log_line("copy rpm files to cache_dir: %s" % rpm_cache_dir)
286 "cp -f %s %s && poldek --mo=nodiff --mkidxz -s %s/" % \
287 (' '.join(b.files), rpm_cache_dir, rpm_cache_dir),
288 logfile = b.logfile, user = "root"
291 ll("test-build: not copying to " + rpm_cache_dir)
292 ll("Begin-TLD-Builder-Info")
293 if "upgrade" in r.flags:
294 b.upgraded = install.upgrade_from_batch(r, b)
297 ll("End-TLD-Builder-Info")
300 local = r.tmp_dir + os.path.basename(f)
301 chroot.cp(f, outfile = local, rm = True)
304 # cleanup all remains from this build
309 'topdir' : b.get_topdir(),
310 }, logfile = b.logfile)
313 c="file:SRPMS:%s\n" % b.src_rpm
315 c=c + "file:ARCH:%s\n" % os.path.basename(f)
319 if config.gen_upinfo and b.files != [] and 'test-build' not in r.flags:
320 fname = r.tmp_dir + b.src_rpm + ".uploadinfo"
322 f.write(uploadinfo(b))
324 ftp.add(fname, "uploadinfo")
330 def handle_request(r):
333 build.build_all(r, build_rpm)
334 report.send_report(r, is_src = False)
341 f = open("/proc/loadavg")
342 if float(f.readline().split()[2]) > config.max_load:
349 def main_for(builder):
354 q = B_Queue(path.queue_file + "-" + config.builder)
360 req = pick_request(q)
363 # high priority tasks have priority < 0, normal tasks >= 0
364 if req.priority >= 0:
366 # allow only one build in given builder at once
367 if not lock.lock("building-rpm-for-%s" % config.builder, non_block = 1):
371 # not more then job_slots builds at once
373 for slot in range(config.job_slots):
374 if lock.lock("building-rpm-slot-%d" % slot, non_block = 1):
380 # record fact that we got lock for this builder, load balancer
381 # will use it for fair-queuing
382 l = lock.lock("got-lock")
383 f = open(path.got_lock_file, "a")
384 f.write(config.builder + "\n")
388 # be able to avoid locking with very low priority
389 if req.priority > -1000:
392 # allow only one build in given builder at once
393 if not lock.lock("building-high-priority-rpm-for-%s" % config.builder, non_block = 1):
396 msg = "HIGH PRIORITY: "
398 msg += "handling request %s (%d) for %s from %s, priority %s" \
399 % (req.id, req.no, config.builder, req.requester, req.priority)
411 q = B_Queue(path.queue_file + "-" + config.builder)
414 previouslen=len(q.requests)
415 q.requests=list(filter(otherreqs, q.requests))
416 if len(q.requests)<previouslen:
421 if len(sys.argv) < 2:
422 raise Exception("fatal: need to have builder name as first arg")
423 return main_for(sys.argv[1])
425 if __name__ == '__main__':