1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
3 from xml.dom.minidom import *
4 from datetime import datetime
7 import xml.sax.saxutils
19 from config import config
21 __all__ = ['parse_request', 'parse_requests']
25 for n in e.childNodes:
26 if n.nodeType != Element.TEXT_NODE:
27 log.panic("xml: text expected in <%s>, got %d" % (e.nodeName, n.nodeType))
31 def attr(e, a, default = None):
33 return e.attributes[a].value
40 return xml.sax.saxutils.escape(s)
42 # return timestamp with timezone information
43 # so we could parse it in javascript
45 # as strftime %z is unofficial, and does not work, need to make it numeric ourselves
46 date = time.strftime("%a %b %d %Y %H:%M:%S", time.localtime(t))
47 # NOTE: the altzone is showing CURRENT timezone, not what the "t" reflects
48 # NOTE: when DST is off timezone gets it right, altzone not
50 tzoffset = time.altzone
52 tzoffset = time.timezone
53 tz = '%+05d' % (-tzoffset / 3600 * 100)
54 return date + ' ' + tz
56 # return date in iso8601 format
57 def iso8601(ts, timezone='UTC'):
58 tz = pytz.timezone(timezone)
59 dt = datetime.fromtimestamp(ts, tz)
63 return e.nodeType == Element.TEXT_NODE and e.nodeValue.strip() == ""
66 def __init__(self, e):
69 self.id = attr(e, "id")
70 self.no = int(attr(e, "no"))
72 self.time = time.time()
75 self.requester_email = ""
76 self.flags = attr(e, "flags", "").split()
77 for c in e.childNodes:
78 if is_blank(c): continue
80 if c.nodeType != Element.ELEMENT_NODE:
81 log.panic("xml: evil group child %d" % c.nodeType)
82 if c.nodeName == "batch":
83 self.batches.append(Batch(c))
84 elif c.nodeName == "requester":
85 self.requester = text(c)
86 self.requester_email = attr(c, "email", "")
87 elif c.nodeName == "priority":
88 self.priority = int(text(c))
89 elif c.nodeName == "time":
90 self.time = int(text(c))
91 elif c.nodeName == "maxjobs":
92 self.max_jobs = int(text(c))
94 log.panic("xml: evil group child (%s)" % c.nodeName)
95 # note that we also check that group is sorted WRT deps
97 for b in self.batches:
100 for dep in b.depends_on:
103 if id(m[dep]) != id(b):
106 log.panic("xml: dependency not found in group")
108 if self.requester_email == "" and self.requester != "":
109 self.requester_email = acl.user(self.requester).mail_to()
112 f.write("group: %d (id=%s pri=%d)\n" % (self.no, self.id, self.priority))
113 f.write(" from: %s\n" % self.requester)
114 f.write(" flags: %s\n" % ' '.join(self.flags))
115 f.write(" time: %s\n" % time.asctime(time.localtime(self.time)))
116 for b in self.batches:
120 # return structure usable for json encoding
123 for b in self.batches:
124 batches.append(b.dump_json())
130 requester=self.requester,
131 priority=self.priority,
132 max_jobs=self.max_jobs,
137 def dump_html(self, f):
139 "<div id=\"%(no)d\" class=\"request %(flags)s\">\n"
140 "<a href=\"#%(no)d\">%(no)d</a>. "
141 "<time title=\"%(datetime)s\" datetime=\"%(datetime)s\">%(time)s</time> "
142 "from <b class=requester>%(requester)s</b> "
143 "<small>%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s</small>\n"
146 'id': '<a href="srpms/%(id)s">%(id)s</a>' % {'id': self.id},
147 'time': escape(tzdate(self.time)),
148 'datetime': escape(iso8601(self.time)),
149 'requester': escape(self.requester),
150 'priority': self.priority,
151 'max_jobs': self.max_jobs,
152 'flags': ' '.join(self.flags)
155 for b in self.batches:
156 b.dump_html(f, self.id)
160 def write_to(self, f):
162 <group id="%s" no="%d" flags="%s">
163 <requester email='%s'>%s</requester>
165 <priority>%d</priority>
166 <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, ' '.join(self.flags),
167 escape(self.requester_email), escape(self.requester),
168 self.time, self.priority, self.max_jobs))
169 for b in self.batches:
171 f.write(" </group>\n\n")
175 for b in self.batches:
180 # transform php package name (52) to version (5.2)
181 def php_name_to_ver(v):
182 return '.'.join(list(v))
184 # transform php version (5.2) to package name (52)
185 def php_ver_to_name(v):
186 return v.replace('.', '')
191 def __init__(self, e):
192 self.bconds_with = []
193 self.bconds_without = []
195 self.builders_status = {}
196 self.builders_status_time = {}
197 self.builders_status_buildtime = {}
206 self.command_flags = []
209 self.b_id = attr(e, "id")
210 self.depends_on = attr(e, "depends-on").split()
217 def get_topdir(self):
218 if not self.__topdir:
219 self.__topdir = tempfile.mkdtemp(prefix='B.', dir='/tmp')
222 def parse_xml(self, e):
223 for c in e.childNodes:
224 if is_blank(c): continue
226 if c.nodeType != Element.ELEMENT_NODE:
227 log.panic("xml: evil batch child %d" % c.nodeType)
228 if c.nodeName == "src-rpm":
229 self.src_rpm = text(c)
230 elif c.nodeName == "spec":
231 # normalize specname, specname is used as buildlog and we don't
232 # want to be exposed to directory traversal attacks
233 self.spec = text(c).split('/')[-1]
234 elif c.nodeName == "command":
235 self.spec = "COMMAND"
236 self.command = text(c).strip()
237 self.command_flags = attr(c, "flags", "").split()
238 elif c.nodeName == "info":
240 elif c.nodeName == "kernel":
241 self.kernel = text(c)
242 elif c.nodeName == "define":
243 define = attr(c, "name")
244 self.defines[define] = text(c)
245 elif c.nodeName == "target":
246 self.target.append(text(c))
247 elif c.nodeName == "skip":
248 self.skip.append(text(c))
249 elif c.nodeName == "branch":
250 self.branch = text(c)
251 elif c.nodeName == "builder":
253 self.builders.append(key)
254 self.builders_status[key] = attr(c, "status", "?")
255 self.builders_status_time[key] = attr(c, "time", "0")
256 self.builders_status_buildtime[key] = "0" #attr(c, "buildtime", "0")
257 elif c.nodeName == "with":
258 self.bconds_with.append(text(c))
259 elif c.nodeName == "without":
260 self.bconds_without.append(text(c))
262 log.panic("xml: evil batch child (%s)" % c.nodeName)
264 def get_package_name(self):
265 if len(self.spec) <= 5:
267 return self.spec[:-5]
271 return tmpdir for this batch job building
273 # it's better to have TMPDIR and BUILD dir on same partition:
274 # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2
275 # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory
276 path = os.path.join(self.get_topdir(), 'BUILD', 'tmp')
281 for b in self.builders:
282 s = self.builders_status[b]
283 if not s.startswith("OK") and not s.startswith("SKIP") and not s.startswith("UNSUPP") and not s.startswith("FAIL"):
288 f.write(" batch: %s/%s\n" % (self.src_rpm, self.spec))
289 f.write(" info: %s\n" % self.info)
290 f.write(" kernel: %s\n" % self.kernel)
291 f.write(" defines: %s\n" % self.defines_string())
292 f.write(" target: %s\n" % self.target_string())
293 f.write(" branch: %s\n" % self.branch)
294 f.write(" bconds: %s\n" % self.bconds_string())
296 for b in self.builders:
297 builders.append("%s:%s" % (b, self.builders_status[b]))
298 f.write(" builders: %s\n" % ' '.join(builders))
300 def is_command(self):
301 return self.command != ""
303 # return structure usable for json encoding
306 command=self.command,
307 command_flags=self.command_flags,
311 package=self.spec[:-5],
312 src_rpm=self.src_rpm,
314 bconds_with=self.bconds_with,
315 bconds_without=self.bconds_without,
319 defines=self.defines,
321 builders=self.builders,
322 builders_status=self.builders_status,
323 builders_status_time=self.builders_status_time,
324 builders_status_buildtime=self.builders_status_buildtime,
327 def dump_html(self, f, rid):
329 if self.is_command():
330 desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
332 cmd = "/usr/bin/git ls-remote --heads git://git.tld-linux.org/packages/%s 1>/dev/null 2>&1" % (self.spec[:-5])
333 r = subprocess.call(cmd, shell=True)
335 package_url = "http://git.tld-linux.org/?p=packages/%(package)s.git;a=blob;f=%(spec)s;hb=%(branch)s" % {
336 'spec': urllib.parse.quote(self.spec),
337 'branch': urllib.parse.quote(self.branch),
338 'package': urllib.parse.quote(self.spec[:-5]),
341 package_url = "http://git.pld-linux.org/gitweb.cgi?p=packages/%(package)s.git;f=%(spec)s;h=%(branch)s;a=shortlog" % {
342 'spec': urllib.parse.quote(self.spec),
343 'branch': urllib.parse.quote(self.branch),
344 'package': urllib.parse.quote(self.spec[:-5]),
346 desc = "%(src_rpm)s (<a href=\"%(package_url)s\">%(spec)s -r %(branch)s</a>%(rpmopts)s)" % {
347 'src_rpm': self.src_rpm,
349 'branch': self.branch,
350 'rpmopts': self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string(),
351 'package_url': package_url,
353 f.write("%s <small>[" % desc)
355 for b in self.builders:
356 s = self.builders_status[b]
357 if s.startswith("OK"):
359 elif s.startswith("FAIL"):
361 elif s.startswith("SKIP"):
363 elif s.startswith("UNSUPP"):
369 if (s.startswith("OK") or s.startswith("SKIP") or s.startswith("UNSUPP") or s.startswith("FAIL")) and len(self.spec) > 5:
370 if self.is_command():
373 bl_name = self.spec[:len(self.spec)-5]
374 lin_ar = b.replace('noauto-','')
375 path = "/%s/%s/%s,%s.bz2" % (lin_ar.replace('-','/'), s, bl_name, rid)
377 if s.startswith("OK"):
379 bld = lin_ar.split('-')
380 tree_name = '-'.join(bld[:-1])
381 tree_arch = '-'.join(bld[-1:])
382 link_pre = "<a href=\"http://buildlogs.tld-linux.org/index.php?dist=%s&arch=%s&name=%s&id=%s&action=download\">" \
383 % (urllib.parse.quote(tree_name), urllib.parse.quote(tree_arch), urllib.parse.quote(bl_name), urllib.parse.quote(rid))
389 return time.asctime(time.localtime(t))
393 tooltip = "last update: %(time)s\nbuild time: %(buildtime)s" % {
394 'time' : ftime(self.builders_status_time[b]),
395 'buildtime' : ftime(self.builders_status_buildtime[b]),
397 builders.append(link_pre +
398 "<font color='%(color)s'><b title=\"%(tooltip)s\">%(builder)s:%(status)s</b></font>" % {
402 'tooltip' : html.escape(tooltip, True),
405 f.write("%s]</small></li>\n" % ' '.join(builders))
407 def rpmbuild_opts(self):
409 return all rpmbuild options related to this build
411 rpmopts = self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string()
413 "--define '_topdir %s' " % self.get_topdir() + \
414 "--define '_specdir %{_topdir}' " \
415 "--define '_sourcedir %{_specdir}' " \
416 "--define '_rpmdir %{_topdir}/RPMS' " \
417 "--define '_builddir %{_topdir}/BUILD' "
418 return rpmdefs + rpmopts
420 def php_ignores(self, php_version):
421 # available php versions in distro
422 php_versions = ['7.2', '7.3', '7.4', '8.0']
424 # remove current php version
426 php_versions.remove(php_version)
428 log.notice("Attempt to remove inexistent key '%s' from %s" % (php_version, php_versions))
431 # map them to poldek ignores
434 for v in list(map(php_ver_to_name, php_versions)):
435 res.append("php%s-*" % v)
439 # build ignore package list
440 # currently only php ignore is filled based on build context
444 # add php version based ignores
445 if 'php_suffix' in self.defines:
446 # current version if -D php_suffix is present
447 php_version = php_name_to_ver(self.defines['php_suffix'])
449 php_version = self.DEFAULT_PHP
451 ignores.extend(self.php_ignores(php_version))
453 # return empty string if the list is empty
454 if len(ignores) == 0:
458 return "--ignore=%s" % s
460 return " ".join(list(map(add_ignore, ignores)))
462 def kernel_string(self):
464 if self.kernel != "":
465 r = " --define 'alt_kernel " + self.kernel + "'"
468 def target_string(self):
469 if len(self.target) > 0:
470 return " --target " + ",".join(self.target)
474 def bconds_string(self):
476 for b in self.bconds_with:
477 r = r + " --with " + b
478 for b in self.bconds_without:
479 r = r + " --without " + b
482 def defines_string(self):
484 for key,value in self.defines.items():
485 r += " --define '%s %s'" % (key, value)
488 def defines_xml(self):
490 for key,value in self.defines.items():
491 r += "<define name='%s'>%s</define>\n" % (escape(key), escape(value))
494 def default_target(self, arch):
495 self.target.append("%s-tld-linux" % arch)
497 def write_to(self, f):
499 <batch id='%s' depends-on='%s'>
500 <src-rpm>%s</src-rpm>
501 <command flags="%s">%s</command>
504 <info>%s</info>\n""" % (self.b_id,
505 ' '.join(list(map(lambda b: b.b_id, self.depends_on))),
506 escape(self.src_rpm),
507 escape(' '.join(self.command_flags)), escape(self.command),
508 escape(self.spec), escape(self.branch), escape(self.info)))
509 if self.kernel != "":
510 f.write(" <kernel>%s</kernel>\n" % escape(self.kernel))
511 for b in self.bconds_with:
512 f.write(" <with>%s</with>\n" % escape(b))
513 for b in self.target:
514 f.write(" <target>%s</target>\n" % escape(b))
515 for b in self.bconds_without:
516 f.write(" <without>%s</without>\n" % escape(b))
518 f.write(" %s\n" % self.defines_xml())
519 for b in self.builders:
520 if b in self.builders_status_buildtime:
521 t = self.builders_status_buildtime[b]
524 f.write(" <builder status='%s' time='%s' buildtime='%s'>%s</builder>\n" % \
525 (escape(self.builders_status[b]), self.builders_status_time[b], t, escape(b)))
526 f.write(" </batch>\n")
528 def log_line(self, l):
530 if self.logfile != None:
531 util.append_to(self.logfile, l)
533 def expand_builders(batch, all_builders):
535 for bld in batch.builders:
537 for my_bld in all_builders:
538 if fnmatch.fnmatch(my_bld, bld):
547 def __init__(self, e):
549 self.kind = 'notification'
550 self.group_id = attr(e, "group-id")
551 self.builder = attr(e, "builder")
553 self.batches_buildtime = {}
554 for c in e.childNodes:
555 if is_blank(c): continue
556 if c.nodeType != Element.ELEMENT_NODE:
557 log.panic("xml: evil notification child %d" % c.nodeType)
558 if c.nodeName == "batch":
560 status = attr(c, "status")
561 buildtime = attr(c, "buildtime", "0")
562 if not status.startswith("OK") and not status.startswith("SKIP") and not status.startswith("UNSUPP") and not status.startswith("FAIL"):
563 log.panic("xml notification: bad status: %s" % status)
564 self.batches[id] = status
565 self.batches_buildtime[id] = buildtime
567 log.panic("xml: evil notification child (%s)" % c.nodeName)
569 # return structure usable for json encoding
573 builder=self.builder,
574 batches=self.batches,
575 batches_buildtime=self.batches_buildtime,
578 def apply_to(self, q):
580 if r.kind == "group":
582 if b.b_id in self.batches:
583 b.builders_status[self.builder] = self.batches[b.b_id]
584 b.builders_status_time[self.builder] = time.time()
585 b.builders_status_buildtime[self.builder] = "0" #self.batches_buildtime[b.b_id]
587 def build_request(e):
588 if e.nodeType != Element.ELEMENT_NODE:
589 log.panic("xml: evil request element")
590 if e.nodeName == "group":
592 elif e.nodeName == "notification":
593 return Notification(e)
594 elif e.nodeName == "command":
598 log.panic("xml: evil request [%s]" % e.nodeName)
600 def parse_request(f):
602 return build_request(d.documentElement)
604 def parse_requests(f):
607 for r in d.documentElement.childNodes:
608 if is_blank(r): continue
609 res.append(build_request(r))