--- /dev/null
+# vi: encoding=utf-8 ts=8 sts=4 sw=4 et
+
+from xml.dom.minidom import *
+from datetime import datetime
+import string
+import time
+import xml.sax.saxutils
+import fnmatch
+import os
+import urllib
+import cgi
+import pytz
+import tempfile
+
+import util
+import log
+from acl import acl
+from config import config
+from subprocess import call
+
+__all__ = ['parse_request', 'parse_requests']
+
+def text(e):
+ res = ""
+ for n in e.childNodes:
+ if n.nodeType != Element.TEXT_NODE:
+ log.panic("xml: text expected in <%s>, got %d" % (e.nodeName, n.nodeType))
+ res += n.nodeValue
+ return res
+
+def attr(e, a, default = None):
+ try:
+ return e.attributes[a].value
+ except:
+ if default != None:
+ return default
+ raise
+
+def escape(s):
+ return xml.sax.saxutils.escape(s)
+
+# return date in iso8601 format
+def iso8601(ts, timezone='UTC'):
+ tz = pytz.timezone(timezone)
+ dt = datetime.fromtimestamp(ts, tz)
+ return dt.isoformat()
+
+def is_blank(e):
+ return e.nodeType == Element.TEXT_NODE and string.strip(e.nodeValue) == ""
+
+class Group:
+ def __init__(self, e):
+ self.batches = []
+ self.kind = 'group'
+ self.id = attr(e, "id")
+ self.no = int(attr(e, "no"))
+ self.priority = 2
+ self.time = time.time()
+ self.requester = ""
+ self.max_jobs = 0
+ self.requester_email = ""
+ self.flags = string.split(attr(e, "flags", ""))
+ for c in e.childNodes:
+ if is_blank(c): continue
+
+ if c.nodeType != Element.ELEMENT_NODE:
+ log.panic("xml: evil group child %d" % c.nodeType)
+ if c.nodeName == "batch":
+ self.batches.append(Batch(c))
+ elif c.nodeName == "requester":
+ self.requester = text(c)
+ self.requester_email = attr(c, "email", "")
+ elif c.nodeName == "priority":
+ self.priority = int(text(c))
+ elif c.nodeName == "time":
+ self.time = int(text(c))
+ elif c.nodeName == "maxjobs":
+ self.max_jobs = int(text(c))
+ else:
+ log.panic("xml: evil group child (%s)" % c.nodeName)
+ # note that we also check that group is sorted WRT deps
+ m = {}
+ for b in self.batches:
+ deps = []
+ m[b.b_id] = b
+ for dep in b.depends_on:
+ if m.has_key(dep):
+ # avoid self-deps
+ if id(m[dep]) != id(b):
+ deps.append(m[dep])
+ else:
+ log.panic("xml: dependency not found in group")
+ b.depends_on = deps
+ if self.requester_email == "" and self.requester != "":
+ self.requester_email = acl.user(self.requester).mail_to()
+
+ def dump(self, f):
+ f.write("group: %d (id=%s pri=%d)\n" % (self.no, self.id, self.priority))
+ f.write(" from: %s\n" % self.requester)
+ f.write(" flags: %s\n" % string.join(self.flags))
+ f.write(" time: %s\n" % time.asctime(time.localtime(self.time)))
+ for b in self.batches:
+ b.dump(f)
+ f.write("\n")
+
+ def dump_html(self, f):
+ f.write(
+ "<div id=\"%(no)d\" class=\"request %(flags)s\">\n"
+ "<a href=\"#%(no)d\">%(no)d</a>. "
+ "<time class=\"timeago\" datetime=\"%(datetime)s\">%(time)s</time> "
+ "from <b class=requester>%(requester)s</b> "
+ "<small>%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s</small>\n"
+ % {
+ 'no': self.no,
+ 'id': '<a href="srpms/%(id)s">%(id)s</a>' % {'id': self.id},
+ 'time': escape(time.strftime("%a %b %d %Y %H:%M:%S %z", time.localtime(self.time))),
+ 'datetime': escape(iso8601(self.time)),
+ 'requester': escape(self.requester),
+ 'priority': self.priority,
+ 'max_jobs': self.max_jobs,
+ 'flags': string.join(self.flags)
+ })
+ f.write("<ol>\n")
+ for b in self.batches:
+ b.dump_html(f, self.id)
+ f.write("</ol>\n")
+ f.write("</div>\n")
+
+ def write_to(self, f):
+ f.write("""
+ <group id="%s" no="%d" flags="%s">
+ <requester email='%s'>%s</requester>
+ <time>%d</time>
+ <priority>%d</priority>
+ <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, string.join(self.flags),
+ escape(self.requester_email), escape(self.requester),
+ self.time, self.priority, self.max_jobs))
+ for b in self.batches:
+ b.write_to(f)
+ f.write(" </group>\n\n")
+
+ def is_done(self):
+ ok = 1
+ for b in self.batches:
+ if not b.is_done():
+ ok = 0
+ return ok
+
+class Batch:
+ def __init__(self, e):
+ self.bconds_with = []
+ self.bconds_without = []
+ self.builders = []
+ self.builders_status = {}
+ self.builders_status_time = {}
+ self.builders_status_buildtime = {}
+ self.kernel = ""
+ self.defines = {}
+ self.target = []
+ self.branch = ""
+ self.src_rpm = ""
+ self.info = ""
+ self.spec = ""
+ self.command = ""
+ self.command_flags = []
+ self.skip = []
+ self.gb_id = ""
+ self.b_id = attr(e, "id")
+ self.depends_on = string.split(attr(e, "depends-on"))
+ self.upgraded = True
+
+ self.parse_xml(e)
+
+ self.__topdir = None
+
+ def get_topdir(self):
+ if not self.__topdir:
+ self.__topdir = tempfile.mkdtemp(prefix='B.', dir='/tmp')
+ return self.__topdir
+
+ def parse_xml(self, e):
+ for c in e.childNodes:
+ if is_blank(c): continue
+
+ if c.nodeType != Element.ELEMENT_NODE:
+ log.panic("xml: evil batch child %d" % c.nodeType)
+ if c.nodeName == "src-rpm":
+ self.src_rpm = text(c)
+ elif c.nodeName == "spec":
+ # normalize specname, specname is used as buildlog and we don't
+ # want to be exposed to directory traversal attacks
+ self.spec = text(c).split('/')[-1]
+ elif c.nodeName == "command":
+ self.spec = "COMMAND"
+ self.command = text(c).strip()
+ self.command_flags = string.split(attr(c, "flags", ""))
+ elif c.nodeName == "info":
+ self.info = text(c)
+ elif c.nodeName == "kernel":
+ self.kernel = text(c)
+ elif c.nodeName == "define":
+ define = attr(c, "name")
+ self.defines[define] = text(c)
+ elif c.nodeName == "target":
+ self.target.append(text(c))
+ elif c.nodeName == "skip":
+ self.skip.append(text(c))
+ elif c.nodeName == "branch":
+ self.branch = text(c)
+ elif c.nodeName == "builder":
+ key = text(c)
+ self.builders.append(key)
+ self.builders_status[key] = attr(c, "status", "?")
+ self.builders_status_time[key] = attr(c, "time", "0")
+ self.builders_status_buildtime[key] = "0" #attr(c, "buildtime", "0")
+ elif c.nodeName == "with":
+ self.bconds_with.append(text(c))
+ elif c.nodeName == "without":
+ self.bconds_without.append(text(c))
+ else:
+ log.panic("xml: evil batch child (%s)" % c.nodeName)
+
+ def get_package_name(self):
+ if len(self.spec) <= 5:
+ return None
+ return self.spec[:-5]
+
+ def tmpdir(self):
+ """
+ return tmpdir for this batch job building
+ """
+ # it's better to have TMPDIR and BUILD dir on same partition:
+ # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2
+ # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory
+ path = os.path.join(self.get_topdir(), 'BUILD', 'tmp')
+ return path
+
+ def is_done(self):
+ ok = 1
+ for b in self.builders:
+ s = self.builders_status[b]
+ if not s.startswith("OK") and not s.startswith("SKIP") and not s.startswith("UNSUPP") and not s.startswith("FAIL"):
+ ok = 0
+ return ok
+
+ def dump(self, f):
+ f.write(" batch: %s/%s\n" % (self.src_rpm, self.spec))
+ f.write(" info: %s\n" % self.info)
+ f.write(" kernel: %s\n" % self.kernel)
+ f.write(" defines: %s\n" % self.defines_string())
+ f.write(" target: %s\n" % self.target_string())
+ f.write(" branch: %s\n" % self.branch)
+ f.write(" bconds: %s\n" % self.bconds_string())
+ builders = []
+ for b in self.builders:
+ builders.append("%s:%s" % (b, self.builders_status[b]))
+ f.write(" builders: %s\n" % string.join(builders))
+
+ def is_command(self):
+ return self.command != ""
+
+ def dump_html(self, f, rid):
+ f.write("<li>\n")
+ if self.is_command():
+ desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
+ else:
+ cmd = "/usr/bin/git ls-remote --heads git://git.tld-linux.org/packages/%s 1>/dev/null 2>&1" % (self.spec[:-5])
+ r = call(cmd, shell=True)
+ if r == 0:
+ dist = "tld"
+ else:
+ dist = "pld"
+ package_url = "http://git.%(dist)s-linux.org/?p=packages/%(package)s.git;a=blob;f=%(spec)s;hb=%(branch)s" % {
+ 'dist': dist,
+ 'spec': urllib.quote(self.spec),
+ 'branch': urllib.quote(self.branch),
+ 'package': urllib.quote(self.spec[:-5]),
+ }
+ desc = "%(src_rpm)s (<a href=\"%(package_url)s\">%(spec)s -r %(branch)s</a>%(rpmopts)s)" % {
+ 'src_rpm': self.src_rpm,
+ 'spec': self.spec,
+ 'branch': self.branch,
+ 'rpmopts': self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string(),
+ 'package_url': package_url,
+ }
+ f.write("%s <small>[" % desc)
+ builders = []
+ for b in self.builders:
+ s = self.builders_status[b]
+ if s.startswith("OK"):
+ c = "green"
+ elif s.startswith("FAIL"):
+ c = "red"
+ elif s.startswith("SKIP"):
+ c = "blue"
+ elif s.startswith("UNSUPP"):
+ c = "fuchsia"
+ else:
+ c = "black"
+ link_pre = ""
+ link_post = ""
+ if (s.startswith("OK") or s.startswith("SKIP") or s.startswith("UNSUPP") or s.startswith("FAIL")) and len(self.spec) > 5:
+ if self.is_command():
+ bl_name = "command"
+ else:
+ bl_name = self.spec[:len(self.spec)-5]
+ lin_ar = b.replace('noauto-','')
+ path = "/%s/%s/%s,%s.bz2" % (lin_ar.replace('-','/'), s, bl_name, rid)
+ is_ok = 0
+ if s.startswith("OK"):
+ is_ok = 1
+ bld = lin_ar.split('-')
+ tree_name = '-'.join(bld[:-1])
+ tree_arch = '-'.join(bld[-1:])
+ link_pre = "<a href=\"http://buildlogs.tld-linux.org/index.php?dist=%s&arch=%s&name=%s&id=%s&action=download\">" \
+ % (urllib.quote(tree_name), urllib.quote(tree_arch), urllib.quote(bl_name), urllib.quote(rid))
+ link_post = "</a>"
+
+ def ftime(s):
+ t = float(s)
+ if t > 0:
+ return time.asctime(time.localtime(t))
+ else:
+ return 'N/A'
+
+ tooltip = "last update: %(time)s\nbuild time: %(buildtime)s" % {
+ 'time' : ftime(self.builders_status_time[b]),
+ 'buildtime' : ftime(self.builders_status_buildtime[b]),
+ }
+ builders.append(link_pre +
+ "<font color='%(color)s'><b title=\"%(tooltip)s\">%(builder)s:%(status)s</b></font>" % {
+ 'color' : c,
+ 'builder' : b,
+ 'status' : s,
+ 'tooltip' : cgi.escape(tooltip, True),
+ }
+ + link_post)
+ f.write("%s]</small></li>\n" % string.join(builders))
+
+ def rpmbuild_opts(self):
+ """
+ return all rpmbuild options related to this build
+ """
+ rpmopts = self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string()
+ rpmdefs = \
+ "--define '_topdir %s' " % self.get_topdir() + \
+ "--define '_specdir %{_topdir}' " \
+ "--define '_sourcedir %{_specdir}' " \
+ "--define '_rpmdir %{_topdir}/RPMS' " \
+ "--define '_builddir %{_topdir}/BUILD' "
+ return rpmdefs + rpmopts
+
+ def php_ignores(self):
+ # transform php package name (52) to version (5.2)
+ def php_name_to_ver(v):
+ return '.'.join(list(v))
+
+ # transform php version (5.2) to package name (52)
+ def php_ver_to_name(v):
+ return v.replace('.', '')
+
+ # available php versions in distro
+ php_versions = ['4', '5.2', '5.3', '5.4', '5.5', '5.6', '7.0']
+
+ # current version if -D php_suffix is present
+ php_version = php_name_to_ver(self.defines['php_suffix'])
+
+ # remove current php version
+ try:
+ php_versions.remove(php_version)
+ except ValueError:
+ log.notice("Attempt to remove inexistent key '%s' from %s" % (php_version, php_versions))
+ pass
+
+ # map them to poldek ignores
+ # always ignore hhvm
+ res = ['hhvm-*']
+ for v in map(php_ver_to_name, php_versions):
+ res.append("php%s-*" % v)
+
+ return res
+
+ # build ignore package list
+ # currently only php ignore is filled based on build context
+ def ignores(self):
+ ignores = []
+
+ # add php version based ignores
+ if self.defines.has_key('php_suffix'):
+ ignores.extend(self.php_ignores())
+
+ # return empty string if the list is empty
+ if len(ignores) == 0:
+ return ""
+
+ def add_ignore(s):
+ return "--ignore=%s" % s
+
+ return " ".join(map(add_ignore, ignores))
+
+ def kernel_string(self):
+ r = ""
+ if self.kernel != "":
+ r = " --define 'alt_kernel " + self.kernel + "'"
+ return r
+
+ def target_string(self):
+ if len(self.target) > 0:
+ return " --target " + ",".join(self.target)
+ else:
+ return ""
+
+ def bconds_string(self):
+ r = ""
+ for b in self.bconds_with:
+ r = r + " --with " + b
+ for b in self.bconds_without:
+ r = r + " --without " + b
+ return r
+
+ def defines_string(self):
+ r = ""
+ for key,value in self.defines.items():
+ r += " --define '%s %s'" % (key, value)
+ return r
+
+ def defines_xml(self):
+ r = ""
+ for key,value in self.defines.items():
+ r += "<define name='%s'>%s</define>\n" % (escape(key), escape(value))
+ return r
+
+ def default_target(self, arch):
+ self.target.append("%s-tld-linux" % arch)
+
+ def write_to(self, f):
+ f.write("""
+ <batch id='%s' depends-on='%s'>
+ <src-rpm>%s</src-rpm>
+ <command flags="%s">%s</command>
+ <spec>%s</spec>
+ <branch>%s</branch>
+ <info>%s</info>\n""" % (self.b_id,
+ string.join(map(lambda (b): b.b_id, self.depends_on)),
+ escape(self.src_rpm),
+ escape(' '.join(self.command_flags)), escape(self.command),
+ escape(self.spec), escape(self.branch), escape(self.info)))
+ if self.kernel != "":
+ f.write(" <kernel>%s</kernel>\n" % escape(self.kernel))
+ for b in self.bconds_with:
+ f.write(" <with>%s</with>\n" % escape(b))
+ for b in self.target:
+ f.write(" <target>%s</target>\n" % escape(b))
+ for b in self.bconds_without:
+ f.write(" <without>%s</without>\n" % escape(b))
+ if self.defines:
+ f.write(" %s\n" % self.defines_xml())
+ for b in self.builders:
+ if self.builders_status_buildtime.has_key(b):
+ t = self.builders_status_buildtime[b]
+ else:
+ t = "0"
+ f.write(" <builder status='%s' time='%s' buildtime='%s'>%s</builder>\n" % \
+ (escape(self.builders_status[b]), self.builders_status_time[b], t, escape(b)))
+ f.write(" </batch>\n")
+
+ def log_line(self, l):
+ log.notice(l)
+ if self.logfile != None:
+ util.append_to(self.logfile, l)
+
+ def expand_builders(batch, all_builders):
+ all = []
+ for bld in batch.builders:
+ res = []
+ for my_bld in all_builders:
+ if fnmatch.fnmatch(my_bld, bld):
+ res.append(my_bld)
+ if res != []:
+ all.extend(res)
+ else:
+ all.append(bld)
+ batch.builders = all
+
+class Notification:
+ def __init__(self, e):
+ self.batches = []
+ self.kind = 'notification'
+ self.group_id = attr(e, "group-id")
+ self.builder = attr(e, "builder")
+ self.batches = {}
+ self.batches_buildtime = {}
+ for c in e.childNodes:
+ if is_blank(c): continue
+ if c.nodeType != Element.ELEMENT_NODE:
+ log.panic("xml: evil notification child %d" % c.nodeType)
+ if c.nodeName == "batch":
+ id = attr(c, "id")
+ status = attr(c, "status")
+ buildtime = attr(c, "buildtime", "0")
+ if not status.startswith("OK") and not status.startswith("SKIP") and not status.startswith("UNSUPP") and not status.startswith("FAIL"):
+ log.panic("xml notification: bad status: %s" % status)
+ self.batches[id] = status
+ self.batches_buildtime[id] = buildtime
+ else:
+ log.panic("xml: evil notification child (%s)" % c.nodeName)
+
+ def apply_to(self, q):
+ for r in q.requests:
+ if r.kind == "group":
+ for b in r.batches:
+ if self.batches.has_key(b.b_id):
+ b.builders_status[self.builder] = self.batches[b.b_id]
+ b.builders_status_time[self.builder] = time.time()
+ b.builders_status_buildtime[self.builder] = "0" #self.batches_buildtime[b.b_id]
+
+def build_request(e):
+ if e.nodeType != Element.ELEMENT_NODE:
+ log.panic("xml: evil request element")
+ if e.nodeName == "group":
+ return Group(e)
+ elif e.nodeName == "notification":
+ return Notification(e)
+ elif e.nodeName == "command":
+ # FIXME
+ return Command(e)
+ else:
+ log.panic("xml: evil request [%s]" % e.nodeName)
+
+def parse_request(f):
+ d = parseString(f)
+ return build_request(d.documentElement)
+
+def parse_requests(f):
+ d = parseString(f)
+ res = []
+ for r in d.documentElement.childNodes:
+ if is_blank(r): continue
+ res.append(build_request(r))
+ return res
TLD / tld-builder.git / blobdiff