TLD_Builder/request.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from xml.dom.minidom import *
   4 from datetime import datetime
   5 import string
   6 import time
   7 import xml.sax.saxutils
   8 import fnmatch
   9 import os
  10 import urllib
  11 import cgi
  12 import pytz
  13 import tempfile
  14
  15 import util
  16 import log
  17 from acl import acl
  18 from config import config
  19 from subprocess import call
  20
  21 __all__ = ['parse_request', 'parse_requests']
  22
  23 def text(e):
  24     res = ""
  25     for n in e.childNodes:
  26         if n.nodeType != Element.TEXT_NODE:
  27             log.panic("xml: text expected in <%s>, got %d" % (e.nodeName, n.nodeType))
  28         res += n.nodeValue
  29     return res
  30
  31 def attr(e, a, default = None):
  32     try:
  33         return e.attributes[a].value
  34     except:
  35         if default != None:
  36             return default
  37         raise
  38
  39 def escape(s):
  40     return xml.sax.saxutils.escape(s)
  41
  42 # return date in iso8601 format
  43 def iso8601(ts, timezone='UTC'):
  44     tz = pytz.timezone(timezone)
  45     dt = datetime.fromtimestamp(ts, tz)
  46     return dt.isoformat()
  47
  48 def is_blank(e):
  49     return e.nodeType == Element.TEXT_NODE and string.strip(e.nodeValue) == ""
  50
  51 class Group:
  52     def __init__(self, e):
  53         self.batches = []
  54         self.kind = 'group'
  55         self.id = attr(e, "id")
  56         self.no = int(attr(e, "no"))
  57         self.priority = 2
  58         self.time = time.time()
  59         self.requester = ""
  60         self.max_jobs = 0
  61         self.requester_email = ""
  62         self.flags = string.split(attr(e, "flags", ""))
  63         for c in e.childNodes:
  64             if is_blank(c): continue
  65
  66             if c.nodeType != Element.ELEMENT_NODE:
  67                 log.panic("xml: evil group child %d" % c.nodeType)
  68             if c.nodeName == "batch":
  69                 self.batches.append(Batch(c))
  70             elif c.nodeName == "requester":
  71                 self.requester = text(c)
  72                 self.requester_email = attr(c, "email", "")
  73             elif c.nodeName == "priority":
  74                 self.priority = int(text(c))
  75             elif c.nodeName == "time":
  76                 self.time = int(text(c))
  77             elif c.nodeName == "maxjobs":
  78                 self.max_jobs = int(text(c))
  79             else:
  80                 log.panic("xml: evil group child (%s)" % c.nodeName)
  81         # note that we also check that group is sorted WRT deps
  82         m = {}
  83         for b in self.batches:
  84             deps = []
  85             m[b.b_id] = b
  86             for dep in b.depends_on:
  87                 if m.has_key(dep):
  88                     # avoid self-deps
  89                     if id(m[dep]) != id(b):
  90                         deps.append(m[dep])
  91                 else:
  92                     log.panic("xml: dependency not found in group")
  93             b.depends_on = deps
  94         if self.requester_email == "" and self.requester != "":
  95             self.requester_email = acl.user(self.requester).mail_to()
  96
  97     def dump(self, f):
  98         f.write("group: %d (id=%s pri=%d)\n" % (self.no, self.id, self.priority))
  99         f.write("  from: %s\n" % self.requester)
 100         f.write("  flags: %s\n" % string.join(self.flags))
 101         f.write("  time: %s\n" % time.asctime(time.localtime(self.time)))
 102         for b in self.batches:
 103             b.dump(f)
 104         f.write("\n")
 105
 106     def dump_html(self, f):
 107         f.write(
 108             "<div id=\"%(no)d\" class=\"request %(flags)s\">\n"
 109             "<a href=\"#%(no)d\">%(no)d</a>. "
 110             "<time class=\"timeago\" datetime=\"%(datetime)s\">%(time)s</time> "
 111             "from <b class=requester>%(requester)s</b> "
 112             "<small>%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s</small>\n"
 113         % {
 114             'no': self.no,
 115             'id': '<a href="srpms/%(id)s">%(id)s</a>' % {'id': self.id},
 116             'time': escape(time.strftime("%a %b %d %Y %H:%M:%S %z", time.localtime(self.time))),
 117             'datetime': escape(iso8601(self.time)),
 118             'requester': escape(self.requester),
 119             'priority': self.priority,
 120             'max_jobs': self.max_jobs,
 121             'flags': string.join(self.flags)
 122         })
 123         f.write("<ol>\n")
 124         for b in self.batches:
 125             b.dump_html(f, self.id)
 126         f.write("</ol>\n")
 127         f.write("</div>\n")
 128
 129     def write_to(self, f):
 130         f.write("""
 131        <group id="%s" no="%d" flags="%s">
 132          <requester email='%s'>%s</requester>
 133          <time>%d</time>
 134          <priority>%d</priority>
 135          <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, string.join(self.flags),
 136                     escape(self.requester_email), escape(self.requester),
 137                     self.time, self.priority, self.max_jobs))
 138         for b in self.batches:
 139             b.write_to(f)
 140         f.write("       </group>\n\n")
 141
 142     def is_done(self):
 143         ok = 1
 144         for b in self.batches:
 145             if not b.is_done():
 146                 ok = 0
 147         return ok
 148
 149 class Batch:
 150     def __init__(self, e):
 151         self.bconds_with = []
 152         self.bconds_without = []
 153         self.builders = []
 154         self.builders_status = {}
 155         self.builders_status_time = {}
 156         self.builders_status_buildtime = {}
 157         self.kernel = ""
 158         self.defines = {}
 159         self.target = []
 160         self.branch = ""
 161         self.src_rpm = ""
 162         self.info = ""
 163         self.spec = ""
 164         self.command = ""
 165         self.command_flags = []
 166         self.skip = []
 167         self.gb_id = ""
 168         self.b_id = attr(e, "id")
 169         self.depends_on = string.split(attr(e, "depends-on"))
 170         self.upgraded = True
 171
 172         self.parse_xml(e)
 173
 174         self.__topdir = None
 175
 176     def get_topdir(self):
 177         if not self.__topdir:
 178             self.__topdir = tempfile.mkdtemp(prefix='B.', dir='/tmp')
 179         return self.__topdir
 180
 181     def parse_xml(self, e):
 182         for c in e.childNodes:
 183             if is_blank(c): continue
 184
 185             if c.nodeType != Element.ELEMENT_NODE:
 186                 log.panic("xml: evil batch child %d" % c.nodeType)
 187             if c.nodeName == "src-rpm":
 188                 self.src_rpm = text(c)
 189             elif c.nodeName == "spec":
 190                 # normalize specname, specname is used as buildlog and we don't
 191                 # want to be exposed to directory traversal attacks
 192                 self.spec = text(c).split('/')[-1]
 193             elif c.nodeName == "command":
 194                 self.spec = "COMMAND"
 195                 self.command = text(c).strip()
 196                 self.command_flags = string.split(attr(c, "flags", ""))
 197             elif c.nodeName == "info":
 198                 self.info = text(c)
 199             elif c.nodeName == "kernel":
 200                 self.kernel = text(c)
 201             elif c.nodeName == "define":
 202                 define = attr(c, "name")
 203                 self.defines[define] = text(c)
 204             elif c.nodeName == "target":
 205                 self.target.append(text(c))
 206             elif c.nodeName == "skip":
 207                 self.skip.append(text(c))
 208             elif c.nodeName == "branch":
 209                 self.branch = text(c)
 210             elif c.nodeName == "builder":
 211                 key = text(c)
 212                 self.builders.append(key)
 213                 self.builders_status[key] = attr(c, "status", "?")
 214                 self.builders_status_time[key] = attr(c, "time", "0")
 215                 self.builders_status_buildtime[key] = "0" #attr(c, "buildtime", "0")
 216             elif c.nodeName == "with":
 217                 self.bconds_with.append(text(c))
 218             elif c.nodeName == "without":
 219                 self.bconds_without.append(text(c))
 220             else:
 221                 log.panic("xml: evil batch child (%s)" % c.nodeName)
 222
 223     def get_package_name(self):
 224         if len(self.spec) <= 5:
 225             return None
 226         return self.spec[:-5]
 227
 228     def tmpdir(self):
 229         """
 230         return tmpdir for this batch job building
 231         """
 232         # it's better to have TMPDIR and BUILD dir on same partition:
 233         # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2
 234         # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory
 235         path = os.path.join(self.get_topdir(), 'BUILD', 'tmp')
 236         return path
 237
 238     def is_done(self):
 239         ok = 1
 240         for b in self.builders:
 241             s = self.builders_status[b]
 242             if not s.startswith("OK") and not s.startswith("SKIP") and not s.startswith("UNSUPP") and not s.startswith("FAIL"):
 243                 ok = 0
 244         return ok
 245
 246     def dump(self, f):
 247         f.write("  batch: %s/%s\n" % (self.src_rpm, self.spec))
 248         f.write("    info: %s\n" % self.info)
 249         f.write("    kernel: %s\n" % self.kernel)
 250         f.write("    defines: %s\n" % self.defines_string())
 251         f.write("    target: %s\n" % self.target_string())
 252         f.write("    branch: %s\n" % self.branch)
 253         f.write("    bconds: %s\n" % self.bconds_string())
 254         builders = []
 255         for b in self.builders:
 256             builders.append("%s:%s" % (b, self.builders_status[b]))
 257         f.write("    builders: %s\n" % string.join(builders))
 258
 259     def is_command(self):
 260         return self.command != ""
 261
 262     def dump_html(self, f, rid):
 263         f.write("<li>\n")
 264         if self.is_command():
 265             desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
 266         else:
 267             cmd = "/usr/bin/git ls-remote --heads git://git.tld-linux.org/packages/%s 1>/dev/null 2>&1" % (self.spec[:-5])
 268             r = call(cmd, shell=True)
 269             if r == 0:
 270                 dist = "tld"
 271             else:
 272                 dist = "pld"
 273             package_url = "http://git.%(dist)s-linux.org/?p=packages/%(package)s.git;a=blob;f=%(spec)s;hb=%(branch)s" % {
 274                 'dist': dist,
 275                 'spec': urllib.quote(self.spec),
 276                 'branch': urllib.quote(self.branch),
 277                 'package': urllib.quote(self.spec[:-5]),
 278             }
 279             desc = "%(src_rpm)s (<a href=\"%(package_url)s\">%(spec)s -r %(branch)s</a>%(rpmopts)s)" % {
 280                 'src_rpm': self.src_rpm,
 281                 'spec': self.spec,
 282                 'branch': self.branch,
 283                 'rpmopts': self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string(),
 284                 'package_url': package_url,
 285             }
 286         f.write("%s <small>[" % desc)
 287         builders = []
 288         for b in self.builders:
 289             s = self.builders_status[b]
 290             if s.startswith("OK"):
 291                 c = "green"
 292             elif s.startswith("FAIL"):
 293                 c = "red"
 294             elif s.startswith("SKIP"):
 295                 c = "blue"
 296             elif s.startswith("UNSUPP"):
 297                 c = "fuchsia"
 298             else:
 299                 c = "black"
 300             link_pre = ""
 301             link_post = ""
 302             if (s.startswith("OK") or s.startswith("SKIP") or s.startswith("UNSUPP") or s.startswith("FAIL")) and len(self.spec) > 5:
 303                 if self.is_command():
 304                     bl_name = "command"
 305                 else:
 306                     bl_name = self.spec[:len(self.spec)-5]
 307                 lin_ar = b.replace('noauto-','')
 308                 path = "/%s/%s/%s,%s.bz2" % (lin_ar.replace('-','/'), s, bl_name, rid)
 309                 is_ok = 0
 310                 if s.startswith("OK"):
 311                     is_ok = 1
 312                 bld = lin_ar.split('-')
 313                 tree_name = '-'.join(bld[:-1])
 314                 tree_arch = '-'.join(bld[-1:])
 315                 link_pre = "<a href=\"http://buildlogs.tld-linux.org/index.php?dist=%s&arch=%s&name=%s&id=%s&action=download\">" \
 316                         % (urllib.quote(tree_name), urllib.quote(tree_arch), urllib.quote(bl_name), urllib.quote(rid))
 317                 link_post = "</a>"
 318
 319             def ftime(s):
 320                 t = float(s)
 321                 if t > 0:
 322                     return time.asctime(time.localtime(t))
 323                 else:
 324                     return 'N/A'
 325
 326             tooltip = "last update: %(time)s\nbuild time: %(buildtime)s" % {
 327                 'time' : ftime(self.builders_status_time[b]),
 328                 'buildtime' : ftime(self.builders_status_buildtime[b]),
 329             }
 330             builders.append(link_pre +
 331                 "<font color='%(color)s'><b title=\"%(tooltip)s\">%(builder)s:%(status)s</b></font>" % {
 332                     'color' : c,
 333                     'builder' : b,
 334                     'status' : s,
 335                     'tooltip' : cgi.escape(tooltip, True),
 336             }
 337             + link_post)
 338         f.write("%s]</small></li>\n" % string.join(builders))
 339
 340     def rpmbuild_opts(self):
 341         """
 342             return all rpmbuild options related to this build
 343         """
 344         rpmopts = self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string()
 345         rpmdefs = \
 346             "--define '_topdir %s' " % self.get_topdir() + \
 347             "--define '_specdir %{_topdir}' "  \
 348             "--define '_sourcedir %{_specdir}' " \
 349             "--define '_rpmdir %{_topdir}/RPMS' " \
 350             "--define '_builddir %{_topdir}/BUILD' "
 351         return rpmdefs + rpmopts
 352
 353     def php_ignores(self):
 354         # transform php package name (52) to version (5.2)
 355         def php_name_to_ver(v):
 356             return '.'.join(list(v))
 357
 358         # transform php version (5.2) to package name (52)
 359         def php_ver_to_name(v):
 360             return v.replace('.', '')
 361
 362         # available php versions in distro
 363         php_versions = ['4', '5.2', '5.3', '5.4', '5.5', '5.6', '7.0']
 364
 365         # current version if -D php_suffix is present
 366         php_version = php_name_to_ver(self.defines['php_suffix'])
 367
 368         # remove current php version
 369         try:
 370             php_versions.remove(php_version)
 371         except ValueError:
 372             log.notice("Attempt to remove inexistent key '%s' from %s" % (php_version, php_versions))
 373             pass
 374
 375         # map them to poldek ignores
 376         # always ignore hhvm
 377         res = ['hhvm-*']
 378         for v in map(php_ver_to_name, php_versions):
 379             res.append("php%s-*" % v)
 380
 381         return res
 382
 383     # build ignore package list
 384     # currently only php ignore is filled based on build context
 385     def ignores(self):
 386         ignores = []
 387
 388         # add php version based ignores
 389         if self.defines.has_key('php_suffix'):
 390             ignores.extend(self.php_ignores())
 391
 392         # return empty string if the list is empty
 393         if len(ignores) == 0:
 394             return ""
 395
 396         def add_ignore(s):
 397             return "--ignore=%s" % s
 398
 399         return " ".join(map(add_ignore, ignores))
 400
 401     def kernel_string(self):
 402         r = ""
 403         if self.kernel != "":
 404             r = " --define 'alt_kernel " + self.kernel + "'"
 405         return r
 406
 407     def target_string(self):
 408         if len(self.target) > 0:
 409             return " --target " + ",".join(self.target)
 410         else:
 411             return ""
 412
 413     def bconds_string(self):
 414         r = ""
 415         for b in self.bconds_with:
 416             r = r + " --with " + b
 417         for b in self.bconds_without:
 418             r = r + " --without " + b
 419         return r
 420
 421     def defines_string(self):
 422         r = ""
 423         for key,value in self.defines.items():
 424             r += " --define '%s %s'" % (key, value)
 425         return r
 426
 427     def defines_xml(self):
 428         r = ""
 429         for key,value in self.defines.items():
 430             r += "<define name='%s'>%s</define>\n" % (escape(key), escape(value))
 431         return r
 432
 433     def default_target(self, arch):
 434         self.target.append("%s-tld-linux" % arch)
 435
 436     def write_to(self, f):
 437         f.write("""
 438          <batch id='%s' depends-on='%s'>
 439            <src-rpm>%s</src-rpm>
 440            <command flags="%s">%s</command>
 441            <spec>%s</spec>
 442            <branch>%s</branch>
 443            <info>%s</info>\n""" % (self.b_id,
 444                  string.join(map(lambda (b): b.b_id, self.depends_on)),
 445                  escape(self.src_rpm),
 446                  escape(' '.join(self.command_flags)), escape(self.command),
 447                  escape(self.spec), escape(self.branch), escape(self.info)))
 448         if self.kernel != "":
 449             f.write("           <kernel>%s</kernel>\n" % escape(self.kernel))
 450         for b in self.bconds_with:
 451             f.write("           <with>%s</with>\n" % escape(b))
 452         for b in self.target:
 453             f.write("           <target>%s</target>\n" % escape(b))
 454         for b in self.bconds_without:
 455             f.write("           <without>%s</without>\n" % escape(b))
 456         if self.defines:
 457             f.write("           %s\n" % self.defines_xml())
 458         for b in self.builders:
 459             if self.builders_status_buildtime.has_key(b):
 460                 t = self.builders_status_buildtime[b]
 461             else:
 462                 t = "0"
 463             f.write("           <builder status='%s' time='%s' buildtime='%s'>%s</builder>\n" % \
 464                     (escape(self.builders_status[b]), self.builders_status_time[b], t, escape(b)))
 465         f.write("         </batch>\n")
 466
 467     def log_line(self, l):
 468         log.notice(l)
 469         if self.logfile != None:
 470             util.append_to(self.logfile, l)
 471
 472     def expand_builders(batch, all_builders):
 473         all = []
 474         for bld in batch.builders:
 475             res = []
 476             for my_bld in all_builders:
 477                 if fnmatch.fnmatch(my_bld, bld):
 478                     res.append(my_bld)
 479             if res != []:
 480                 all.extend(res)
 481             else:
 482                 all.append(bld)
 483         batch.builders = all
 484
 485 class Notification:
 486     def __init__(self, e):
 487         self.batches = []
 488         self.kind = 'notification'
 489         self.group_id = attr(e, "group-id")
 490         self.builder = attr(e, "builder")
 491         self.batches = {}
 492         self.batches_buildtime = {}
 493         for c in e.childNodes:
 494             if is_blank(c): continue
 495             if c.nodeType != Element.ELEMENT_NODE:
 496                 log.panic("xml: evil notification child %d" % c.nodeType)
 497             if c.nodeName == "batch":
 498                 id = attr(c, "id")
 499                 status = attr(c, "status")
 500                 buildtime = attr(c, "buildtime", "0")
 501                 if not status.startswith("OK") and not status.startswith("SKIP") and not status.startswith("UNSUPP") and not status.startswith("FAIL"):
 502                     log.panic("xml notification: bad status: %s" % status)
 503                 self.batches[id] = status
 504                 self.batches_buildtime[id] = buildtime
 505             else:
 506                 log.panic("xml: evil notification child (%s)" % c.nodeName)
 507
 508     def apply_to(self, q):
 509         for r in q.requests:
 510             if r.kind == "group":
 511                 for b in r.batches:
 512                     if self.batches.has_key(b.b_id):
 513                         b.builders_status[self.builder] = self.batches[b.b_id]
 514                         b.builders_status_time[self.builder] = time.time()
 515                         b.builders_status_buildtime[self.builder] = "0" #self.batches_buildtime[b.b_id]
 516
 517 def build_request(e):
 518     if e.nodeType != Element.ELEMENT_NODE:
 519         log.panic("xml: evil request element")
 520     if e.nodeName == "group":
 521         return Group(e)
 522     elif e.nodeName == "notification":
 523         return Notification(e)
 524     elif e.nodeName == "command":
 525         # FIXME
 526         return Command(e)
 527     else:
 528         log.panic("xml: evil request [%s]" % e.nodeName)
 529
 530 def parse_request(f):
 531     d = parseString(f)
 532     return build_request(d.documentElement)
 533
 534 def parse_requests(f):
 535     d = parseString(f)
 536     res = []
 537     for r in d.documentElement.childNodes:
 538         if is_blank(r): continue
 539         res.append(build_request(r))
 540     return res