PLD_Builder/request.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from xml.dom.minidom import *
   4 import string
   5 import time
   6 import xml.sax.saxutils
   7 import fnmatch
   8 import os
   9 import urllib
  10 import cgi
  11
  12 import util
  13 import log
  14 from acl import acl
  15 from config import config
  16 from subprocess import call
  17
  18 __all__ = ['parse_request', 'parse_requests']
  19
  20 def text(e):
  21     res = ""
  22     for n in e.childNodes:
  23         if n.nodeType != Element.TEXT_NODE:
  24             log.panic("xml: text expected in <%s>, got %d" % (e.nodeName, n.nodeType))
  25         res += n.nodeValue
  26     return res
  27
  28 def attr(e, a, default = None):
  29     try:
  30         return e.attributes[a].value
  31     except:
  32         if default != None:
  33             return default
  34         raise
  35
  36 def escape(s):
  37     return xml.sax.saxutils.escape(s)
  38
  39 def is_blank(e):
  40     return e.nodeType == Element.TEXT_NODE and string.strip(e.nodeValue) == ""
  41
  42 class Group:
  43     def __init__(self, e):
  44         self.batches = []
  45         self.kind = 'group'
  46         self.id = attr(e, "id")
  47         self.no = int(attr(e, "no"))
  48         self.priority = 2
  49         self.time = time.time()
  50         self.requester = ""
  51         self.max_jobs = 0
  52         self.requester_email = ""
  53         self.flags = string.split(attr(e, "flags", ""))
  54         for c in e.childNodes:
  55             if is_blank(c): continue
  56
  57             if c.nodeType != Element.ELEMENT_NODE:
  58                 log.panic("xml: evil group child %d" % c.nodeType)
  59             if c.nodeName == "batch":
  60                 self.batches.append(Batch(c))
  61             elif c.nodeName == "requester":
  62                 self.requester = text(c)
  63                 self.requester_email = attr(c, "email", "")
  64             elif c.nodeName == "priority":
  65                 self.priority = int(text(c))
  66             elif c.nodeName == "time":
  67                 self.time = int(text(c))
  68             elif c.nodeName == "maxjobs":
  69                 self.max_jobs = int(text(c))
  70             else:
  71                 log.panic("xml: evil group child (%s)" % c.nodeName)
  72         # note that we also check that group is sorted WRT deps
  73         m = {}
  74         for b in self.batches:
  75             deps = []
  76             m[b.b_id] = b
  77             for dep in b.depends_on:
  78                 if m.has_key(dep):
  79                     # avoid self-deps
  80                     if id(m[dep]) != id(b):
  81                         deps.append(m[dep])
  82                 else:
  83                     log.panic("xml: dependency not found in group")
  84             b.depends_on = deps
  85         if self.requester_email == "" and self.requester != "":
  86             self.requester_email = acl.user(self.requester).mail_to()
  87
  88     def dump(self, f):
  89         f.write("group: %d (id=%s pri=%d)\n" % (self.no, self.id, self.priority))
  90         f.write("  from: %s\n" % self.requester)
  91         f.write("  flags: %s\n" % string.join(self.flags))
  92         f.write("  time: %s\n" % time.asctime(time.localtime(self.time)))
  93         for b in self.batches:
  94             b.dump(f)
  95         f.write("\n")
  96
  97     def dump_html(self, f):
  98         f.write(
  99             "<div id=\"%(no)d\" class=\"%(flags)s\">\n"
 100             "<a href=\"#%(no)d\">%(no)d</a>. <span id=\"tz\">%(time)s</span> from <b>%(requester)s</b> "
 101             "<small>%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s</small>\n"
 102         % {
 103             'no': self.no,
 104             'id': '<a href="srpms/%(id)s">%(id)s</a>' % {'id': self.id},
 105             'time': escape(time.strftime("%a %b %d %Y %H:%M:%S %z", time.localtime(self.time))),
 106             'requester': escape(self.requester),
 107             'priority': self.priority,
 108             'max_jobs': self.max_jobs,
 109             'flags': string.join(self.flags)
 110         })
 111         f.write("<ul>\n")
 112         for b in self.batches:
 113             b.dump_html(f, self.id)
 114         f.write("</ul>\n")
 115         f.write("</div>\n")
 116
 117     def write_to(self, f):
 118         f.write("""
 119        <group id="%s" no="%d" flags="%s">
 120          <requester email='%s'>%s</requester>
 121          <time>%d</time>
 122          <priority>%d</priority>
 123          <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, string.join(self.flags),
 124                     escape(self.requester_email), escape(self.requester),
 125                     self.time, self.priority, self.max_jobs))
 126         for b in self.batches:
 127             b.write_to(f)
 128         f.write("       </group>\n\n")
 129
 130     def is_done(self):
 131         ok = 1
 132         for b in self.batches:
 133             if not b.is_done():
 134                 ok = 0
 135         return ok
 136
 137 class Batch:
 138     def __init__(self, e):
 139         self.bconds_with = []
 140         self.bconds_without = []
 141         self.builders = []
 142         self.builders_status = {}
 143         self.builders_status_time = {}
 144         self.builders_status_buildtime = {}
 145         self.kernel = ""
 146         self.target = []
 147         self.branch = ""
 148         self.src_rpm = ""
 149         self.info = ""
 150         self.spec = ""
 151         self.command = ""
 152         self.command_flags = []
 153         self.skip = []
 154         self.gb_id = ""
 155         self.b_id = attr(e, "id")
 156         self.depends_on = string.split(attr(e, "depends-on"))
 157         self.upgraded = True
 158         for c in e.childNodes:
 159             if is_blank(c): continue
 160
 161             if c.nodeType != Element.ELEMENT_NODE:
 162                 log.panic("xml: evil batch child %d" % c.nodeType)
 163             if c.nodeName == "src-rpm":
 164                 self.src_rpm = text(c)
 165             elif c.nodeName == "spec":
 166                 # normalize specname, specname is used as buildlog and we don't
 167                 # want to be exposed to directory traversal attacks
 168                 self.spec = text(c).split('/')[-1]
 169             elif c.nodeName == "command":
 170                 self.spec = "COMMAND"
 171                 self.command = text(c).strip()
 172                 self.command_flags = string.split(attr(c, "flags", ""))
 173             elif c.nodeName == "info":
 174                 self.info = text(c)
 175             elif c.nodeName == "kernel":
 176                 self.kernel = text(c)
 177             elif c.nodeName == "target":
 178                 self.target.append(text(c))
 179             elif c.nodeName == "skip":
 180                 self.skip.append(text(c))
 181             elif c.nodeName == "branch":
 182                 self.branch = text(c)
 183             elif c.nodeName == "builder":
 184                 key = text(c)
 185                 self.builders.append(key)
 186                 self.builders_status[key] = attr(c, "status", "?")
 187                 self.builders_status_time[key] = attr(c, "time", "0")
 188                 self.builders_status_buildtime[key] = "0" #attr(c, "buildtime", "0")
 189             elif c.nodeName == "with":
 190                 self.bconds_with.append(text(c))
 191             elif c.nodeName == "without":
 192                 self.bconds_without.append(text(c))
 193             else:
 194                 log.panic("xml: evil batch child (%s)" % c.nodeName)
 195
 196     def is_done(self):
 197         ok = 1
 198         for b in self.builders:
 199             s = self.builders_status[b]
 200             if not s.startswith("OK") and not s.startswith("SKIP") and not s.startswith("UNSUPP") and not s.startswith("FAIL"):
 201                 ok = 0
 202         return ok
 203
 204     def dump(self, f):
 205         f.write("  batch: %s/%s\n" % (self.src_rpm, self.spec))
 206         f.write("    info: %s\n" % self.info)
 207         f.write("    kernel: %s\n" % self.kernel)
 208         f.write("    target: %s\n" % self.target_string())
 209         f.write("    branch: %s\n" % self.branch)
 210         f.write("    bconds: %s\n" % self.bconds_string())
 211         builders = []
 212         for b in self.builders:
 213             builders.append("%s:%s" % (b, self.builders_status[b]))
 214         f.write("    builders: %s\n" % string.join(builders))
 215
 216     def is_command(self):
 217         return self.command != ""
 218
 219     def dump_html(self, f, rid):
 220         f.write("<li>\n")
 221         if self.is_command():
 222             desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
 223         else:
 224             cmd = "/usr/bin/git ls-remote --heads git://git.tld-linux.org/packages/%s" % (self.spec[:-5])
 225             r = call(cmd, shell=True)
 226             if r == 0:
 227                 dist = "tld"
 228             else:
 229                 dist = "pld"
 230             package_url = "http://git.%(dist)s-linux.org/?p=packages/%(package)s.git;a=blob;f=%(spec)s;hb=%(branch)s" % {
 231                 'dist': dist,
 232                 'spec': self.spec,
 233                 'branch': self.branch,
 234                 'package': self.spec[:-5],
 235             }
 236             desc = "%(src_rpm)s (<a href=\"%(package_url)s\">%(spec)s -r %(branch)s</a>%(bconds)s)" % {
 237                 'src_rpm': self.src_rpm,
 238                 'spec': self.spec,
 239                 'branch': self.branch,
 240                 'bconds': self.bconds_string() + self.kernel_string() + self.target_string(),
 241                 'package_url': package_url,
 242             }
 243         f.write("%s <small>[" % desc)
 244         builders = []
 245         for b in self.builders:
 246             s = self.builders_status[b]
 247             if s.startswith("OK"):
 248                 c = "green"
 249             elif s.startswith("FAIL"):
 250                 c = "red"
 251             elif s.startswith("SKIP"):
 252                 c = "blue"
 253             elif s.startswith("UNSUPP"):
 254                 c = "fuchsia"
 255             else:
 256                 c = "black"
 257             link_pre = ""
 258             link_post = ""
 259             if (s.startswith("OK") or s.startswith("SKIP") or s.startswith("UNSUPP") or s.startswith("FAIL")) and len(self.spec) > 5:
 260                 if self.is_command():
 261                     bl_name = "command"
 262                 else:
 263                     bl_name = self.spec[:len(self.spec)-5]
 264                 lin_ar = b.replace('noauto-','')
 265                 path = "/%s/%s/%s,%s.bz2" % (lin_ar.replace('-','/'), s, bl_name, rid)
 266                 is_ok = 0
 267                 if s.startswith("OK"):
 268                     is_ok = 1
 269                 bld = lin_ar.split('-')
 270                 tree_name = '-'.join(bld[:-1])
 271                 tree_arch = '-'.join(bld[-1:])
 272                 link_pre = "<a href=\"http://buildlogs.tld-linux.org/index.php?dist=%s&arch=%s&name=%s&id=%s&action=download\">" \
 273                         % (urllib.quote(tree_name), urllib.quote(tree_arch), urllib.quote(bl_name), urllib.quote(rid))
 274                 link_post = "</a>"
 275
 276             def ftime(s):
 277                 t = float(s)
 278                 if t > 0:
 279                     return time.asctime(time.localtime(t))
 280                 else:
 281                     return 'N/A'
 282
 283             tooltip = "last update: %(time)s\nbuild time: %(buildtime)s" % {
 284                 'time' : ftime(self.builders_status_time[b]),
 285                 'buildtime' : ftime(self.builders_status_buildtime[b]),
 286             }
 287             builders.append(link_pre +
 288                 "<font color='%(color)s'><b title=\"%(tooltip)s\">%(builder)s:%(status)s</b></font>" % {
 289                     'color' : c,
 290                     'builder' : b,
 291                     'status' : s,
 292                     'tooltip' : cgi.escape(tooltip, True),
 293             }
 294             + link_post)
 295         f.write("%s]</small></li>\n" % string.join(builders))
 296
 297     def rpmbuild_opts(self):
 298         """
 299             return all rpmbuild options related to this build
 300         """
 301         bconds = self.bconds_string() + self.kernel_string() + self.target_string()
 302         rpmdefs = \
 303             "--define '_topdir %(echo $HOME/rpm)' " \
 304             "--define '_specdir %{_topdir}/packages/%{name}' "  \
 305             "--define '_sourcedir %{_specdir}' " \
 306             "--define '_builddir %{_topdir}/BUILD/%{name}' "
 307         return rpmdefs + bconds
 308
 309     def kernel_string(self):
 310         r = ""
 311         if self.kernel != "":
 312             r = " --define 'alt_kernel " + self.kernel + "'"
 313         return r
 314
 315     def target_string(self):
 316         if len(self.target) > 0:
 317             return " --target " + ",".join(self.target)
 318         else:
 319             return ""
 320
 321     def bconds_string(self):
 322         r = ""
 323         for b in self.bconds_with:
 324             r = r + " --with " + b
 325         for b in self.bconds_without:
 326             r = r + " --without " + b
 327         return r
 328
 329     def default_target(self, arch):
 330         self.target.append("%s-tld-linux" % arch)
 331
 332     def write_to(self, f):
 333         f.write("""
 334          <batch id='%s' depends-on='%s'>
 335            <src-rpm>%s</src-rpm>
 336            <command flags="%s">%s</command>
 337            <spec>%s</spec>
 338            <branch>%s</branch>
 339            <info>%s</info>\n""" % (self.b_id,
 340                  string.join(map(lambda (b): b.b_id, self.depends_on)),
 341                  escape(self.src_rpm),
 342                  escape(' '.join(self.command_flags)), escape(self.command),
 343                  escape(self.spec), escape(self.branch), escape(self.info)))
 344         if self.kernel != "":
 345             f.write("           <kernel>%s</kernel>\n" % escape(self.kernel))
 346         for b in self.bconds_with:
 347             f.write("           <with>%s</with>\n" % escape(b))
 348         for b in self.target:
 349             f.write("           <target>%s</target>\n" % escape(b))
 350         for b in self.bconds_without:
 351             f.write("           <without>%s</without>\n" % escape(b))
 352         for b in self.builders:
 353             if self.builders_status_buildtime.has_key(b):
 354                 t = self.builders_status_buildtime[b]
 355             else:
 356                 t = "0"
 357             f.write("           <builder status='%s' time='%s' buildtime='%s'>%s</builder>\n" % \
 358                     (escape(self.builders_status[b]), self.builders_status_time[b], t, escape(b)))
 359         f.write("         </batch>\n")
 360
 361     def log_line(self, l):
 362         log.notice(l)
 363         if self.logfile != None:
 364             util.append_to(self.logfile, l)
 365
 366     def expand_builders(batch, all_builders):
 367         all = []
 368         for bld in batch.builders:
 369             res = []
 370             for my_bld in all_builders:
 371                 if fnmatch.fnmatch(my_bld, bld):
 372                     res.append(my_bld)
 373             if res != []:
 374                 all.extend(res)
 375             else:
 376                 all.append(bld)
 377         batch.builders = all
 378
 379 class Notification:
 380     def __init__(self, e):
 381         self.batches = []
 382         self.kind = 'notification'
 383         self.group_id = attr(e, "group-id")
 384         self.builder = attr(e, "builder")
 385         self.batches = {}
 386         self.batches_buildtime = {}
 387         for c in e.childNodes:
 388             if is_blank(c): continue
 389             if c.nodeType != Element.ELEMENT_NODE:
 390                 log.panic("xml: evil notification child %d" % c.nodeType)
 391             if c.nodeName == "batch":
 392                 id = attr(c, "id")
 393                 status = attr(c, "status")
 394                 buildtime = attr(c, "buildtime", "0")
 395                 if not status.startswith("OK") and not status.startswith("SKIP") and not status.startswith("UNSUPP") and not status.startswith("FAIL"):
 396                     log.panic("xml notification: bad status: %s" % status)
 397                 self.batches[id] = status
 398                 self.batches_buildtime[id] = buildtime
 399             else:
 400                 log.panic("xml: evil notification child (%s)" % c.nodeName)
 401
 402     def apply_to(self, q):
 403         for r in q.requests:
 404             if r.kind == "group":
 405                 for b in r.batches:
 406                     if self.batches.has_key(b.b_id):
 407                         b.builders_status[self.builder] = self.batches[b.b_id]
 408                         b.builders_status_time[self.builder] = time.time()
 409                         b.builders_status_buildtime[self.builder] = "0" #self.batches_buildtime[b.b_id]
 410
 411 def build_request(e):
 412     if e.nodeType != Element.ELEMENT_NODE:
 413         log.panic("xml: evil request element")
 414     if e.nodeName == "group":
 415         return Group(e)
 416     elif e.nodeName == "notification":
 417         return Notification(e)
 418     elif e.nodeName == "command":
 419         # FIXME
 420         return Command(e)
 421     else:
 422         log.panic("xml: evil request [%s]" % e.nodeName)
 423
 424 def parse_request(f):
 425     d = parseString(f)
 426     return build_request(d.documentElement)
 427
 428 def parse_requests(f):
 429     d = parseString(f)
 430     res = []
 431     for r in d.documentElement.childNodes:
 432         if is_blank(r): continue
 433         res.append(build_request(r))
 434     return res