TLD_Builder/request.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from xml.dom.minidom import *
   4 from datetime import datetime
   5 import string
   6 import time
   7 import xml.sax.saxutils
   8 import fnmatch
   9 import os
  10 import urllib
  11 import cgi
  12 import pytz
  13 import tempfile
  14
  15 import util
  16 import log
  17 from acl import acl
  18 from config import config
  19
  20 __all__ = ['parse_request', 'parse_requests']
  21
  22 def text(e):
  23     res = ""
  24     for n in e.childNodes:
  25         if n.nodeType != Element.TEXT_NODE:
  26             log.panic("xml: text expected in <%s>, got %d" % (e.nodeName, n.nodeType))
  27         res += n.nodeValue
  28     return res
  29
  30 def attr(e, a, default = None):
  31     try:
  32         return e.attributes[a].value
  33     except:
  34         if default != None:
  35             return default
  36         raise
  37
  38 def escape(s):
  39     return xml.sax.saxutils.escape(s)
  40
  41 # return timestamp with timezone information
  42 # so we could parse it in javascript
  43 def tzdate(t):
  44     # as strftime %z is unofficial, and does not work, need to make it numeric ourselves
  45     date = time.strftime("%a %b %d %Y %H:%M:%S", time.localtime(t))
  46     # NOTE: the altzone is showing CURRENT timezone, not what the "t" reflects
  47     # NOTE: when DST is off timezone gets it right, altzone not
  48     if time.daylight:
  49         tzoffset = time.altzone
  50     else:
  51         tzoffset = time.timezone
  52     tz = '%+05d' % (-tzoffset / 3600 * 100)
  53     return date + ' ' + tz
  54
  55 # return date in iso8601 format
  56 def iso8601(ts, timezone='UTC'):
  57     tz = pytz.timezone(timezone)
  58     dt = datetime.fromtimestamp(ts, tz)
  59     return dt.isoformat()
  60
  61 def is_blank(e):
  62     return e.nodeType == Element.TEXT_NODE and e.nodeValue.strip() == ""
  63
  64 class Group:
  65     def __init__(self, e):
  66         self.batches = []
  67         self.kind = 'group'
  68         self.id = attr(e, "id")
  69         self.no = int(attr(e, "no"))
  70         self.priority = 2
  71         self.time = time.time()
  72         self.requester = ""
  73         self.max_jobs = 0
  74         self.requester_email = ""
  75         self.flags = attr(e, "flags", "").split()
  76         for c in e.childNodes:
  77             if is_blank(c): continue
  78
  79             if c.nodeType != Element.ELEMENT_NODE:
  80                 log.panic("xml: evil group child %d" % c.nodeType)
  81             if c.nodeName == "batch":
  82                 self.batches.append(Batch(c))
  83             elif c.nodeName == "requester":
  84                 self.requester = text(c)
  85                 self.requester_email = attr(c, "email", "")
  86             elif c.nodeName == "priority":
  87                 self.priority = int(text(c))
  88             elif c.nodeName == "time":
  89                 self.time = int(text(c))
  90             elif c.nodeName == "maxjobs":
  91                 self.max_jobs = int(text(c))
  92             else:
  93                 log.panic("xml: evil group child (%s)" % c.nodeName)
  94         # note that we also check that group is sorted WRT deps
  95         m = {}
  96         for b in self.batches:
  97             deps = []
  98             m[b.b_id] = b
  99             for dep in b.depends_on:
 100                 if dep in m:
 101                     # avoid self-deps
 102                     if id(m[dep]) != id(b):
 103                         deps.append(m[dep])
 104                 else:
 105                     log.panic("xml: dependency not found in group")
 106             b.depends_on = deps
 107         if self.requester_email == "" and self.requester != "":
 108             self.requester_email = acl.user(self.requester).mail_to()
 109
 110     def dump(self, f):
 111         f.write("group: %d (id=%s pri=%d)\n" % (self.no, self.id, self.priority))
 112         f.write("  from: %s\n" % self.requester)
 113         f.write("  flags: %s\n" % string.join(self.flags))
 114         f.write("  time: %s\n" % time.asctime(time.localtime(self.time)))
 115         for b in self.batches:
 116             b.dump(f)
 117         f.write("\n")
 118
 119     # return structure usable for json encoding
 120     def dump_json(self):
 121         batches = []
 122         for b in self.batches:
 123             batches.append(b.dump_json())
 124
 125         return dict(
 126             no=self.no,
 127             id=self.id,
 128             time=self.time,
 129             requester=self.requester,
 130             priority=self.priority,
 131             max_jobs=self.max_jobs,
 132             flags=self.flags,
 133             batches=batches,
 134         )
 135
 136     def dump_html(self, f):
 137         f.write(
 138             "<div id=\"%(no)d\" class=\"request %(flags)s\">\n"
 139             "<a href=\"#%(no)d\">%(no)d</a>. "
 140             "<time class=\"timeago\" title=\"%(datetime)s\" datetime=\"%(datetime)s\">%(time)s</time> "
 141             "from <b class=requester>%(requester)s</b> "
 142             "<small>%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s</small>\n"
 143         % {
 144             'no': self.no,
 145             'id': '<a href="srpms/%(id)s">%(id)s</a>' % {'id': self.id},
 146             'time': escape(tzdate(self.time)),
 147             'datetime': escape(iso8601(self.time)),
 148             'requester': escape(self.requester),
 149             'priority': self.priority,
 150             'max_jobs': self.max_jobs,
 151             'flags': string.join(self.flags)
 152         })
 153         f.write("<ol>\n")
 154         for b in self.batches:
 155             b.dump_html(f, self.id)
 156         f.write("</ol>\n")
 157         f.write("</div>\n")
 158
 159     def write_to(self, f):
 160         f.write("""
 161        <group id="%s" no="%d" flags="%s">
 162          <requester email='%s'>%s</requester>
 163          <time>%d</time>
 164          <priority>%d</priority>
 165          <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, string.join(self.flags),
 166                     escape(self.requester_email), escape(self.requester),
 167                     self.time, self.priority, self.max_jobs))
 168         for b in self.batches:
 169             b.write_to(f)
 170         f.write("       </group>\n\n")
 171
 172     def is_done(self):
 173         ok = 1
 174         for b in self.batches:
 175             if not b.is_done():
 176                 ok = 0
 177         return ok
 178
 179 # transform php package name (52) to version (5.2)
 180 def php_name_to_ver(v):
 181     return '.'.join(list(v))
 182
 183 # transform php version (5.2) to package name (52)
 184 def php_ver_to_name(v):
 185     return v.replace('.', '')
 186
 187 class Batch:
 188     DEFAULT_PHP = '5.3'
 189
 190     def __init__(self, e):
 191         self.bconds_with = []
 192         self.bconds_without = []
 193         self.builders = []
 194         self.builders_status = {}
 195         self.builders_status_time = {}
 196         self.builders_status_buildtime = {}
 197         self.kernel = ""
 198         self.defines = {}
 199         self.target = []
 200         self.branch = ""
 201         self.src_rpm = ""
 202         self.info = ""
 203         self.spec = ""
 204         self.command = ""
 205         self.command_flags = []
 206         self.skip = []
 207         self.gb_id = ""
 208         self.b_id = attr(e, "id")
 209         self.depends_on = attr(e, "depends-on").split()
 210         self.upgraded = True
 211
 212         self.parse_xml(e)
 213
 214         self.__topdir = None
 215
 216     def get_topdir(self):
 217         if not self.__topdir:
 218             self.__topdir = tempfile.mkdtemp(prefix='B.', dir='/tmp')
 219         return self.__topdir
 220
 221     def parse_xml(self, e):
 222         for c in e.childNodes:
 223             if is_blank(c): continue
 224
 225             if c.nodeType != Element.ELEMENT_NODE:
 226                 log.panic("xml: evil batch child %d" % c.nodeType)
 227             if c.nodeName == "src-rpm":
 228                 self.src_rpm = text(c)
 229             elif c.nodeName == "spec":
 230                 # normalize specname, specname is used as buildlog and we don't
 231                 # want to be exposed to directory traversal attacks
 232                 self.spec = text(c).split('/')[-1]
 233             elif c.nodeName == "command":
 234                 self.spec = "COMMAND"
 235                 self.command = text(c).strip()
 236                 self.command_flags = attr(c, "flags", "").split()
 237             elif c.nodeName == "info":
 238                 self.info = text(c)
 239             elif c.nodeName == "kernel":
 240                 self.kernel = text(c)
 241             elif c.nodeName == "define":
 242                 define = attr(c, "name")
 243                 self.defines[define] = text(c)
 244             elif c.nodeName == "target":
 245                 self.target.append(text(c))
 246             elif c.nodeName == "skip":
 247                 self.skip.append(text(c))
 248             elif c.nodeName == "branch":
 249                 self.branch = text(c)
 250             elif c.nodeName == "builder":
 251                 key = text(c)
 252                 self.builders.append(key)
 253                 self.builders_status[key] = attr(c, "status", "?")
 254                 self.builders_status_time[key] = attr(c, "time", "0")
 255                 self.builders_status_buildtime[key] = "0" #attr(c, "buildtime", "0")
 256             elif c.nodeName == "with":
 257                 self.bconds_with.append(text(c))
 258             elif c.nodeName == "without":
 259                 self.bconds_without.append(text(c))
 260             else:
 261                 log.panic("xml: evil batch child (%s)" % c.nodeName)
 262
 263     def get_package_name(self):
 264         if len(self.spec) <= 5:
 265             return None
 266         return self.spec[:-5]
 267
 268     def tmpdir(self):
 269         """
 270         return tmpdir for this batch job building
 271         """
 272         # it's better to have TMPDIR and BUILD dir on same partition:
 273         # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2
 274         # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory
 275         path = os.path.join(self.get_topdir(), 'BUILD', 'tmp')
 276         return path
 277
 278     def is_done(self):
 279         ok = 1
 280         for b in self.builders:
 281             s = self.builders_status[b]
 282             if not s.startswith("OK") and not s.startswith("SKIP") and not s.startswith("UNSUPP") and not s.startswith("FAIL"):
 283                 ok = 0
 284         return ok
 285
 286     def dump(self, f):
 287         f.write("  batch: %s/%s\n" % (self.src_rpm, self.spec))
 288         f.write("    info: %s\n" % self.info)
 289         f.write("    kernel: %s\n" % self.kernel)
 290         f.write("    defines: %s\n" % self.defines_string())
 291         f.write("    target: %s\n" % self.target_string())
 292         f.write("    branch: %s\n" % self.branch)
 293         f.write("    bconds: %s\n" % self.bconds_string())
 294         builders = []
 295         for b in self.builders:
 296             builders.append("%s:%s" % (b, self.builders_status[b]))
 297         f.write("    builders: %s\n" % string.join(builders))
 298
 299     def is_command(self):
 300         return self.command != ""
 301
 302     # return structure usable for json encoding
 303     def dump_json(self):
 304         return dict(
 305             command=self.command,
 306             command_flags=self.command_flags,
 307
 308             spec=self.spec,
 309             branch=self.branch,
 310             package=self.spec[:-5],
 311             src_rpm=self.src_rpm,
 312
 313             bconds_with=self.bconds_with,
 314             bconds_without=self.bconds_without,
 315
 316             kernel=self.kernel,
 317             target=self.target,
 318             defines=self.defines,
 319
 320             builders=self.builders,
 321             builders_status=self.builders_status,
 322             builders_status_time=self.builders_status_time,
 323             builders_status_buildtime=self.builders_status_buildtime,
 324         )
 325
 326     def dump_html(self, f, rid):
 327         f.write("<li>\n")
 328         if self.is_command():
 329             desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
 330         else:
 331             cmd = "/usr/bin/git ls-remote --heads git://git.tld-linux.org/packages/%s 1>/dev/null 2>&1" % (self.spec[:-5])
 332             r = call(cmd, shell=True)
 333             if r == 0:
 334                 package_url = "http://git.tld-linux.org/?p=packages/%(package)s.git;a=blob;f=%(spec)s;hb=%(branch)s" % {
 335                     'spec': urllib.quote(self.spec),
 336                     'branch': urllib.quote(self.branch),
 337                     'package': urllib.quote(self.spec[:-5]),
 338                 }
 339             else:
 340                 package_url = "http://git.pld-linux.org/gitweb.cgi?p=packages/%(package)s.git;f=%(spec)s;h=%(branch)s;a=shortlog" % {
 341                     'spec': urllib.quote(self.spec),
 342                     'branch': urllib.quote(self.branch),
 343                     'package': urllib.quote(self.spec[:-5]),
 344                 }
 345             desc = "%(src_rpm)s (<a href=\"%(package_url)s\">%(spec)s -r %(branch)s</a>%(rpmopts)s)" % {
 346                 'src_rpm': self.src_rpm,
 347                 'spec': self.spec,
 348                 'branch': self.branch,
 349                 'rpmopts': self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string(),
 350                 'package_url': package_url,
 351             }
 352         f.write("%s <small>[" % desc)
 353         builders = []
 354         for b in self.builders:
 355             s = self.builders_status[b]
 356             if s.startswith("OK"):
 357                 c = "green"
 358             elif s.startswith("FAIL"):
 359                 c = "red"
 360             elif s.startswith("SKIP"):
 361                 c = "blue"
 362             elif s.startswith("UNSUPP"):
 363                 c = "fuchsia"
 364             else:
 365                 c = "black"
 366             link_pre = ""
 367             link_post = ""
 368             if (s.startswith("OK") or s.startswith("SKIP") or s.startswith("UNSUPP") or s.startswith("FAIL")) and len(self.spec) > 5:
 369                 if self.is_command():
 370                     bl_name = "command"
 371                 else:
 372                     bl_name = self.spec[:len(self.spec)-5]
 373                 lin_ar = b.replace('noauto-','')
 374                 path = "/%s/%s/%s,%s.bz2" % (lin_ar.replace('-','/'), s, bl_name, rid)
 375                 is_ok = 0
 376                 if s.startswith("OK"):
 377                     is_ok = 1
 378                 bld = lin_ar.split('-')
 379                 tree_name = '-'.join(bld[:-1])
 380                 tree_arch = '-'.join(bld[-1:])
 381                 link_pre = "<a href=\"http://buildlogs.tld-linux.org/index.php?dist=%s&arch=%s&name=%s&id=%s&action=download\">" \
 382                     % (urllib.quote(tree_name), urllib.quote(tree_arch), urllib.quote(bl_name), urllib.quote(rid))
 383                 link_post = "</a>"
 384
 385             def ftime(s):
 386                 t = float(s)
 387                 if t > 0:
 388                     return time.asctime(time.localtime(t))
 389                 else:
 390                     return 'N/A'
 391
 392             tooltip = "last update: %(time)s\nbuild time: %(buildtime)s" % {
 393                 'time' : ftime(self.builders_status_time[b]),
 394                 'buildtime' : ftime(self.builders_status_buildtime[b]),
 395             }
 396             builders.append(link_pre +
 397                 "<font color='%(color)s'><b title=\"%(tooltip)s\">%(builder)s:%(status)s</b></font>" % {
 398                     'color' : c,
 399                     'builder' : b,
 400                     'status' : s,
 401                     'tooltip' : cgi.escape(tooltip, True),
 402             }
 403             + link_post)
 404         f.write("%s]</small></li>\n" % string.join(builders))
 405
 406     def rpmbuild_opts(self):
 407         """
 408             return all rpmbuild options related to this build
 409         """
 410         rpmopts = self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string()
 411         rpmdefs = \
 412             "--define '_topdir %s' " % self.get_topdir() + \
 413             "--define '_specdir %{_topdir}' "  \
 414             "--define '_sourcedir %{_specdir}' " \
 415             "--define '_rpmdir %{_topdir}/RPMS' " \
 416             "--define '_builddir %{_topdir}/BUILD' "
 417         return rpmdefs + rpmopts
 418
 419     def php_ignores(self, php_version):
 420         # available php versions in distro
 421         php_versions = ['7.2', '7.3', '7.4', '8.0']
 422
 423         # remove current php version
 424         try:
 425             php_versions.remove(php_version)
 426         except ValueError:
 427             log.notice("Attempt to remove inexistent key '%s' from %s" % (php_version, php_versions))
 428             pass
 429
 430         # map them to poldek ignores
 431         # always ignore hhvm
 432         res = ['hhvm-*']
 433         for v in list(map(php_ver_to_name, php_versions)):
 434             res.append("php%s-*" % v)
 435
 436         return res
 437
 438     # build ignore package list
 439     # currently only php ignore is filled based on build context
 440     def ignores(self):
 441         ignores = []
 442
 443         # add php version based ignores
 444         if 'php_suffix' in self.defines:
 445             # current version if -D php_suffix is present
 446             php_version = php_name_to_ver(self.defines['php_suffix'])
 447         else:
 448             php_version = self.DEFAULT_PHP
 449
 450         ignores.extend(self.php_ignores(php_version))
 451
 452         # return empty string if the list is empty
 453         if len(ignores) == 0:
 454             return ""
 455
 456         def add_ignore(s):
 457             return "--ignore=%s" % s
 458
 459         return " ".join(list(map(add_ignore, ignores)))
 460
 461     def kernel_string(self):
 462         r = ""
 463         if self.kernel != "":
 464             r = " --define 'alt_kernel " + self.kernel + "'"
 465         return r
 466
 467     def target_string(self):
 468         if len(self.target) > 0:
 469             return " --target " + ",".join(self.target)
 470         else:
 471             return ""
 472
 473     def bconds_string(self):
 474         r = ""
 475         for b in self.bconds_with:
 476             r = r + " --with " + b
 477         for b in self.bconds_without:
 478             r = r + " --without " + b
 479         return r
 480
 481     def defines_string(self):
 482         r = ""
 483         for key,value in self.defines.items():
 484             r += " --define '%s %s'" % (key, value)
 485         return r
 486
 487     def defines_xml(self):
 488         r = ""
 489         for key,value in self.defines.items():
 490             r += "<define name='%s'>%s</define>\n" % (escape(key), escape(value))
 491         return r
 492
 493     def default_target(self, arch):
 494         self.target.append("%s-tld-linux" % arch)
 495
 496     def write_to(self, f):
 497         f.write("""
 498          <batch id='%s' depends-on='%s'>
 499            <src-rpm>%s</src-rpm>
 500            <command flags="%s">%s</command>
 501            <spec>%s</spec>
 502            <branch>%s</branch>
 503            <info>%s</info>\n""" % (self.b_id,
 504                  string.join(list(map(lambda b: b.b_id, self.depends_on))),
 505                  escape(self.src_rpm),
 506                  escape(' '.join(self.command_flags)), escape(self.command),
 507                  escape(self.spec), escape(self.branch), escape(self.info)))
 508         if self.kernel != "":
 509             f.write("           <kernel>%s</kernel>\n" % escape(self.kernel))
 510         for b in self.bconds_with:
 511             f.write("           <with>%s</with>\n" % escape(b))
 512         for b in self.target:
 513             f.write("           <target>%s</target>\n" % escape(b))
 514         for b in self.bconds_without:
 515             f.write("           <without>%s</without>\n" % escape(b))
 516         if self.defines:
 517             f.write("           %s\n" % self.defines_xml())
 518         for b in self.builders:
 519             if b in self.builders_status_buildtime:
 520                 t = self.builders_status_buildtime[b]
 521             else:
 522                 t = "0"
 523             f.write("           <builder status='%s' time='%s' buildtime='%s'>%s</builder>\n" % \
 524                     (escape(self.builders_status[b]), self.builders_status_time[b], t, escape(b)))
 525         f.write("         </batch>\n")
 526
 527     def log_line(self, l):
 528         log.notice(l)
 529         if self.logfile != None:
 530             util.append_to(self.logfile, l)
 531
 532     def expand_builders(batch, all_builders):
 533         all = []
 534         for bld in batch.builders:
 535             res = []
 536             for my_bld in all_builders:
 537                 if fnmatch.fnmatch(my_bld, bld):
 538                     res.append(my_bld)
 539             if res != []:
 540                 all.extend(res)
 541             else:
 542                 all.append(bld)
 543         batch.builders = all
 544
 545 class Notification:
 546     def __init__(self, e):
 547         self.batches = []
 548         self.kind = 'notification'
 549         self.group_id = attr(e, "group-id")
 550         self.builder = attr(e, "builder")
 551         self.batches = {}
 552         self.batches_buildtime = {}
 553         for c in e.childNodes:
 554             if is_blank(c): continue
 555             if c.nodeType != Element.ELEMENT_NODE:
 556                 log.panic("xml: evil notification child %d" % c.nodeType)
 557             if c.nodeName == "batch":
 558                 id = attr(c, "id")
 559                 status = attr(c, "status")
 560                 buildtime = attr(c, "buildtime", "0")
 561                 if not status.startswith("OK") and not status.startswith("SKIP") and not status.startswith("UNSUPP") and not status.startswith("FAIL"):
 562                     log.panic("xml notification: bad status: %s" % status)
 563                 self.batches[id] = status
 564                 self.batches_buildtime[id] = buildtime
 565             else:
 566                 log.panic("xml: evil notification child (%s)" % c.nodeName)
 567
 568     # return structure usable for json encoding
 569     def dump_json(self):
 570         return dict(
 571             id=self.group_id,
 572             builder=self.builder,
 573             batches=self.batches,
 574             batches_buildtime=self.batches_buildtime,
 575         )
 576
 577     def apply_to(self, q):
 578         for r in q.requests:
 579             if r.kind == "group":
 580                 for b in r.batches:
 581                     if b.b_id in self.batches:
 582                         b.builders_status[self.builder] = self.batches[b.b_id]
 583                         b.builders_status_time[self.builder] = time.time()
 584                         b.builders_status_buildtime[self.builder] = "0" #self.batches_buildtime[b.b_id]
 585
 586 def build_request(e):
 587     if e.nodeType != Element.ELEMENT_NODE:
 588         log.panic("xml: evil request element")
 589     if e.nodeName == "group":
 590         return Group(e)
 591     elif e.nodeName == "notification":
 592         return Notification(e)
 593     elif e.nodeName == "command":
 594         # FIXME
 595         return Command(e)
 596     else:
 597         log.panic("xml: evil request [%s]" % e.nodeName)
 598
 599 def parse_request(f):
 600     d = parseString(f)
 601     return build_request(d.documentElement)
 602
 603 def parse_requests(f):
 604     d = parseString(f)
 605     res = []
 606     for r in d.documentElement.childNodes:
 607         if is_blank(r): continue
 608         res.append(build_request(r))
 609     return res