TLD_Builder/request.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from xml.dom.minidom import *
   4 from datetime import datetime
   5 import string
   6 import time
   7 import xml.sax.saxutils
   8 import fnmatch
   9 import os
  10 import urllib.parse
  11 import html
  12 import pytz
  13 import tempfile
  14 import subprocess
  15
  16 import util
  17 import log
  18 from acl import acl
  19 from config import config
  20
  21 __all__ = ['parse_request', 'parse_requests']
  22
  23 def text(e):
  24     res = ""
  25     for n in e.childNodes:
  26         if n.nodeType != Element.TEXT_NODE:
  27             log.panic("xml: text expected in <%s>, got %d" % (e.nodeName, n.nodeType))
  28         res += n.nodeValue
  29     return res
  30
  31 def attr(e, a, default = None):
  32     try:
  33         return e.attributes[a].value
  34     except:
  35         if default != None:
  36             return default
  37         raise
  38
  39 def escape(s):
  40     return xml.sax.saxutils.escape(s)
  41
  42 # return timestamp with timezone information
  43 # so we could parse it in javascript
  44 def tzdate(t):
  45     # as strftime %z is unofficial, and does not work, need to make it numeric ourselves
  46     date = time.strftime("%a %b %d %Y %H:%M:%S", time.localtime(t))
  47     # NOTE: the altzone is showing CURRENT timezone, not what the "t" reflects
  48     # NOTE: when DST is off timezone gets it right, altzone not
  49     if time.daylight:
  50         tzoffset = time.altzone
  51     else:
  52         tzoffset = time.timezone
  53     tz = '%+05d' % (-tzoffset / 3600 * 100)
  54     return date + ' ' + tz
  55
  56 # return date in iso8601 format
  57 def iso8601(ts, timezone='UTC'):
  58     tz = pytz.timezone(timezone)
  59     dt = datetime.fromtimestamp(ts, tz)
  60     return dt.isoformat()
  61
  62 def is_blank(e):
  63     return e.nodeType == Element.TEXT_NODE and e.nodeValue.strip() == ""
  64
  65 class Group:
  66     def __init__(self, e):
  67         self.batches = []
  68         self.kind = 'group'
  69         self.id = attr(e, "id")
  70         self.no = int(attr(e, "no"))
  71         self.priority = 2
  72         self.time = time.time()
  73         self.requester = ""
  74         self.max_jobs = 0
  75         self.requester_email = ""
  76         self.flags = attr(e, "flags", "").split()
  77         for c in e.childNodes:
  78             if is_blank(c): continue
  79
  80             if c.nodeType != Element.ELEMENT_NODE:
  81                 log.panic("xml: evil group child %d" % c.nodeType)
  82             if c.nodeName == "batch":
  83                 self.batches.append(Batch(c))
  84             elif c.nodeName == "requester":
  85                 self.requester = text(c)
  86                 self.requester_email = attr(c, "email", "")
  87             elif c.nodeName == "priority":
  88                 self.priority = int(text(c))
  89             elif c.nodeName == "time":
  90                 self.time = int(text(c))
  91             elif c.nodeName == "maxjobs":
  92                 self.max_jobs = int(text(c))
  93             else:
  94                 log.panic("xml: evil group child (%s)" % c.nodeName)
  95         # note that we also check that group is sorted WRT deps
  96         m = {}
  97         for b in self.batches:
  98             deps = []
  99             m[b.b_id] = b
 100             for dep in b.depends_on:
 101                 if dep in m:
 102                     # avoid self-deps
 103                     if id(m[dep]) != id(b):
 104                         deps.append(m[dep])
 105                 else:
 106                     log.panic("xml: dependency not found in group")
 107             b.depends_on = deps
 108         if self.requester_email == "" and self.requester != "":
 109             self.requester_email = acl.user(self.requester).mail_to()
 110
 111     def dump(self, f):
 112         f.write("group: %d (id=%s pri=%d)\n" % (self.no, self.id, self.priority))
 113         f.write("  from: %s\n" % self.requester)
 114         f.write("  flags: %s\n" % ' '.join(self.flags))
 115         f.write("  time: %s\n" % time.asctime(time.localtime(self.time)))
 116         for b in self.batches:
 117             b.dump(f)
 118         f.write("\n")
 119
 120     # return structure usable for json encoding
 121     def dump_json(self):
 122         batches = []
 123         for b in self.batches:
 124             batches.append(b.dump_json())
 125
 126         return dict(
 127             no=self.no,
 128             id=self.id,
 129             time=self.time,
 130             requester=self.requester,
 131             priority=self.priority,
 132             max_jobs=self.max_jobs,
 133             flags=self.flags,
 134             batches=batches,
 135         )
 136
 137     def dump_html(self, f):
 138         f.write(
 139             "<div id=\"%(no)d\" class=\"request %(flags)s\">\n"
 140             "<a href=\"#%(no)d\">%(no)d</a>. "
 141             "<time class=\"timeago\" title=\"%(datetime)s\" datetime=\"%(datetime)s\">%(time)s</time> "
 142             "from <b class=requester>%(requester)s</b> "
 143             "<small>%(id)s, prio=%(priority)d, jobs=%(max_jobs)d, %(flags)s</small>\n"
 144         % {
 145             'no': self.no,
 146             'id': '<a href="srpms/%(id)s">%(id)s</a>' % {'id': self.id},
 147             'time': escape(tzdate(self.time)),
 148             'datetime': escape(iso8601(self.time)),
 149             'requester': escape(self.requester),
 150             'priority': self.priority,
 151             'max_jobs': self.max_jobs,
 152             'flags': ' '.join(self.flags)
 153         })
 154         f.write("<ol>\n")
 155         for b in self.batches:
 156             b.dump_html(f, self.id)
 157         f.write("</ol>\n")
 158         f.write("</div>\n")
 159
 160     def write_to(self, f):
 161         f.write("""
 162        <group id="%s" no="%d" flags="%s">
 163          <requester email='%s'>%s</requester>
 164          <time>%d</time>
 165          <priority>%d</priority>
 166          <maxjobs>%d</maxjobs>\n""" % (self.id, self.no, ' '.join(self.flags),
 167                     escape(self.requester_email), escape(self.requester),
 168                     self.time, self.priority, self.max_jobs))
 169         for b in self.batches:
 170             b.write_to(f)
 171         f.write("       </group>\n\n")
 172
 173     def is_done(self):
 174         ok = 1
 175         for b in self.batches:
 176             if not b.is_done():
 177                 ok = 0
 178         return ok
 179
 180 # transform php package name (52) to version (5.2)
 181 def php_name_to_ver(v):
 182     return '.'.join(list(v))
 183
 184 # transform php version (5.2) to package name (52)
 185 def php_ver_to_name(v):
 186     return v.replace('.', '')
 187
 188 class Batch:
 189     DEFAULT_PHP = '5.3'
 190
 191     def __init__(self, e):
 192         self.bconds_with = []
 193         self.bconds_without = []
 194         self.builders = []
 195         self.builders_status = {}
 196         self.builders_status_time = {}
 197         self.builders_status_buildtime = {}
 198         self.kernel = ""
 199         self.defines = {}
 200         self.target = []
 201         self.branch = ""
 202         self.src_rpm = ""
 203         self.info = ""
 204         self.spec = ""
 205         self.command = ""
 206         self.command_flags = []
 207         self.skip = []
 208         self.gb_id = ""
 209         self.b_id = attr(e, "id")
 210         self.depends_on = attr(e, "depends-on").split()
 211         self.upgraded = True
 212
 213         self.parse_xml(e)
 214
 215         self.__topdir = None
 216
 217     def get_topdir(self):
 218         if not self.__topdir:
 219             self.__topdir = tempfile.mkdtemp(prefix='B.', dir='/tmp')
 220         return self.__topdir
 221
 222     def parse_xml(self, e):
 223         for c in e.childNodes:
 224             if is_blank(c): continue
 225
 226             if c.nodeType != Element.ELEMENT_NODE:
 227                 log.panic("xml: evil batch child %d" % c.nodeType)
 228             if c.nodeName == "src-rpm":
 229                 self.src_rpm = text(c)
 230             elif c.nodeName == "spec":
 231                 # normalize specname, specname is used as buildlog and we don't
 232                 # want to be exposed to directory traversal attacks
 233                 self.spec = text(c).split('/')[-1]
 234             elif c.nodeName == "command":
 235                 self.spec = "COMMAND"
 236                 self.command = text(c).strip()
 237                 self.command_flags = attr(c, "flags", "").split()
 238             elif c.nodeName == "info":
 239                 self.info = text(c)
 240             elif c.nodeName == "kernel":
 241                 self.kernel = text(c)
 242             elif c.nodeName == "define":
 243                 define = attr(c, "name")
 244                 self.defines[define] = text(c)
 245             elif c.nodeName == "target":
 246                 self.target.append(text(c))
 247             elif c.nodeName == "skip":
 248                 self.skip.append(text(c))
 249             elif c.nodeName == "branch":
 250                 self.branch = text(c)
 251             elif c.nodeName == "builder":
 252                 key = text(c)
 253                 self.builders.append(key)
 254                 self.builders_status[key] = attr(c, "status", "?")
 255                 self.builders_status_time[key] = attr(c, "time", "0")
 256                 self.builders_status_buildtime[key] = "0" #attr(c, "buildtime", "0")
 257             elif c.nodeName == "with":
 258                 self.bconds_with.append(text(c))
 259             elif c.nodeName == "without":
 260                 self.bconds_without.append(text(c))
 261             else:
 262                 log.panic("xml: evil batch child (%s)" % c.nodeName)
 263
 264     def get_package_name(self):
 265         if len(self.spec) <= 5:
 266             return None
 267         return self.spec[:-5]
 268
 269     def tmpdir(self):
 270         """
 271         return tmpdir for this batch job building
 272         """
 273         # it's better to have TMPDIR and BUILD dir on same partition:
 274         # + /usr/bin/bzip2 -dc /home/services/builder/rpm/packages/kernel/patch-2.6.27.61.bz2
 275         # patch: **** Can't rename file /tmp/B.a1b1d3/poKWwRlp to drivers/scsi/hosts.c : No such file or directory
 276         path = os.path.join(self.get_topdir(), 'BUILD', 'tmp')
 277         return path
 278
 279     def is_done(self):
 280         ok = 1
 281         for b in self.builders:
 282             s = self.builders_status[b]
 283             if not s.startswith("OK") and not s.startswith("SKIP") and not s.startswith("UNSUPP") and not s.startswith("FAIL"):
 284                 ok = 0
 285         return ok
 286
 287     def dump(self, f):
 288         f.write("  batch: %s/%s\n" % (self.src_rpm, self.spec))
 289         f.write("    info: %s\n" % self.info)
 290         f.write("    kernel: %s\n" % self.kernel)
 291         f.write("    defines: %s\n" % self.defines_string())
 292         f.write("    target: %s\n" % self.target_string())
 293         f.write("    branch: %s\n" % self.branch)
 294         f.write("    bconds: %s\n" % self.bconds_string())
 295         builders = []
 296         for b in self.builders:
 297             builders.append("%s:%s" % (b, self.builders_status[b]))
 298         f.write("    builders: %s\n" % ' '.join(builders))
 299
 300     def is_command(self):
 301         return self.command != ""
 302
 303     # return structure usable for json encoding
 304     def dump_json(self):
 305         return dict(
 306             command=self.command,
 307             command_flags=self.command_flags,
 308
 309             spec=self.spec,
 310             branch=self.branch,
 311             package=self.spec[:-5],
 312             src_rpm=self.src_rpm,
 313
 314             bconds_with=self.bconds_with,
 315             bconds_without=self.bconds_without,
 316
 317             kernel=self.kernel,
 318             target=self.target,
 319             defines=self.defines,
 320
 321             builders=self.builders,
 322             builders_status=self.builders_status,
 323             builders_status_time=self.builders_status_time,
 324             builders_status_buildtime=self.builders_status_buildtime,
 325         )
 326
 327     def dump_html(self, f, rid):
 328         f.write("<li>\n")
 329         if self.is_command():
 330             desc = "SH: <pre>%s</pre> flags: [%s]" % (self.command, ' '.join(self.command_flags))
 331         else:
 332             cmd = "/usr/bin/git ls-remote --heads git://git.tld-linux.org/packages/%s 1>/dev/null 2>&1" % (self.spec[:-5])
 333             r = subprocess.call(cmd, shell=True)
 334             if r == 0:
 335                 package_url = "http://git.tld-linux.org/?p=packages/%(package)s.git;a=blob;f=%(spec)s;hb=%(branch)s" % {
 336                     'spec': urllib.parse.quote(self.spec),
 337                     'branch': urllib.parse.quote(self.branch),
 338                     'package': urllib.parse.quote(self.spec[:-5]),
 339                 }
 340             else:
 341                 package_url = "http://git.pld-linux.org/gitweb.cgi?p=packages/%(package)s.git;f=%(spec)s;h=%(branch)s;a=shortlog" % {
 342                     'spec': urllib.parse.quote(self.spec),
 343                     'branch': urllib.parse.quote(self.branch),
 344                     'package': urllib.parse.quote(self.spec[:-5]),
 345                 }
 346             desc = "%(src_rpm)s (<a href=\"%(package_url)s\">%(spec)s -r %(branch)s</a>%(rpmopts)s)" % {
 347                 'src_rpm': self.src_rpm,
 348                 'spec': self.spec,
 349                 'branch': self.branch,
 350                 'rpmopts': self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string(),
 351                 'package_url': package_url,
 352             }
 353         f.write("%s <small>[" % desc)
 354         builders = []
 355         for b in self.builders:
 356             s = self.builders_status[b]
 357             if s.startswith("OK"):
 358                 c = "green"
 359             elif s.startswith("FAIL"):
 360                 c = "red"
 361             elif s.startswith("SKIP"):
 362                 c = "blue"
 363             elif s.startswith("UNSUPP"):
 364                 c = "fuchsia"
 365             else:
 366                 c = "black"
 367             link_pre = ""
 368             link_post = ""
 369             if (s.startswith("OK") or s.startswith("SKIP") or s.startswith("UNSUPP") or s.startswith("FAIL")) and len(self.spec) > 5:
 370                 if self.is_command():
 371                     bl_name = "command"
 372                 else:
 373                     bl_name = self.spec[:len(self.spec)-5]
 374                 lin_ar = b.replace('noauto-','')
 375                 path = "/%s/%s/%s,%s.bz2" % (lin_ar.replace('-','/'), s, bl_name, rid)
 376                 is_ok = 0
 377                 if s.startswith("OK"):
 378                     is_ok = 1
 379                 bld = lin_ar.split('-')
 380                 tree_name = '-'.join(bld[:-1])
 381                 tree_arch = '-'.join(bld[-1:])
 382                 link_pre = "<a href=\"http://buildlogs.tld-linux.org/index.php?dist=%s&arch=%s&name=%s&id=%s&action=download\">" \
 383                     % (urllib.parse.quote(tree_name), urllib.parse.quote(tree_arch), urllib.parse.quote(bl_name), urllib.parse.quote(rid))
 384                 link_post = "</a>"
 385
 386             def ftime(s):
 387                 t = float(s)
 388                 if t > 0:
 389                     return time.asctime(time.localtime(t))
 390                 else:
 391                     return 'N/A'
 392
 393             tooltip = "last update: %(time)s\nbuild time: %(buildtime)s" % {
 394                 'time' : ftime(self.builders_status_time[b]),
 395                 'buildtime' : ftime(self.builders_status_buildtime[b]),
 396             }
 397             builders.append(link_pre +
 398                 "<font color='%(color)s'><b title=\"%(tooltip)s\">%(builder)s:%(status)s</b></font>" % {
 399                     'color' : c,
 400                     'builder' : b,
 401                     'status' : s,
 402                     'tooltip' : html.escape(tooltip, True),
 403             }
 404             + link_post)
 405         f.write("%s]</small></li>\n" % ' '.join(builders))
 406
 407     def rpmbuild_opts(self):
 408         """
 409             return all rpmbuild options related to this build
 410         """
 411         rpmopts = self.bconds_string() + self.kernel_string() + self.target_string() + self.defines_string()
 412         rpmdefs = \
 413             "--define '_topdir %s' " % self.get_topdir() + \
 414             "--define '_specdir %{_topdir}' "  \
 415             "--define '_sourcedir %{_specdir}' " \
 416             "--define '_rpmdir %{_topdir}/RPMS' " \
 417             "--define '_builddir %{_topdir}/BUILD' "
 418         return rpmdefs + rpmopts
 419
 420     def php_ignores(self, php_version):
 421         # available php versions in distro
 422         php_versions = ['7.2', '7.3', '7.4', '8.0']
 423
 424         # remove current php version
 425         try:
 426             php_versions.remove(php_version)
 427         except ValueError:
 428             log.notice("Attempt to remove inexistent key '%s' from %s" % (php_version, php_versions))
 429             pass
 430
 431         # map them to poldek ignores
 432         # always ignore hhvm
 433         res = ['hhvm-*']
 434         for v in list(map(php_ver_to_name, php_versions)):
 435             res.append("php%s-*" % v)
 436
 437         return res
 438
 439     # build ignore package list
 440     # currently only php ignore is filled based on build context
 441     def ignores(self):
 442         ignores = []
 443
 444         # add php version based ignores
 445         if 'php_suffix' in self.defines:
 446             # current version if -D php_suffix is present
 447             php_version = php_name_to_ver(self.defines['php_suffix'])
 448         else:
 449             php_version = self.DEFAULT_PHP
 450
 451         ignores.extend(self.php_ignores(php_version))
 452
 453         # return empty string if the list is empty
 454         if len(ignores) == 0:
 455             return ""
 456
 457         def add_ignore(s):
 458             return "--ignore=%s" % s
 459
 460         return " ".join(list(map(add_ignore, ignores)))
 461
 462     def kernel_string(self):
 463         r = ""
 464         if self.kernel != "":
 465             r = " --define 'alt_kernel " + self.kernel + "'"
 466         return r
 467
 468     def target_string(self):
 469         if len(self.target) > 0:
 470             return " --target " + ",".join(self.target)
 471         else:
 472             return ""
 473
 474     def bconds_string(self):
 475         r = ""
 476         for b in self.bconds_with:
 477             r = r + " --with " + b
 478         for b in self.bconds_without:
 479             r = r + " --without " + b
 480         return r
 481
 482     def defines_string(self):
 483         r = ""
 484         for key,value in self.defines.items():
 485             r += " --define '%s %s'" % (key, value)
 486         return r
 487
 488     def defines_xml(self):
 489         r = ""
 490         for key,value in self.defines.items():
 491             r += "<define name='%s'>%s</define>\n" % (escape(key), escape(value))
 492         return r
 493
 494     def default_target(self, arch):
 495         self.target.append("%s-tld-linux" % arch)
 496
 497     def write_to(self, f):
 498         f.write("""
 499          <batch id='%s' depends-on='%s'>
 500            <src-rpm>%s</src-rpm>
 501            <command flags="%s">%s</command>
 502            <spec>%s</spec>
 503            <branch>%s</branch>
 504            <info>%s</info>\n""" % (self.b_id,
 505                  ' '.join(list(map(lambda b: b.b_id, self.depends_on))),
 506                  escape(self.src_rpm),
 507                  escape(' '.join(self.command_flags)), escape(self.command),
 508                  escape(self.spec), escape(self.branch), escape(self.info)))
 509         if self.kernel != "":
 510             f.write("           <kernel>%s</kernel>\n" % escape(self.kernel))
 511         for b in self.bconds_with:
 512             f.write("           <with>%s</with>\n" % escape(b))
 513         for b in self.target:
 514             f.write("           <target>%s</target>\n" % escape(b))
 515         for b in self.bconds_without:
 516             f.write("           <without>%s</without>\n" % escape(b))
 517         if self.defines:
 518             f.write("           %s\n" % self.defines_xml())
 519         for b in self.builders:
 520             if b in self.builders_status_buildtime:
 521                 t = self.builders_status_buildtime[b]
 522             else:
 523                 t = "0"
 524             f.write("           <builder status='%s' time='%s' buildtime='%s'>%s</builder>\n" % \
 525                     (escape(self.builders_status[b]), self.builders_status_time[b], t, escape(b)))
 526         f.write("         </batch>\n")
 527
 528     def log_line(self, l):
 529         log.notice(l)
 530         if self.logfile != None:
 531             util.append_to(self.logfile, l)
 532
 533     def expand_builders(batch, all_builders):
 534         all = []
 535         for bld in batch.builders:
 536             res = []
 537             for my_bld in all_builders:
 538                 if fnmatch.fnmatch(my_bld, bld):
 539                     res.append(my_bld)
 540             if res != []:
 541                 all.extend(res)
 542             else:
 543                 all.append(bld)
 544         batch.builders = all
 545
 546 class Notification:
 547     def __init__(self, e):
 548         self.batches = []
 549         self.kind = 'notification'
 550         self.group_id = attr(e, "group-id")
 551         self.builder = attr(e, "builder")
 552         self.batches = {}
 553         self.batches_buildtime = {}
 554         for c in e.childNodes:
 555             if is_blank(c): continue
 556             if c.nodeType != Element.ELEMENT_NODE:
 557                 log.panic("xml: evil notification child %d" % c.nodeType)
 558             if c.nodeName == "batch":
 559                 id = attr(c, "id")
 560                 status = attr(c, "status")
 561                 buildtime = attr(c, "buildtime", "0")
 562                 if not status.startswith("OK") and not status.startswith("SKIP") and not status.startswith("UNSUPP") and not status.startswith("FAIL"):
 563                     log.panic("xml notification: bad status: %s" % status)
 564                 self.batches[id] = status
 565                 self.batches_buildtime[id] = buildtime
 566             else:
 567                 log.panic("xml: evil notification child (%s)" % c.nodeName)
 568
 569     # return structure usable for json encoding
 570     def dump_json(self):
 571         return dict(
 572             id=self.group_id,
 573             builder=self.builder,
 574             batches=self.batches,
 575             batches_buildtime=self.batches_buildtime,
 576         )
 577
 578     def apply_to(self, q):
 579         for r in q.requests:
 580             if r.kind == "group":
 581                 for b in r.batches:
 582                     if b.b_id in self.batches:
 583                         b.builders_status[self.builder] = self.batches[b.b_id]
 584                         b.builders_status_time[self.builder] = time.time()
 585                         b.builders_status_buildtime[self.builder] = "0" #self.batches_buildtime[b.b_id]
 586
 587 def build_request(e):
 588     if e.nodeType != Element.ELEMENT_NODE:
 589         log.panic("xml: evil request element")
 590     if e.nodeName == "group":
 591         return Group(e)
 592     elif e.nodeName == "notification":
 593         return Notification(e)
 594     elif e.nodeName == "command":
 595         # FIXME
 596         return Command(e)
 597     else:
 598         log.panic("xml: evil request [%s]" % e.nodeName)
 599
 600 def parse_request(f):
 601     d = parseString(f)
 602     return build_request(d.documentElement)
 603
 604 def parse_requests(f):
 605     d = parseString(f)
 606     res = []
 607     for r in d.documentElement.childNodes:
 608         if is_blank(r): continue
 609         res.append(build_request(r))
 610     return res