modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from __future__ import print_function
   4
   5 import os
   6 import config
   7 import string
   8 try:
   9     import urllib.request as urlmess
  10 except ImportError:
  11     import urllib as urlmess
  12     pass
  13 import re
  14 import rpm
  15 from common import fileexists, noarchcachedir
  16 from baseftptree import BasePkg, BaseFtpTree
  17 from sign import is_signed
  18
  19 errnum = 0
  20 quietmode = False
  21
  22 class SomeError(Exception):
  23     def __init__(self):
  24         return
  25
  26     def __str__(self):
  27         return "An Error occured!"
  28
  29 def bailoutonerror():
  30     if not errnum == 0:
  31         print("%d error(s) encountered... aborting" % errnum)
  32         raise SomeError()
  33
  34 def pinfo(msg):
  35     print('INFO: ' + msg)
  36
  37 def perror(msg):
  38     global errnum
  39     errnum = errnum + 1
  40     print('ERR: ' + msg)
  41
  42 def pwarning(msg):
  43     print('WARN: ' + msg)
  44
  45 def rm(file, test = False):
  46     if test:
  47         if not os.path.exists(file):
  48             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  49     else:
  50         try:
  51             os.remove(file)
  52         except OSError as e:
  53             pinfo("os.remove(%s): %s" % (file, e))
  54             #raise
  55
  56 def mv(src, dst, test = False):
  57     fsrc = src
  58     fdst = dst + '/' + src.split('/')[-1]
  59     if test:
  60         if not os.path.exists(fsrc):
  61             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  62         if not os.path.exists(dst):
  63             pinfo("TEST destination doesn't exist: %s" % dst)
  64     else:
  65         try:
  66             os.rename(fsrc, fdst)
  67         except OSError as e:
  68             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  69             raise
  70
  71 class Pkg(BasePkg):
  72     def __init__(self, nvr, tree):
  73         BasePkg.__init__(self, nvr, tree)
  74         self.name = '-'.join(nvr.split('-')[:-2])
  75         self.version = nvr.split('-')[-2]
  76         self.release = nvr.split('-')[-1]
  77         self.marked4removal = False
  78         self.marked4moving = False
  79         self.marked4movingpool = []
  80         self.errors = []
  81         self.warnings = []
  82
  83     def __cmp__(self, pkg):
  84         if self.name > pkg.name:
  85             return 1
  86         elif self.name < pkg.name:
  87             return -1
  88         else:
  89             return rpm.labelCompare(('0', self.version, self.release),
  90                                     ('0', pkg.version, pkg.release))
  91
  92
  93     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  94     # so this static function
  95     def is_debuginfo(self, nvr):
  96         """
  97         returns true if NVR is debuginfo package and separate debuginfo is enabled
  98         """
  99         if not config.separate_debuginfo:
 100             return False
 101         pkg = nvr.split('-')[:-2]
 102         return pkg[-1] == 'debuginfo' or pkg[-1] == 'debugsource'
 103
 104     def is_sourcefile(self, file):
 105         """
 106         returns true if file is source package
 107         """
 108         return file[-8:] == '.src.rpm'
 109
 110     # returns true if package build is integer
 111     def is_release(self):
 112         """
 113         To account Release tags with subver macros, we consider integer release
 114         if it contains odd number of dots:
 115
 116         1 -> True
 117         0.1 -> False
 118         0.%{subver}.%{rel}, %{rel} = 1 -> 0.20010.1 -> True
 119         0.%{subver}.%{rel}, %{rel} = 0.1 -> 0.20010.0.1 -> False
 120         """
 121         return self.release.count('.') % 2 == 0
 122
 123     def mark4moving(self):
 124         if not self.marked4moving:
 125             # Only one pkg in this pool can be marked for moving
 126             for pkg in self.marked4movingpool:
 127                 pkg.unmark4moving()
 128             self.tree.marked4moving.append(self)
 129             self.marked4moving=True
 130
 131     def unmark4moving(self):
 132         if self.marked4moving:
 133             self.tree.marked4moving.remove(self)
 134             self.marked4moving=False
 135
 136     def mark4removal(self):
 137         if not self.marked4removal:
 138             self.tree.marked4removal.append(self)
 139             self.marked4removal=True
 140
 141     def error(self, msg):
 142         self.errors.append(msg)
 143         if not quietmode:
 144             perror('%s %s' % (self.nvr, msg))
 145
 146     def warning(self, msg):
 147         self.warnings.append(msg)
 148         if not quietmode:
 149             pwarning('%s %s' % (self.nvr, msg))
 150
 151     def load(self, content=None):
 152         BasePkg.load(self, content)
 153         if 'move' in self.info:
 154             self.mark4moving()
 155
 156     def writeinfo(self):
 157         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 158         for bid in self.build.keys():
 159             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 160         for key in self.info.keys():
 161             f.write("info:%s:%s\n" % (key, ':'.join(self.info[key])))
 162         for arch in self.files.keys():
 163             for rpm in self.files[arch]:
 164                 f.write("file:%s:%s\n" % (arch, rpm))
 165
 166     def remove(self, test = False):
 167         """
 168         Remove package from ftp
 169         """
 170         for arch in self.files.keys():
 171             for rpm in self.files[arch]:
 172                 if self.is_debuginfo(rpm):
 173                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 174                 else:
 175                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 176                 if arch == 'noarch':
 177                     if fileexists(noarchcachedir + rpm + '.filelist'):
 178                         rm(noarchcachedir + rpm + '.filelist', test)
 179                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 180                         rm(noarchcachedir + rpm + '.reqlist', test)
 181         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 182
 183     def rpmfiles(self, debugfiles = True, sourcefiles  = True):
 184         """
 185         Return rpm files related to this package
 186         """
 187         files = []
 188         for arch, rpms in self.files.items():
 189             for nvr in rpms:
 190                 if self.is_debuginfo(nvr):
 191                     if debugfiles:
 192                         files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 193                 else:
 194                     if self.is_sourcefile(nvr):
 195                         if sourcefiles:
 196                             files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 197                     else:
 198                         files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 199         return files
 200
 201     def obsoletes(self):
 202         """
 203         Return obsoletes for all packages in Pkg:
 204
 205         {'php-geshi': set(['geshi'])}
 206
 207         """
 208         def rpmhdr(pkg):
 209             ts = rpm.ts()
 210             ts.setVSFlags(rpm.RPMVSF_NODSAHEADER)
 211             fdno = os.open(pkg, os.O_RDONLY)
 212             hdr = ts.hdrFromFdno(fdno)
 213             os.close(fdno)
 214             return hdr
 215
 216         obsoletes = {}
 217         for rpmfile in self.rpmfiles():
 218             if not os.path.exists(rpmfile):
 219                 continue
 220             hdr = rpmhdr(rpmfile)
 221             if not hdr[rpm.RPMTAG_OBSOLETES]:
 222                 continue
 223
 224             name = hdr[rpm.RPMTAG_NAME]
 225             if not name in obsoletes:
 226                 obsoletes[name] = set()
 227
 228             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
 229                 obsoletes[name].add(tag)
 230
 231         return obsoletes
 232
 233     def move(self, dsttree, test = False):
 234         if dsttree.has_key(self.nvr):
 235             movedany = False
 236             for arch in self.files.keys():
 237                 if arch in dsttree[self.nvr].files.keys():
 238                     msg = ""
 239                     if test:
 240                         msg = "TEST "
 241                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 242                     for rpm in self.files[arch]:
 243                         if self.is_debuginfo(rpm):
 244                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 245                         else:
 246                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 247                 else:
 248                     movedany = True
 249                     dsttree[self.nvr].files[arch] = self.files[arch]
 250                     for rpm in self.files[arch]:
 251                         if self.is_debuginfo(rpm):
 252                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 253                         else:
 254                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 255             if not test and movedany:
 256                 for bid in self.build.keys():
 257                     dsttree[self.nvr].build[bid] = self.build[bid]
 258                 dsttree[self.nvr].writeinfo()
 259             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 260         else:
 261             # move files
 262             for arch in self.files.keys():
 263                 for rpm in self.files[arch]:
 264                     if self.is_debuginfo(rpm):
 265                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 266                     else:
 267                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 268
 269             # move metadata
 270             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 271
 272 class FtpTree(BaseFtpTree):
 273     def __init__(self, tree, loadall=False):
 274         BaseFtpTree.__init__(self, tree)
 275         self.loadedpkgs = {}
 276         self.marked4removal = []
 277         self.marked4moving = []
 278         self.pkgnames = []
 279         self.__loadpkgnames()
 280         if loadall:
 281             for pkgname in self.pkgnames:
 282                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 283         # Tests:
 284         self.do_checkbuild = True
 285
 286     def __getitem__(self, key):
 287         if key in self.loadedpkgs:
 288             return self.loadedpkgs[key]
 289         elif key in self.pkgnames:
 290             pkg=Pkg(key, self)
 291             self.loadedpkgs[key]=pkg
 292             return pkg
 293         else:
 294             raise KeyError(key)
 295
 296     def has_key(self, key):
 297         if key in self.pkgnames:
 298             return True
 299         else:
 300             return False
 301
 302     def keys(self):
 303         return self.pkgnames
 304
 305     def values(self):
 306         return self.loadedpkgs.values()
 307
 308     def checktree(self, dsttree):
 309         self.__checkbuild(self.loadedpkgs.values())
 310         self.__checkarchs(dsttree, self.loadedpkgs.values())
 311
 312     def testmove(self, dsttree, archivetree = None):
 313         self.__checkbuild(self.marked4moving)
 314         self.__checkarchs(dsttree, self.marked4moving)
 315         if not self.treename.count("archive"):
 316             self.__checkduplicates(self.marked4moving)
 317
 318         self.__checksigns(dsttree, self.marked4moving, test = True)
 319         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
 320         self.__checkforrelease(dsttree, self.marked4moving, test = True)
 321
 322         if not self.treename.count("archive"):
 323             self.__rmolderfromsrc(test = True)
 324             self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 325
 326         for pkg in self.marked4moving:
 327             pkg.move(dsttree, test = True)
 328
 329     def movepkgs(self, dsttree, archivetree = None):
 330         if self.do_checkbuild:
 331             self.__checkbuild(self.marked4moving)
 332         bailoutonerror()
 333
 334         self.__checkarchs(dsttree, self.marked4moving)
 335         bailoutonerror()
 336
 337         self.__checksigns(dsttree, self.marked4moving)
 338         bailoutonerror()
 339
 340         if not self.treename.count("archive"):
 341             self.__rmolderfromsrc()
 342             self.__rmotherfromdst(dsttree, archivetree = archivetree)
 343
 344         for pkg in self.marked4moving:
 345             pkg.move(dsttree)
 346
 347     def rpmfiles(self, debugfiles = True, sourcefiles = True):
 348         if self.do_checkbuild:
 349             self.__checkbuild(self.marked4moving)
 350
 351         files = []
 352         for pkg in self.marked4moving:
 353             files += pkg.rpmfiles(debugfiles = debugfiles, sourcefiles = sourcefiles)
 354         return files
 355
 356     def removepkgs(self):
 357         if self.do_checkbuild:
 358             self.__checkbuild(self.marked4removal)
 359         bailoutonerror()
 360         for pkg in self.marked4removal:
 361             pkg.remove()
 362
 363     def mark4removal(self, wannabepkgs):
 364         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 365
 366     def mark4moving(self, wannabepkgs):
 367         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 368
 369     # Internal functions below
 370     def __arch_stringify(self, list):
 371         ret = []
 372         dist = config.ftp_dist;
 373         for arch in list:
 374             ret.append(dist + '-' + arch)
 375         return ' '.join(ret)
 376
 377     def __loadpkgnames(self):
 378         def checkfiletype(name):
 379             if name[-13:]=='.src.rpm.info':
 380                 return True
 381             else:
 382                 return False
 383         pkglist = list(filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata')))
 384         self.pkgnames = list(map((lambda x: x[:-13]), pkglist))
 385
 386     def __mark4something(self, wannabepkgs, markfunction):
 387         def chopoffextension(pkg):
 388             found = pkg.find('.src.rpm')
 389             if found == -1:
 390                 return pkg
 391             else:
 392                 return pkg[:found]
 393
 394         for wannabepkg in wannabepkgs:
 395             pkgname = chopoffextension(wannabepkg)
 396             if pkgname in self.pkgnames:
 397                 if not pkgname in self.loadedpkgs.keys():
 398                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 399                 markfunction(self.loadedpkgs[pkgname])
 400             else:
 401                 perror('%s not found in source tree' % pkgname)
 402         bailoutonerror()
 403
 404     def __checkbuild(self, marked):
 405         """
 406         Checks queue file if all arches are built
 407
 408         Reads config.builderqueue to grab the info
 409         """
 410         f = urlmess.urlopen(config.builderqueue)
 411         requests = {}
 412         reid = re.compile(r'^.*id=(.*) pri.*$')
 413         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 414         for i in re.findall(regb, f.read().decode('utf-8')):
 415             if i[0] == 'g':
 416                 id = reid.sub(r'\1', i)
 417                 requests[id] = ""
 418             elif i[0]=='b':
 419                 requests[id] = requests[id] + i
 420         f.close()
 421
 422         for pkg in marked:
 423             for bid in pkg.build.keys():
 424                 if bid in requests and not requests[bid].find('?') == -1:
 425                     pkg.error("(buildid %s) building not finished" % bid)
 426
 427     def __checkarchs(self, dsttree, marked):
 428         """
 429         Checks marked pkgs it is built on all archs.
 430         """
 431         for pkg in marked:
 432             if len(pkg.files.keys()) <= 1:
 433                 pkg.error('has only src.rpm built')
 434                 continue
 435             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 436
 437             # check if we're not removing some archs
 438             if otherpkgnames:
 439                 curarchs = []
 440                 missingarchs = []
 441                 for somepkg in otherpkgnames:
 442                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 443                 for arch in curarchs:
 444                     if arch not in pkg.files.keys():
 445                         missingarchs.append(arch)
 446                 if missingarchs:
 447                     pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
 448             else:
 449                 # warn if a package isn't built for all archs
 450                 # ftp_archs + SRPMS
 451                 ftp_archs_num = len(config.ftp_archs) + 1
 452                 if (config.separate_noarch and 'noarch' in pkg.files.keys()):
 453                     # ftp_archs + SRPMS + noarch subpackages
 454                     ftp_archs_num += 1
 455                     # plain simple noarch package
 456                     if (len(pkg.files.keys()) == 2):
 457                         continue
 458
 459                 if len(pkg.files.keys()) != ftp_archs_num:
 460                     missingarchs = []
 461                     for arch in config.ftp_archs:
 462                         if arch not in pkg.files.keys():
 463                             missingarchs.append(arch)
 464                     pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
 465
 466     def __checkduplicates(self, marked):
 467         """
 468         Checks if marked packages contain duplicate packages (with different versions)
 469         """
 470         for pkg in marked:
 471             olderpkgnames = self.__find_older_pkgs(pkg)
 472             for i in olderpkgnames:
 473                 markednames = [str(x) for x in marked]
 474                 if i in markednames:
 475                     pkg.error('duplicate package: %s' % i)
 476
 477     def __rmolderfromsrc(self, test = False):
 478         for pkg in self.marked4moving:
 479             olderpkgnames = self.__find_older_pkgs(pkg)
 480             for i in olderpkgnames:
 481                 Pkg(i, self).remove(test)
 482
 483     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 484         for pkg in self.marked4moving:
 485             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 486             for i in pkgnames:
 487                 if archivetree == None:
 488                     Pkg(i, dsttree).remove(test)
 489                 else:
 490                     Pkg(i, dsttree).move(archivetree, test = test)
 491
 492     # Used more than once filter functions
 493     def __find_other_pkgs(self, pkg, tree):
 494         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 495         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 496         def filter_other_pkgs(x):
 497             if ziewre.match(x) and not x == pkg.nvr:
 498                 return True
 499             else:
 500                 return False
 501         return list(filter(filter_other_pkgs, tree.pkgnames))
 502
 503     def __find_older_pkgs(self, pkg):
 504         def filter_older_pkgs(x):
 505             c = x.split('-')
 506             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 507                                                         ('0', c[-2], c[-1]))
 508             if rc == 1: # pkg > x
 509                 return True
 510             else:
 511                 return False
 512         return list(filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self)))
 513
 514     def __checksigns(self, tree, pkgs, test = False):
 515         """
 516         Checks if pkgs in tree are all signed.
 517
 518         in case of test = true, error flag is set for unsigned packages
 519         """
 520         if not tree.treename in config.signed_trees:
 521             return
 522
 523         for pkg in pkgs:
 524             unsigned = 0
 525             for file in pkg.rpmfiles():
 526                 if not is_signed(file):
 527                     unsigned += 1
 528
 529             if unsigned != 0:
 530                 if test == True:
 531                     if not quietmode:
 532                         pkg.warning('%d files not signed' % unsigned)
 533                 else:
 534                     pkg.error('%d files not signed' % unsigned)
 535
 536     def __checkforobsoletes(self, tree, pkgs, test = False):
 537         """
 538         Checks queue file if package obsoletes something in destination tree and suggest for removal.
 539
 540         Only NAME tag is compared, i.e virtual packages do not get reported.
 541
 542         """
 543         if test != True:
 544             return
 545
 546         def findbyname(name):
 547             def x(nvr):
 548                 return '-'.join(nvr.split('-')[:-2]) == name
 549             return list(filter(x, tree.pkgnames))
 550
 551         for pkg in pkgs:
 552             obsoletes = pkg.obsoletes()
 553             if not obsoletes:
 554                 continue
 555
 556             for pn, setlist in obsoletes.items():
 557                 for item in setlist:
 558                     p = findbyname(item)
 559                     if p:
 560                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))
 561
 562     def __checkforrelease(self, tree, pkgs, test = False):
 563         """
 564         Checks queue file if package release is non integer.
 565
 566         """
 567         if test != True:
 568             return
 569
 570         for pkg in pkgs:
 571             if not pkg.is_release():
 572                 pkg.warning('non-integer release: %s' % pkg.release)