modules/ftptree.py

   1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
   2
   3 from __future__ import print_function
   4
   5 import os
   6 import config
   7 import string
   8 try:
   9     import urllib.request as urlmess
  10 except ImportError:
  11     import urllib as urlmess
  12     pass
  13 import re
  14 import rpm
  15 from common import fileexists, noarchcachedir
  16 from baseftptree import BasePkg, BaseFtpTree
  17 from sign import is_signed
  18
  19 errnum = 0
  20 quietmode = False
  21
  22 class SomeError(Exception):
  23     def __init__(self):
  24         return
  25
  26     def __str__(self):
  27         return "An Error occured!"
  28
  29 def bailoutonerror():
  30     if not errnum == 0:
  31         print("%d error(s) encountered... aborting" % errnum)
  32         raise SomeError()
  33
  34 def pinfo(msg):
  35     print('INFO: ' + msg)
  36
  37 def perror(msg):
  38     global errnum
  39     errnum = errnum + 1
  40     print('ERR: ' + msg)
  41
  42 def pwarning(msg):
  43     print('WARN: ' + msg)
  44
  45 def rm(file, test = False):
  46     if test:
  47         if not os.path.exists(file):
  48             pinfo("TEST os.remove(%s): file doesn't exists" % file)
  49     else:
  50         try:
  51             os.remove(file)
  52         except OSError as e:
  53             pinfo("os.remove(%s): %s" % (file, e))
  54             #raise
  55
  56 def mv(src, dst, test = False):
  57     fsrc = src
  58     fdst = dst + '/' + src.split('/')[-1]
  59     if test:
  60         if not os.path.exists(fsrc):
  61             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
  62         if not os.path.exists(dst):
  63             pinfo("TEST destination doesn't exist: %s" % dst)
  64     else:
  65         try:
  66             os.rename(fsrc, fdst)
  67         except OSError as e:
  68             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
  69             raise
  70
  71 class Pkg(BasePkg):
  72     def __init__(self, nvr, tree):
  73         BasePkg.__init__(self, nvr, tree)
  74         self.name = '-'.join(nvr.split('-')[:-2])
  75         self.version = nvr.split('-')[-2]
  76         self.release = nvr.split('-')[-1]
  77         self.marked4removal = False
  78         self.marked4moving = False
  79         self.marked4movingpool = []
  80         self.errors = []
  81         self.warnings = []
  82
  83     def __cmp__(self, pkg):
  84         if self.name > pkg.name:
  85             return 1
  86         elif self.name < pkg.name:
  87             return -1
  88         else:
  89             return rpm.labelCompare(('0', self.version, self.release),
  90                                     ('0', pkg.version, pkg.release))
  91
  92
  93     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
  94     # so this static function
  95     def is_debuginfo(self, nvr):
  96         """
  97         returns true if NVR is debuginfo package and separate debuginfo is enabled
  98         """
  99         if not config.separate_debuginfo:
 100             return False
 101         pkg = nvr.split('-')[:-2]
 102         return pkg[-1] == 'debuginfo' or pkg[-1] == 'debugsource'
 103
 104     def is_sourcefile(self, file):
 105         """
 106         returns true if file is source package
 107         """
 108         return file[-8:] == '.src.rpm'
 109
 110     def mark4moving(self):
 111         if not self.marked4moving:
 112             # Only one pkg in this pool can be marked for moving
 113             for pkg in self.marked4movingpool:
 114                 pkg.unmark4moving()
 115             self.tree.marked4moving.append(self)
 116             self.marked4moving=True
 117
 118     def unmark4moving(self):
 119         if self.marked4moving:
 120             self.tree.marked4moving.remove(self)
 121             self.marked4moving=False
 122
 123     def mark4removal(self):
 124         if not self.marked4removal:
 125             self.tree.marked4removal.append(self)
 126             self.marked4removal=True
 127
 128     def error(self, msg):
 129         self.errors.append(msg)
 130         if not quietmode:
 131             perror('%s %s' % (self.nvr, msg))
 132
 133     def warning(self, msg):
 134         self.warnings.append(msg)
 135         if not quietmode:
 136             pwarning('%s %s' % (self.nvr, msg))
 137
 138     def load(self, content=None):
 139         BasePkg.load(self, content)
 140         if 'move' in self.info:
 141             self.mark4moving()
 142
 143     def writeinfo(self):
 144         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
 145         for bid in self.build.keys():
 146             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
 147         for key in self.info.keys():
 148             f.write("info:%s:%s\n" % (key, ':'.join(self.info[key])))
 149         for arch in self.files.keys():
 150             for rpm in self.files[arch]:
 151                 f.write("file:%s:%s\n" % (arch, rpm))
 152
 153     def remove(self, test = False):
 154         """
 155         Remove package from ftp
 156         """
 157         for arch in self.files.keys():
 158             for rpm in self.files[arch]:
 159                 if self.is_debuginfo(rpm):
 160                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 161                 else:
 162                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 163                 if arch == 'noarch':
 164                     if fileexists(noarchcachedir + rpm + '.filelist'):
 165                         rm(noarchcachedir + rpm + '.filelist', test)
 166                     if fileexists(noarchcachedir + rpm + '.reqlist'):
 167                         rm(noarchcachedir + rpm + '.reqlist', test)
 168         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 169
 170     def rpmfiles(self, debugfiles = True, sourcefiles  = True):
 171         """
 172         Return rpm files related to this package
 173         """
 174         files = []
 175         for arch, rpms in self.files.items():
 176             for nvr in rpms:
 177                 if self.is_debuginfo(nvr):
 178                     if debugfiles:
 179                         files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
 180                 else:
 181                     if self.is_sourcefile(nvr):
 182                         if sourcefiles:
 183                             files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 184                     else:
 185                         files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
 186         return files
 187
 188     def obsoletes(self):
 189         """
 190         Return obsoletes for all packages in Pkg:
 191
 192         {'php-geshi': set(['geshi'])}
 193
 194         """
 195         def rpmhdr(pkg):
 196             ts = rpm.ts()
 197             ts.setVSFlags(rpm.RPMVSF_NODSAHEADER)
 198             fdno = os.open(pkg, os.O_RDONLY)
 199             hdr = ts.hdrFromFdno(fdno)
 200             os.close(fdno)
 201             return hdr
 202
 203         obsoletes = {}
 204         for rpmfile in self.rpmfiles():
 205             if not os.path.exists(rpmfile):
 206                 continue
 207             hdr = rpmhdr(rpmfile)
 208             if not hdr[rpm.RPMTAG_OBSOLETES]:
 209                 continue
 210
 211             name = hdr[rpm.RPMTAG_NAME]
 212             if not name in obsoletes:
 213                 obsoletes[name] = set()
 214
 215             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
 216                 obsoletes[name].add(tag)
 217
 218         return obsoletes
 219
 220     def move(self, dsttree, test = False):
 221         if dsttree.has_key(self.nvr):
 222             movedany = False
 223             for arch in self.files.keys():
 224                 if arch in dsttree[self.nvr].files.keys():
 225                     msg = ""
 226                     if test:
 227                         msg = "TEST "
 228                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
 229                     for rpm in self.files[arch]:
 230                         if self.is_debuginfo(rpm):
 231                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
 232                         else:
 233                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
 234                 else:
 235                     movedany = True
 236                     dsttree[self.nvr].files[arch] = self.files[arch]
 237                     for rpm in self.files[arch]:
 238                         if self.is_debuginfo(rpm):
 239                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 240                         else:
 241                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 242             if not test and movedany:
 243                 for bid in self.build.keys():
 244                     dsttree[self.nvr].build[bid] = self.build[bid]
 245                 dsttree[self.nvr].writeinfo()
 246             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
 247         else:
 248             # move files
 249             for arch in self.files.keys():
 250                 for rpm in self.files[arch]:
 251                     if self.is_debuginfo(rpm):
 252                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
 253                     else:
 254                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
 255
 256             # move metadata
 257             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
 258
 259 class FtpTree(BaseFtpTree):
 260     def __init__(self, tree, loadall=False):
 261         BaseFtpTree.__init__(self, tree)
 262         self.loadedpkgs = {}
 263         self.marked4removal = []
 264         self.marked4moving = []
 265         self.pkgnames = []
 266         self.__loadpkgnames()
 267         if loadall:
 268             for pkgname in self.pkgnames:
 269                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
 270         # Tests:
 271         self.do_checkbuild = True
 272
 273     def __getitem__(self, key):
 274         if key in self.loadedpkgs:
 275             return self.loadedpkgs[key]
 276         elif key in self.pkgnames:
 277             pkg=Pkg(key, self)
 278             self.loadedpkgs[key]=pkg
 279             return pkg
 280         else:
 281             raise KeyError(key)
 282
 283     def has_key(self, key):
 284         if key in self.pkgnames:
 285             return True
 286         else:
 287             return False
 288
 289     def keys(self):
 290         return self.pkgnames
 291
 292     def values(self):
 293         return self.loadedpkgs.values()
 294
 295     def checktree(self, dsttree):
 296         self.__checkbuild(self.loadedpkgs.values())
 297         self.__checkarchs(dsttree, self.loadedpkgs.values())
 298
 299     def testmove(self, dsttree, archivetree = None):
 300         self.__checkbuild(self.marked4moving)
 301         self.__checkarchs(dsttree, self.marked4moving)
 302         if not self.treename.count("archive"):
 303             self.__checkduplicates(self.marked4moving)
 304
 305         self.__checksigns(dsttree, self.marked4moving, test = True)
 306         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
 307
 308         if not self.treename.count("archive"):
 309             self.__rmolderfromsrc(test = True)
 310             self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
 311
 312         for pkg in self.marked4moving:
 313             pkg.move(dsttree, test = True)
 314
 315     def movepkgs(self, dsttree, archivetree = None):
 316         if self.do_checkbuild:
 317             self.__checkbuild(self.marked4moving)
 318         bailoutonerror()
 319
 320         self.__checkarchs(dsttree, self.marked4moving)
 321         bailoutonerror()
 322
 323         self.__checksigns(dsttree, self.marked4moving)
 324         bailoutonerror()
 325
 326         if not self.treename.count("archive"):
 327             self.__rmolderfromsrc()
 328             self.__rmotherfromdst(dsttree, archivetree = archivetree)
 329
 330         for pkg in self.marked4moving:
 331             pkg.move(dsttree)
 332
 333     def rpmfiles(self, debugfiles = True, sourcefiles = True):
 334         if self.do_checkbuild:
 335             self.__checkbuild(self.marked4moving)
 336
 337         files = []
 338         for pkg in self.marked4moving:
 339             files += pkg.rpmfiles(debugfiles = debugfiles, sourcefiles = sourcefiles)
 340         return files
 341
 342     def removepkgs(self):
 343         if self.do_checkbuild:
 344             self.__checkbuild(self.marked4removal)
 345         bailoutonerror()
 346         for pkg in self.marked4removal:
 347             pkg.remove()
 348
 349     def mark4removal(self, wannabepkgs):
 350         self.__mark4something(wannabepkgs, Pkg.mark4removal)
 351
 352     def mark4moving(self, wannabepkgs):
 353         self.__mark4something(wannabepkgs, Pkg.mark4moving)
 354
 355     # Internal functions below
 356     def __arch_stringify(self, list):
 357         ret = []
 358         dist = config.ftp_dist;
 359         for arch in list:
 360             ret.append(dist + '-' + arch)
 361         return ' '.join(ret)
 362
 363     def __loadpkgnames(self):
 364         def checkfiletype(name):
 365             if name[-13:]=='.src.rpm.info':
 366                 return True
 367             else:
 368                 return False
 369         pkglist = list(filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata')))
 370         self.pkgnames = list(map((lambda x: x[:-13]), pkglist))
 371
 372     def __mark4something(self, wannabepkgs, markfunction):
 373         def chopoffextension(pkg):
 374             found = pkg.find('.src.rpm')
 375             if found == -1:
 376                 return pkg
 377             else:
 378                 return pkg[:found]
 379
 380         for wannabepkg in wannabepkgs:
 381             pkgname = chopoffextension(wannabepkg)
 382             if pkgname in self.pkgnames:
 383                 if not pkgname in self.loadedpkgs.keys():
 384                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
 385                 markfunction(self.loadedpkgs[pkgname])
 386             else:
 387                 perror('%s not found in source tree' % pkgname)
 388         bailoutonerror()
 389
 390     def __checkbuild(self, marked):
 391         """
 392         Checks queue file if all arches are built
 393
 394         Reads config.builderqueue to grab the info
 395         """
 396         f = urlmess.urlopen(config.builderqueue)
 397         requests = {}
 398         reid = re.compile(r'^.*id=(.*) pri.*$')
 399         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
 400         for i in re.findall(regb, f.read().decode('utf-8')):
 401             if i[0] == 'g':
 402                 id = reid.sub(r'\1', i)
 403                 requests[id] = ""
 404             elif i[0]=='b':
 405                 requests[id] = requests[id] + i
 406         f.close()
 407
 408         for pkg in marked:
 409             for bid in pkg.build.keys():
 410                 if bid in requests and not requests[bid].find('?') == -1:
 411                     pkg.error("(buildid %s) building not finished" % bid)
 412
 413     def __checkarchs(self, dsttree, marked):
 414         """
 415         Checks marked pkgs it is built on all archs.
 416         """
 417         for pkg in marked:
 418             if len(pkg.files.keys()) <= 1:
 419                 pkg.error('has only src.rpm built')
 420                 continue
 421             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
 422
 423             # check if we're not removing some archs
 424             if otherpkgnames:
 425                 curarchs = []
 426                 missingarchs = []
 427                 for somepkg in otherpkgnames:
 428                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
 429                 for arch in curarchs:
 430                     if arch not in pkg.files.keys():
 431                         missingarchs.append(arch)
 432                 if missingarchs:
 433                     pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
 434             else:
 435                 # warn if a package isn't built for all archs
 436                 # ftp_archs + SRPMS
 437                 ftp_archs_num = len(config.ftp_archs) + 1
 438                 if (config.separate_noarch and 'noarch' in pkg.files.keys()):
 439                     # ftp_archs + SRPMS + noarch subpackages
 440                     ftp_archs_num += 1
 441                     # plain simple noarch package
 442                     if (len(pkg.files.keys()) == 2):
 443                         continue
 444
 445                 if len(pkg.files.keys()) != ftp_archs_num:
 446                     missingarchs = []
 447                     for arch in config.ftp_archs:
 448                         if arch not in pkg.files.keys():
 449                             missingarchs.append(arch)
 450                     pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
 451
 452     def __checkduplicates(self, marked):
 453         """
 454         Checks if marked packages contain duplicate packages (with different versions)
 455         """
 456         for pkg in marked:
 457             olderpkgnames = self.__find_older_pkgs(pkg)
 458             for i in olderpkgnames:
 459                 markednames = [str(x) for x in marked]
 460                 if i in markednames:
 461                     pkg.error('duplicate package: %s' % i)
 462
 463     def __rmolderfromsrc(self, test = False):
 464         for pkg in self.marked4moving:
 465             olderpkgnames = self.__find_older_pkgs(pkg)
 466             for i in olderpkgnames:
 467                 Pkg(i, self).remove(test)
 468
 469     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
 470         for pkg in self.marked4moving:
 471             pkgnames = self.__find_other_pkgs(pkg, dsttree)
 472             for i in pkgnames:
 473                 if archivetree == None:
 474                     Pkg(i, dsttree).remove(test)
 475                 else:
 476                     Pkg(i, dsttree).move(archivetree, test = test)
 477
 478     # Used more than once filter functions
 479     def __find_other_pkgs(self, pkg, tree):
 480         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
 481         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
 482         def filter_other_pkgs(x):
 483             if ziewre.match(x) and not x == pkg.nvr:
 484                 return True
 485             else:
 486                 return False
 487         return list(filter(filter_other_pkgs, tree.pkgnames))
 488
 489     def __find_older_pkgs(self, pkg):
 490         def filter_older_pkgs(x):
 491             c = x.split('-')
 492             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
 493                                                         ('0', c[-2], c[-1]))
 494             if rc == 1: # pkg > x
 495                 return True
 496             else:
 497                 return False
 498         return list(filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self)))
 499
 500     def __checksigns(self, tree, pkgs, test = False):
 501         """
 502         Checks if pkgs in tree are all signed.
 503
 504         in case of test = true, error flag is set for unsigned packages
 505         """
 506         if not tree.treename in config.signed_trees:
 507             return
 508
 509         for pkg in pkgs:
 510             unsigned = 0
 511             for file in pkg.rpmfiles():
 512                 if not is_signed(file):
 513                     unsigned += 1
 514
 515             if unsigned != 0:
 516                 if test == True:
 517                     if not quietmode:
 518                         pkg.warning('%d files not signed' % unsigned)
 519                 else:
 520                     pkg.error('%d files not signed' % unsigned)
 521
 522     def __checkforobsoletes(self, tree, pkgs, test = False):
 523         """
 524         Checks queue file if package obsoletes something in destination tree and suggest for removal.
 525
 526         Only NAME tag is compared, i.e virtual packages do not get reported.
 527
 528         """
 529         if test != True:
 530             return
 531
 532         def findbyname(name):
 533             def x(nvr):
 534                 return '-'.join(nvr.split('-')[:-2]) == name
 535             return list(filter(x, tree.pkgnames))
 536
 537         for pkg in pkgs:
 538             obsoletes = pkg.obsoletes()
 539             if not obsoletes:
 540                 continue
 541
 542             for pn, setlist in obsoletes.items():
 543                 for item in setlist:
 544                     p = findbyname(item)
 545                     if p:
 546                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))