1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
3 from __future__ import print_function
9 import urllib.request as urlmess
11 import urllib as urlmess
15 from common import fileexists, noarchcachedir
16 from baseftptree import BasePkg, BaseFtpTree
17 from sign import is_signed
22 class SomeError(Exception):
27 return "An Error occured!"
31 print("%d error(s) encountered... aborting" % errnum)
45 def rm(file, test = False):
47 if not os.path.exists(file):
48 pinfo("TEST os.remove(%s): file doesn't exists" % file)
53 pinfo("os.remove(%s): %s" % (file, e))
56 def mv(src, dst, test = False):
58 fdst = dst + '/' + src.split('/')[-1]
60 if not os.path.exists(fsrc):
61 pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
62 if not os.path.exists(dst):
63 pinfo("TEST destination doesn't exist: %s" % dst)
68 pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
72 def __init__(self, nvr, tree):
73 BasePkg.__init__(self, nvr, tree)
74 self.name = '-'.join(nvr.split('-')[:-2])
75 self.version = nvr.split('-')[-2]
76 self.release = nvr.split('-')[-1]
77 self.marked4removal = False
78 self.marked4moving = False
79 self.marked4movingpool = []
83 def __cmp__(self, pkg):
84 if self.name > pkg.name:
86 elif self.name < pkg.name:
89 return rpm.labelCompare(('0', self.version, self.release),
90 ('0', pkg.version, pkg.release))
93 # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
94 # so this static function
95 def is_debuginfo(self, nvr):
97 returns true if NVR is debuginfo package and separate debuginfo is enabled
99 if not config.separate_debuginfo:
101 pkg = nvr.split('-')[:-2]
102 return pkg[-1] == 'debuginfo' or pkg[-1] == 'debugsource'
104 def is_sourcefile(self, file):
106 returns true if file is source package
108 return file[-8:] == '.src.rpm'
110 def mark4moving(self):
111 if not self.marked4moving:
112 # Only one pkg in this pool can be marked for moving
113 for pkg in self.marked4movingpool:
115 self.tree.marked4moving.append(self)
116 self.marked4moving=True
118 def unmark4moving(self):
119 if self.marked4moving:
120 self.tree.marked4moving.remove(self)
121 self.marked4moving=False
123 def mark4removal(self):
124 if not self.marked4removal:
125 self.tree.marked4removal.append(self)
126 self.marked4removal=True
128 def error(self, msg):
129 self.errors.append(msg)
131 perror('%s %s' % (self.nvr, msg))
133 def warning(self, msg):
134 self.warnings.append(msg)
136 pwarning('%s %s' % (self.nvr, msg))
138 def load(self, content=None):
139 BasePkg.load(self, content)
140 if 'move' in self.info:
144 f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
145 for bid in self.build.keys():
146 f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
147 for key in self.info.keys():
148 f.write("info:%s:%s\n" % (key, ':'.join(self.info[key])))
149 for arch in self.files.keys():
150 for rpm in self.files[arch]:
151 f.write("file:%s:%s\n" % (arch, rpm))
153 def remove(self, test = False):
155 Remove package from ftp
157 for arch in self.files.keys():
158 for rpm in self.files[arch]:
159 if self.is_debuginfo(rpm):
160 rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
162 rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
164 if fileexists(noarchcachedir + rpm + '.filelist'):
165 rm(noarchcachedir + rpm + '.filelist', test)
166 if fileexists(noarchcachedir + rpm + '.reqlist'):
167 rm(noarchcachedir + rpm + '.reqlist', test)
168 rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
170 def rpmfiles(self, debugfiles = True, sourcefiles = True):
172 Return rpm files related to this package
175 for arch, rpms in self.files.items():
177 if self.is_debuginfo(nvr):
179 files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
181 if self.is_sourcefile(nvr):
183 files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
185 files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
190 Return obsoletes for all packages in Pkg:
192 {'php-geshi': set(['geshi'])}
197 ts.setVSFlags(rpm.RPMVSF_NODSAHEADER)
198 fdno = os.open(pkg, os.O_RDONLY)
199 hdr = ts.hdrFromFdno(fdno)
204 for rpmfile in self.rpmfiles():
205 if not os.path.exists(rpmfile):
207 hdr = rpmhdr(rpmfile)
208 if not hdr[rpm.RPMTAG_OBSOLETES]:
211 name = hdr[rpm.RPMTAG_NAME]
212 if not name in obsoletes:
213 obsoletes[name] = set()
215 for tag in hdr[rpm.RPMTAG_OBSOLETES]:
216 obsoletes[name].add(tag)
220 def move(self, dsttree, test = False):
221 if dsttree.has_key(self.nvr):
223 for arch in self.files.keys():
224 if arch in dsttree[self.nvr].files.keys():
228 pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
229 for rpm in self.files[arch]:
230 if self.is_debuginfo(rpm):
231 rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
233 rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
236 dsttree[self.nvr].files[arch] = self.files[arch]
237 for rpm in self.files[arch]:
238 if self.is_debuginfo(rpm):
239 mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
241 mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
242 if not test and movedany:
243 for bid in self.build.keys():
244 dsttree[self.nvr].build[bid] = self.build[bid]
245 dsttree[self.nvr].writeinfo()
246 rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
249 for arch in self.files.keys():
250 for rpm in self.files[arch]:
251 if self.is_debuginfo(rpm):
252 mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
254 mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
257 mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
259 class FtpTree(BaseFtpTree):
260 def __init__(self, tree, loadall=False):
261 BaseFtpTree.__init__(self, tree)
263 self.marked4removal = []
264 self.marked4moving = []
266 self.__loadpkgnames()
268 for pkgname in self.pkgnames:
269 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
271 self.do_checkbuild = True
273 def __getitem__(self, key):
274 if key in self.loadedpkgs:
275 return self.loadedpkgs[key]
276 elif key in self.pkgnames:
278 self.loadedpkgs[key]=pkg
283 def has_key(self, key):
284 if key in self.pkgnames:
293 return self.loadedpkgs.values()
295 def checktree(self, dsttree):
296 self.__checkbuild(self.loadedpkgs.values())
297 self.__checkarchs(dsttree, self.loadedpkgs.values())
299 def testmove(self, dsttree, archivetree = None):
300 self.__checkbuild(self.marked4moving)
301 self.__checkarchs(dsttree, self.marked4moving)
302 if not self.treename.count("archive"):
303 self.__checkduplicates(self.marked4moving)
305 self.__checksigns(dsttree, self.marked4moving, test = True)
306 self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
308 if not self.treename.count("archive"):
309 self.__rmolderfromsrc(test = True)
310 self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
312 for pkg in self.marked4moving:
313 pkg.move(dsttree, test = True)
315 def movepkgs(self, dsttree, archivetree = None):
316 if self.do_checkbuild:
317 self.__checkbuild(self.marked4moving)
320 self.__checkarchs(dsttree, self.marked4moving)
323 self.__checksigns(dsttree, self.marked4moving)
326 if not self.treename.count("archive"):
327 self.__rmolderfromsrc()
328 self.__rmotherfromdst(dsttree, archivetree = archivetree)
330 for pkg in self.marked4moving:
333 def rpmfiles(self, debugfiles = True, sourcefiles = True):
334 if self.do_checkbuild:
335 self.__checkbuild(self.marked4moving)
338 for pkg in self.marked4moving:
339 files += pkg.rpmfiles(debugfiles = debugfiles, sourcefiles = sourcefiles)
342 def removepkgs(self):
343 if self.do_checkbuild:
344 self.__checkbuild(self.marked4removal)
346 for pkg in self.marked4removal:
349 def mark4removal(self, wannabepkgs):
350 self.__mark4something(wannabepkgs, Pkg.mark4removal)
352 def mark4moving(self, wannabepkgs):
353 self.__mark4something(wannabepkgs, Pkg.mark4moving)
355 # Internal functions below
356 def __arch_stringify(self, list):
358 dist = config.ftp_dist;
360 ret.append(dist + '-' + arch)
363 def __loadpkgnames(self):
364 def checkfiletype(name):
365 if name[-13:]=='.src.rpm.info':
369 pkglist = list(filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata')))
370 self.pkgnames = list(map((lambda x: x[:-13]), pkglist))
372 def __mark4something(self, wannabepkgs, markfunction):
373 def chopoffextension(pkg):
374 found = pkg.find('.src.rpm')
380 for wannabepkg in wannabepkgs:
381 pkgname = chopoffextension(wannabepkg)
382 if pkgname in self.pkgnames:
383 if not pkgname in self.loadedpkgs.keys():
384 self.loadedpkgs[pkgname]=Pkg(pkgname, self)
385 markfunction(self.loadedpkgs[pkgname])
387 perror('%s not found in source tree' % pkgname)
390 def __checkbuild(self, marked):
392 Checks queue file if all arches are built
394 Reads config.builderqueue to grab the info
396 f = urlmess.urlopen(config.builderqueue)
398 reid = re.compile(r'^.*id=(.*) pri.*$')
399 regb = re.compile(r'^group:.*$|builders:.*$', re.M)
400 for i in re.findall(regb, f.read().decode('utf-8')):
402 id = reid.sub(r'\1', i)
405 requests[id] = requests[id] + i
409 for bid in pkg.build.keys():
410 if bid in requests and not requests[bid].find('?') == -1:
411 pkg.error("(buildid %s) building not finished" % bid)
413 def __checkarchs(self, dsttree, marked):
415 Checks marked pkgs it is built on all archs.
418 if len(pkg.files.keys()) <= 1:
419 pkg.error('has only src.rpm built')
421 otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
423 # check if we're not removing some archs
427 for somepkg in otherpkgnames:
428 curarchs.extend(Pkg(somepkg, dsttree).files.keys())
429 for arch in curarchs:
430 if arch not in pkg.files.keys():
431 missingarchs.append(arch)
433 pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
435 # warn if a package isn't built for all archs
437 ftp_archs_num = len(config.ftp_archs) + 1
438 if (config.separate_noarch and 'noarch' in pkg.files.keys()):
439 # ftp_archs + SRPMS + noarch subpackages
441 # plain simple noarch package
442 if (len(pkg.files.keys()) == 2):
445 if len(pkg.files.keys()) != ftp_archs_num:
447 for arch in config.ftp_archs:
448 if arch not in pkg.files.keys():
449 missingarchs.append(arch)
450 pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
452 def __checkduplicates(self, marked):
454 Checks if marked packages contain duplicate packages (with different versions)
457 olderpkgnames = self.__find_older_pkgs(pkg)
458 for i in olderpkgnames:
459 markednames = [str(x) for x in marked]
461 pkg.error('duplicate package: %s' % i)
463 def __rmolderfromsrc(self, test = False):
464 for pkg in self.marked4moving:
465 olderpkgnames = self.__find_older_pkgs(pkg)
466 for i in olderpkgnames:
467 Pkg(i, self).remove(test)
469 def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
470 for pkg in self.marked4moving:
471 pkgnames = self.__find_other_pkgs(pkg, dsttree)
473 if archivetree == None:
474 Pkg(i, dsttree).remove(test)
476 Pkg(i, dsttree).move(archivetree, test = test)
478 # Used more than once filter functions
479 def __find_other_pkgs(self, pkg, tree):
480 escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
481 ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
482 def filter_other_pkgs(x):
483 if ziewre.match(x) and not x == pkg.nvr:
487 return list(filter(filter_other_pkgs, tree.pkgnames))
489 def __find_older_pkgs(self, pkg):
490 def filter_older_pkgs(x):
492 rc = rpm.labelCompare(('0', pkg.version, pkg.release),
494 if rc == 1: # pkg > x
498 return list(filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self)))
500 def __checksigns(self, tree, pkgs, test = False):
502 Checks if pkgs in tree are all signed.
504 in case of test = true, error flag is set for unsigned packages
506 if not tree.treename in config.signed_trees:
511 for file in pkg.rpmfiles():
512 if not is_signed(file):
518 pkg.warning('%d files not signed' % unsigned)
520 pkg.error('%d files not signed' % unsigned)
522 def __checkforobsoletes(self, tree, pkgs, test = False):
524 Checks queue file if package obsoletes something in destination tree and suggest for removal.
526 Only NAME tag is compared, i.e virtual packages do not get reported.
532 def findbyname(name):
534 return '-'.join(nvr.split('-')[:-2]) == name
535 return list(filter(x, tree.pkgnames))
538 obsoletes = pkg.obsoletes()
542 for pn, setlist in obsoletes.items():
546 pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))