]> TLD Linux GIT Repositories - tld-ftp-admin.git/blob - modules/ftptree.py
0266342aba810229170991c5215869b044616c1d
[tld-ftp-admin.git] / modules / ftptree.py
1 # vi: encoding=utf-8 ts=8 sts=4 sw=4 et
2
3 from __future__ import print_function
4
5 import os
6 import config
7 import string
8 try:
9     import urllib.request as urlmess
10 except ImportError:
11     import urllib as urlmess
12     pass
13 import re
14 import rpm
15 from common import fileexists, noarchcachedir
16 from baseftptree import BasePkg, BaseFtpTree
17 from sign import is_signed
18
19 errnum = 0
20 quietmode = False
21
22 class SomeError(Exception):
23     def __init__(self):
24         return
25
26     def __str__(self):
27         return "An Error occured!"
28
29 def bailoutonerror():
30     if not errnum == 0:
31         print("%d error(s) encountered... aborting" % errnum)
32         raise SomeError()
33
34 def pinfo(msg):
35     print('INFO: ' + msg)
36
37 def perror(msg):
38     global errnum
39     errnum = errnum + 1
40     print('ERR: ' + msg)
41
42 def pwarning(msg):
43     print('WARN: ' + msg)
44
45 def rm(file, test = False):
46     if test:
47         if not os.path.exists(file):
48             pinfo("TEST os.remove(%s): file doesn't exists" % file)
49     else:
50         try:
51             os.remove(file)
52         except OSError as e:
53             pinfo("os.remove(%s): %s" % (file, e))
54             #raise
55
56 def mv(src, dst, test = False):
57     fsrc = src
58     fdst = dst + '/' + src.split('/')[-1]
59     if test:
60         if not os.path.exists(fsrc):
61             pinfo("TEST os.rename(%s, %s): source doesn't exists" % (fsrc, fdst))
62         if not os.path.exists(dst):
63             pinfo("TEST destination doesn't exist: %s" % dst)
64     else:
65         try:
66             os.rename(fsrc, fdst)
67         except OSError as e:
68             pinfo("os.rename(%s, %s): %s" % (fsrc, fdst, e))
69             raise
70
71 class Pkg(BasePkg):
72     def __init__(self, nvr, tree):
73         BasePkg.__init__(self, nvr, tree)
74         self.name = '-'.join(nvr.split('-')[:-2])
75         self.version = nvr.split('-')[-2]
76         self.release = nvr.split('-')[-1]
77         self.marked4removal = False
78         self.marked4moving = False
79         self.marked4movingpool = []
80         self.errors = []
81         self.warnings = []
82
83     def __cmp__(self, pkg):
84         if self.name > pkg.name:
85             return 1
86         elif self.name < pkg.name:
87             return -1
88         else:
89             return rpm.labelCompare(('0', self.version, self.release),
90                                     ('0', pkg.version, pkg.release))
91
92
93     # unfortunately can't do new Pkg(NVR), and have no "tree" in this pkg context
94     # so this static function
95     def is_debuginfo(self, nvr):
96         """
97         returns true if NVR is debuginfo package and separate debuginfo is enabled
98         """
99         if not config.separate_debuginfo:
100             return False
101         pkg = nvr.split('-')[:-2]
102         return pkg[-1] == 'debuginfo' or pkg[-1] == 'debugsource'
103
104     def is_sourcefile(self, file):
105         """
106         returns true if file is source package
107         """
108         return file[-8:] == '.src.rpm'
109
110     def mark4moving(self):
111         if not self.marked4moving:
112             # Only one pkg in this pool can be marked for moving
113             for pkg in self.marked4movingpool:
114                 pkg.unmark4moving()
115             self.tree.marked4moving.append(self)
116             self.marked4moving=True
117
118     def unmark4moving(self):
119         if self.marked4moving:
120             self.tree.marked4moving.remove(self)
121             self.marked4moving=False
122
123     def mark4removal(self):
124         if not self.marked4removal:
125             self.tree.marked4removal.append(self)
126             self.marked4removal=True
127
128     def error(self, msg):
129         self.errors.append(msg)
130         if not quietmode:
131             perror('%s %s' % (self.nvr, msg))
132
133     def warning(self, msg):
134         self.warnings.append(msg)
135         if not quietmode:
136             pwarning('%s %s' % (self.nvr, msg))
137
138     def load(self, content=None):
139         BasePkg.load(self, content)
140         if 'move' in self.info:
141             self.mark4moving()
142
143     def writeinfo(self):
144         f = open(self.tree.basedir+'/SRPMS/.metadata/'+self.nvr+'.src.rpm.info', 'w')
145         for bid in self.build.keys():
146             f.write("info:build:%s:requester:%s\ninfo:build:%s:requester_email:%s\n" % (bid, self.build[bid].requester, bid, self.build[bid].requester_email))
147         for key in self.info.keys():
148             f.write("info:%s:%s\n" % (key, ':'.join(self.info[key])))
149         for arch in self.files.keys():
150             for rpm in self.files[arch]:
151                 f.write("file:%s:%s\n" % (arch, rpm))
152
153     def remove(self, test = False):
154         """
155         Remove package from ftp
156         """
157         for arch in self.files.keys():
158             for rpm in self.files[arch]:
159                 if self.is_debuginfo(rpm):
160                     rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
161                 else:
162                     rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
163                 if arch == 'noarch':
164                     if fileexists(noarchcachedir + rpm + '.filelist'):
165                         rm(noarchcachedir + rpm + '.filelist', test)
166                     if fileexists(noarchcachedir + rpm + '.reqlist'):
167                         rm(noarchcachedir + rpm + '.reqlist', test)
168         rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
169
170     def rpmfiles(self, debugfiles = True, sourcefiles  = True):
171         """
172         Return rpm files related to this package
173         """
174         files = []
175         for arch, rpms in self.files.items():
176             for nvr in rpms:
177                 if self.is_debuginfo(nvr):
178                     if debugfiles:
179                         files.append(self.tree.basedir + '/' + arch + '/debuginfo/' + nvr)
180                 else:
181                     if self.is_sourcefile(nvr):
182                         if sourcefiles:
183                             files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
184                     else:
185                         files.append(self.tree.basedir + '/' + arch + '/RPMS/' + nvr)
186         return files
187
188     def obsoletes(self):
189         """
190         Return obsoletes for all packages in Pkg:
191
192         {'php-geshi': set(['geshi'])}
193
194         """
195         def rpmhdr(pkg):
196             ts = rpm.ts()
197             ts.setVSFlags(rpm.RPMVSF_NODSAHEADER)
198             fdno = os.open(pkg, os.O_RDONLY)
199             hdr = ts.hdrFromFdno(fdno)
200             os.close(fdno)
201             return hdr
202
203         obsoletes = {}
204         for rpmfile in self.rpmfiles():
205             if not os.path.exists(rpmfile):
206                 continue
207             hdr = rpmhdr(rpmfile)
208             if not hdr[rpm.RPMTAG_OBSOLETES]:
209                 continue
210
211             name = hdr[rpm.RPMTAG_NAME]
212             if not name in obsoletes:
213                 obsoletes[name] = set()
214
215             for tag in hdr[rpm.RPMTAG_OBSOLETES]:
216                 obsoletes[name].add(tag)
217
218         return obsoletes
219
220     def move(self, dsttree, test = False):
221         if dsttree.has_key(self.nvr):
222             movedany = False
223             for arch in self.files.keys():
224                 if arch in dsttree[self.nvr].files.keys():
225                     msg = ""
226                     if test:
227                         msg = "TEST "
228                     pinfo("%sArch %s for %s is already present in dest tree; removing from srctree" % (msg, arch, self.nvr))
229                     for rpm in self.files[arch]:
230                         if self.is_debuginfo(rpm):
231                             rm(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, test)
232                         else:
233                             rm(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, test)
234                 else:
235                     movedany = True
236                     dsttree[self.nvr].files[arch] = self.files[arch]
237                     for rpm in self.files[arch]:
238                         if self.is_debuginfo(rpm):
239                             mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
240                         else:
241                             mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
242             if not test and movedany:
243                 for bid in self.build.keys():
244                     dsttree[self.nvr].build[bid] = self.build[bid]
245                 dsttree[self.nvr].writeinfo()
246             rm(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', test)
247         else:
248             # move files
249             for arch in self.files.keys():
250                 for rpm in self.files[arch]:
251                     if self.is_debuginfo(rpm):
252                         mv(self.tree.basedir + '/' + arch + '/debuginfo/' + rpm, dsttree.basedir + '/' + arch + '/debuginfo/', test)
253                     else:
254                         mv(self.tree.basedir + '/' + arch + '/RPMS/' + rpm, dsttree.basedir + '/' + arch + '/RPMS/', test)
255
256             # move metadata
257             mv(self.tree.basedir + '/SRPMS/.metadata/' + self.nvr + '.src.rpm.info', dsttree.basedir + '/SRPMS/.metadata/', test)
258
259 class FtpTree(BaseFtpTree):
260     def __init__(self, tree, loadall=False):
261         BaseFtpTree.__init__(self, tree)
262         self.loadedpkgs = {}
263         self.marked4removal = []
264         self.marked4moving = []
265         self.pkgnames = []
266         self.__loadpkgnames()
267         if loadall:
268             for pkgname in self.pkgnames:
269                 self.loadedpkgs[pkgname] = Pkg(pkgname, self)
270         # Tests:
271         self.do_checkbuild = True
272
273     def __getitem__(self, key):
274         if key in self.loadedpkgs:
275             return self.loadedpkgs[key]
276         elif key in self.pkgnames:
277             pkg=Pkg(key, self)
278             self.loadedpkgs[key]=pkg
279             return pkg
280         else:
281             raise KeyError(key)
282
283     def has_key(self, key):
284         if key in self.pkgnames:
285             return True
286         else:
287             return False
288
289     def keys(self):
290         return self.pkgnames
291
292     def values(self):
293         return self.loadedpkgs.values()
294
295     def checktree(self, dsttree):
296         self.__checkbuild(self.loadedpkgs.values())
297         self.__checkarchs(dsttree, self.loadedpkgs.values())
298
299     def testmove(self, dsttree, archivetree = None):
300         self.__checkbuild(self.marked4moving)
301         self.__checkarchs(dsttree, self.marked4moving)
302         if not self.treename.count("archive"):
303             self.__checkduplicates(self.marked4moving)
304
305         self.__checksigns(dsttree, self.marked4moving, test = True)
306         self.__checkforobsoletes(dsttree, self.marked4moving, test = True)
307
308         if not self.treename.count("archive"):
309             self.__rmolderfromsrc(test = True)
310             self.__rmotherfromdst(dsttree, test = True, archivetree = archivetree)
311
312         for pkg in self.marked4moving:
313             pkg.move(dsttree, test = True)
314
315     def movepkgs(self, dsttree, archivetree = None):
316         if self.do_checkbuild:
317             self.__checkbuild(self.marked4moving)
318         bailoutonerror()
319
320         self.__checkarchs(dsttree, self.marked4moving)
321         bailoutonerror()
322
323         self.__checksigns(dsttree, self.marked4moving)
324         bailoutonerror()
325
326         if not self.treename.count("archive"):
327             self.__rmolderfromsrc()
328             self.__rmotherfromdst(dsttree, archivetree = archivetree)
329
330         for pkg in self.marked4moving:
331             pkg.move(dsttree)
332
333     def rpmfiles(self, debugfiles = True, sourcefiles = True):
334         if self.do_checkbuild:
335             self.__checkbuild(self.marked4moving)
336
337         files = []
338         for pkg in self.marked4moving:
339             files += pkg.rpmfiles(debugfiles = debugfiles, sourcefiles = sourcefiles)
340         return files
341
342     def removepkgs(self):
343         if self.do_checkbuild:
344             self.__checkbuild(self.marked4removal)
345         bailoutonerror()
346         for pkg in self.marked4removal:
347             pkg.remove()
348
349     def mark4removal(self, wannabepkgs):
350         self.__mark4something(wannabepkgs, Pkg.mark4removal)
351
352     def mark4moving(self, wannabepkgs):
353         self.__mark4something(wannabepkgs, Pkg.mark4moving)
354
355     # Internal functions below
356     def __arch_stringify(self, list):
357         ret = []
358         dist = config.ftp_dist;
359         for arch in list:
360             ret.append(dist + '-' + arch)
361         return ' '.join(ret)
362
363     def __loadpkgnames(self):
364         def checkfiletype(name):
365             if name[-13:]=='.src.rpm.info':
366                 return True
367             else:
368                 return False
369         pkglist = list(filter(checkfiletype, os.listdir(self.basedir+'/SRPMS/.metadata')))
370         self.pkgnames = list(map((lambda x: x[:-13]), pkglist))
371
372     def __mark4something(self, wannabepkgs, markfunction):
373         def chopoffextension(pkg):
374             found = pkg.find('.src.rpm')
375             if found == -1:
376                 return pkg
377             else:
378                 return pkg[:found]
379
380         for wannabepkg in wannabepkgs:
381             pkgname = chopoffextension(wannabepkg)
382             if pkgname in self.pkgnames:
383                 if not pkgname in self.loadedpkgs.keys():
384                     self.loadedpkgs[pkgname]=Pkg(pkgname, self)
385                 markfunction(self.loadedpkgs[pkgname])
386             else:
387                 perror('%s not found in source tree' % pkgname)
388         bailoutonerror()
389
390     def __checkbuild(self, marked):
391         """
392         Checks queue file if all arches are built
393
394         Reads config.builderqueue to grab the info
395         """
396         f = urlmess.urlopen(config.builderqueue)
397         requests = {}
398         reid = re.compile(r'^.*id=(.*) pri.*$')
399         regb = re.compile(r'^group:.*$|builders:.*$', re.M)
400         for i in re.findall(regb, f.read().decode('utf-8')):
401             if i[0] == 'g':
402                 id = reid.sub(r'\1', i)
403                 requests[id] = ""
404             elif i[0]=='b':
405                 requests[id] = requests[id] + i
406         f.close()
407
408         for pkg in marked:
409             for bid in pkg.build.keys():
410                 if bid in requests and not requests[bid].find('?') == -1:
411                     pkg.error("(buildid %s) building not finished" % bid)
412
413     def __checkarchs(self, dsttree, marked):
414         """
415         Checks marked pkgs it is built on all archs.
416         """
417         for pkg in marked:
418             if len(pkg.files.keys()) <= 1:
419                 pkg.error('has only src.rpm built')
420                 continue
421             otherpkgnames = self.__find_other_pkgs(pkg, dsttree)
422
423             # check if we're not removing some archs
424             if otherpkgnames:
425                 curarchs = []
426                 missingarchs = []
427                 for somepkg in otherpkgnames:
428                     curarchs.extend(Pkg(somepkg, dsttree).files.keys())
429                 for arch in curarchs:
430                     if arch not in pkg.files.keys():
431                         missingarchs.append(arch)
432                 if missingarchs:
433                     pkg.error('moving would remove archs: %s' % self.__arch_stringify(missingarchs))
434             else:
435                 # warn if a package isn't built for all archs
436                 # ftp_archs + SRPMS
437                 ftp_archs_num = len(config.ftp_archs) + 1
438                 if (config.separate_noarch and 'noarch' in pkg.files.keys()):
439                     # ftp_archs + SRPMS + noarch subpackages
440                     ftp_archs_num += 1
441                     # plain simple noarch package
442                     if (len(pkg.files.keys()) == 2):
443                         continue
444
445                 if len(pkg.files.keys()) != ftp_archs_num:
446                     missingarchs = []
447                     for arch in config.ftp_archs:
448                         if arch not in pkg.files.keys():
449                             missingarchs.append(arch)
450                     pkg.warning('not built for archs: %s' % self.__arch_stringify(missingarchs))
451
452     def __checkduplicates(self, marked):
453         """
454         Checks if marked packages contain duplicate packages (with different versions)
455         """
456         for pkg in marked:
457             olderpkgnames = self.__find_older_pkgs(pkg)
458             for i in olderpkgnames:
459                 markednames = [str(x) for x in marked]
460                 if i in markednames:
461                     pkg.error('duplicate package: %s' % i)
462
463     def __rmolderfromsrc(self, test = False):
464         for pkg in self.marked4moving:
465             olderpkgnames = self.__find_older_pkgs(pkg)
466             for i in olderpkgnames:
467                 Pkg(i, self).remove(test)
468
469     def __rmotherfromdst(self, dsttree, test = False, archivetree = None):
470         for pkg in self.marked4moving:
471             pkgnames = self.__find_other_pkgs(pkg, dsttree)
472             for i in pkgnames:
473                 if archivetree == None:
474                     Pkg(i, dsttree).remove(test)
475                 else:
476                     Pkg(i, dsttree).move(archivetree, test = test)
477
478     # Used more than once filter functions
479     def __find_other_pkgs(self, pkg, tree):
480         escapedpkgname = pkg.name.replace('.', '\.').replace('+', '\+')
481         ziewre = re.compile(escapedpkgname + '-[^-]*-[^-]*$')
482         def filter_other_pkgs(x):
483             if ziewre.match(x) and not x == pkg.nvr:
484                 return True
485             else:
486                 return False
487         return list(filter(filter_other_pkgs, tree.pkgnames))
488
489     def __find_older_pkgs(self, pkg):
490         def filter_older_pkgs(x):
491             c = x.split('-')
492             rc = rpm.labelCompare(('0', pkg.version, pkg.release),
493                                                         ('0', c[-2], c[-1]))
494             if rc == 1: # pkg > x
495                 return True
496             else:
497                 return False
498         return list(filter(filter_older_pkgs, self.__find_other_pkgs(pkg, self)))
499
500     def __checksigns(self, tree, pkgs, test = False):
501         """
502         Checks if pkgs in tree are all signed.
503
504         in case of test = true, error flag is set for unsigned packages
505         """
506         if not tree.treename in config.signed_trees:
507             return
508
509         for pkg in pkgs:
510             unsigned = 0
511             for file in pkg.rpmfiles():
512                 if not is_signed(file):
513                     unsigned += 1
514
515             if unsigned != 0:
516                 if test == True:
517                     if not quietmode:
518                         pkg.warning('%d files not signed' % unsigned)
519                 else:
520                     pkg.error('%d files not signed' % unsigned)
521
522     def __checkforobsoletes(self, tree, pkgs, test = False):
523         """
524         Checks queue file if package obsoletes something in destination tree and suggest for removal.
525
526         Only NAME tag is compared, i.e virtual packages do not get reported.
527
528         """
529         if test != True:
530             return
531
532         def findbyname(name):
533             def x(nvr):
534                 return '-'.join(nvr.split('-')[:-2]) == name
535             return list(filter(x, tree.pkgnames))
536
537         for pkg in pkgs:
538             obsoletes = pkg.obsoletes()
539             if not obsoletes:
540                 continue
541
542             for pn, setlist in obsoletes.items():
543                 for item in setlist:
544                     p = findbyname(item)
545                     if p:
546                         pkg.warning('obsoletes %s (via %s) in dest tree, perhaps you want rmpkg' % (p,pn))