From: Marcin Krol Date: Sun, 2 May 2021 10:21:23 +0000 (+0200) Subject: - more python3 fixes, dropped python2 support X-Git-Url: https://git.tld-linux.org/?p=tld-builder.git;a=commitdiff_plain;h=9ea122b00e3a99b16246e1e38576916a2e25c0aa - more python3 fixes, dropped python2 support --- diff --git a/TLD_Builder/acl.py b/TLD_Builder/acl.py index 0d45634..88ed536 100644 --- a/TLD_Builder/acl.py +++ b/TLD_Builder/acl.py @@ -1,10 +1,7 @@ # vi: encoding=utf-8 ts=8 sts=4 sw=4 et import sys -if sys.version_info[0] == 2: - import ConfigParser -else: - import configparser as ConfigParser +import configparser as ConfigParser import string import fnmatch import os diff --git a/TLD_Builder/bqueue.py b/TLD_Builder/bqueue.py index d5b183f..131b911 100644 --- a/TLD_Builder/bqueue.py +++ b/TLD_Builder/bqueue.py @@ -4,14 +4,12 @@ import re import gzip import time import sys -if sys.version_info[0] == 2: - import StringIO -else: - from io import StringIO +from io import StringIO import os import fcntl import string import tempfile +import shutil # TLD_Builder: import gpg @@ -131,21 +129,26 @@ class B_Queue: sio = StringIO() self._write_to(sio) sio.seek(0) - sio.write(gpg.sign(sio.read())) + sio.write(util.to_str(gpg.sign(sio.read()))) sio.seek(0) (fdno, tmpname) = tempfile.mkstemp(dir=os.path.dirname(name)) f = os.fdopen(fdno, "w") - if re.search(r"\.gz$", name): - fgz = gzip.GzipFile(filename=name, mode="w", compresslevel=6, fileobj=f) - util.sendfile(sio, fgz) - fgz.close() - else: - util.sendfile(sio, f) + util.sendfile(sio, f) f.flush() os.fsync(f.fileno()) f.close() - os.chmod(tmpname, 0o0644) - os.rename(tmpname, name) + if re.search(r"\.gz$", name): + f_in = open(tmpname, 'rb') + f_out = gzip.open(name, 'wb') + shutil.copyfileobj(f_in, f_out) + f_in.close(); + f_out.flush(); + os.fsync(f_out.fileno()) + f_out.close(); + os.remove(tmpname) + else: + os.rename(tmpname, name) + os.chmod(name, 0o0644) def add(self, req): self.requests.append(req) diff --git a/TLD_Builder/chroot.py b/TLD_Builder/chroot.py index f91e1c6..8acd58d 100644 --- a/TLD_Builder/chroot.py +++ b/TLD_Builder/chroot.py @@ -3,6 +3,7 @@ import os import re import random +import util try: from hashlib import md5 as md5 @@ -49,7 +50,7 @@ def run(cmd, user = "builder", logfile = None, logstdout = None): def cp(file, outfile, user="builder", rm=False): m = md5() - m.update(str(random.sample(range(100000), 500)).encode('utf-8')) + m.update(util.to_bytes(str(random.sample(range(100000), 500)))) digest = m.hexdigest() marker_start = "--- FILE BEGIN DIGEST %s ---" % digest @@ -70,11 +71,11 @@ def cp(file, outfile, user="builder", rm=False): me = l.find(marker_end) if me != -1: l = l[:me] - f.write(l) + f.write(util.to_bytes(l)) marker = False break if marker: - f.write(l) + f.write(util.to_bytes(l)) rp = p.close() rf = f.close() if rp == None: diff --git a/TLD_Builder/config.py b/TLD_Builder/config.py index 11ea547..d8c42c4 100644 --- a/TLD_Builder/config.py +++ b/TLD_Builder/config.py @@ -1,10 +1,7 @@ # vi: encoding=utf-8 ts=8 sts=4 sw=4 et import sys -if sys.version_info[0] == 2: - import ConfigParser -else: - import configparser as ConfigParser +import configparser as ConfigParser import string import os import syslog diff --git a/TLD_Builder/gpg.py b/TLD_Builder/gpg.py index b820dcd..942cfb5 100644 --- a/TLD_Builder/gpg.py +++ b/TLD_Builder/gpg.py @@ -4,11 +4,7 @@ import log import subprocess import re import sys -if sys.version_info[0] == 2: - import StringIO -else: - from io import StringIO - +from io import StringIO import util import os import pipeutil @@ -27,7 +23,7 @@ def get_keys(buf): cmd = ['/usr/bin/gpg', '--batch', '--no-tty', '--decrypt'] gpg_run = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) try: - d_stdout, d_stderr = gpg_run.communicate(buf.decode().encode('utf-8')) + d_stdout, d_stderr = gpg_run.communicate(util.to_bytes(buf)) except OSError as e: log.error("gnupg run, does gpg binary exist? : %s" % e) raise @@ -35,7 +31,7 @@ def get_keys(buf): rx = re.compile("^gpg:.*using\s[DR]SA\skey\s(?:ID\s)?(\w+)") keys = [] - for l in d_stderr.decode().split('\n'): + for l in util.to_str(d_stderr).split('\n'): m = rx.match(l) if m: keys.append(m.group(1)) @@ -59,14 +55,14 @@ def verify_sig(buf): cmd = ['/usr/bin/gpg', '--batch', '--no-tty', '--decrypt'] gpg_run = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) try: - d_stdout, d_stderr = gpg_run.communicate(buf.decode().encode('utf-8')) + d_stdout, d_stderr = gpg_run.communicate(util.to_bytes(buf)) except OSError as e: log.error("gnupg run failed, does gpg binary exist? : %s" % e) raise rx = re.compile("^gpg: (Good signature from| aka) .*<([^>]+)>") emails = [] - for l in d_stderr.decode().split('\n'): + for l in util.to_str(d_stderr).split('\n'): m = rx.match(l) if m: emails.append(m.group(2)) @@ -85,12 +81,12 @@ def sign(buf): # TODO: check for gpg return code! gpg_run = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) try: - d_stdout, d_stderr = gpg_run.communicate(buf.decode().encode('utf-8')) + d_stdout, d_stderr = gpg_run.communicate(util.to_bytes(buf)) except OSError as e: log.error("gnupg signing failed, does gpg binary exist? : %s" % e) raise if len(d_stderr): - log.error("gpg: %s" % d_stderr.decode()) + log.error("gpg: %s" % util.to_str(d_stderr)) return d_stdout diff --git a/TLD_Builder/install.py b/TLD_Builder/install.py index d8badcc..7242ce4 100644 --- a/TLD_Builder/install.py +++ b/TLD_Builder/install.py @@ -3,11 +3,7 @@ import re, os import string import sys -if sys.version_info[0] == 2: - import StringIO -else: - from io import StringIO - +from io import StringIO import chroot import util import log diff --git a/TLD_Builder/mailer.py b/TLD_Builder/mailer.py index 7725a60..f76a8bd 100644 --- a/TLD_Builder/mailer.py +++ b/TLD_Builder/mailer.py @@ -3,11 +3,7 @@ import time import os import sys -if sys.version_info[0] == 2: - import StringIO -else: - from io import StringIO - +from io import StringIO from config import config import util import log diff --git a/TLD_Builder/notify.py b/TLD_Builder/notify.py index e13c052..71b269c 100644 --- a/TLD_Builder/notify.py +++ b/TLD_Builder/notify.py @@ -1,11 +1,7 @@ # vi: encoding=utf-8 ts=8 sts=4 sw=4 et import sys -if sys.version_info[0] == 2: - import StringIO -else: - from io import StringIO - +from io import StringIO import mailer import gpg import util @@ -22,7 +18,7 @@ class Notifier: sio = StringIO() self.xml.write("\n") self.xml.seek(0) - sio.write(gpg.sign(self.xml.read())) + sio.write(util.to_str(gpg.sign(self.xml.read()))) self.xml = None sio.seek(0) notifyq.init(r) diff --git a/TLD_Builder/pipeutil.py b/TLD_Builder/pipeutil.py index a3e78c9..fb6434c 100644 --- a/TLD_Builder/pipeutil.py +++ b/TLD_Builder/pipeutil.py @@ -3,10 +3,7 @@ import select import os import sys -if sys.version_info[0] == 2: - import StringIO -else: - from io import StringIO +from io import StringIO def rw_pipe(buf_, infd, outfd): buf = StringIO() diff --git a/TLD_Builder/request.py b/TLD_Builder/request.py index 7638bf2..25ca994 100644 --- a/TLD_Builder/request.py +++ b/TLD_Builder/request.py @@ -8,7 +8,7 @@ import xml.sax.saxutils import fnmatch import os import urllib -import cgi +import html import pytz import tempfile import subprocess @@ -333,15 +333,15 @@ class Batch: r = subprocess.call(cmd, shell=True) if r == 0: package_url = "http://git.tld-linux.org/?p=packages/%(package)s.git;a=blob;f=%(spec)s;hb=%(branch)s" % { - 'spec': urllib.quote(self.spec), - 'branch': urllib.quote(self.branch), - 'package': urllib.quote(self.spec[:-5]), + 'spec': urllib.parse.quote(self.spec), + 'branch': urllib.parse.quote(self.branch), + 'package': urllib.parse.quote(self.spec[:-5]), } else: package_url = "http://git.pld-linux.org/gitweb.cgi?p=packages/%(package)s.git;f=%(spec)s;h=%(branch)s;a=shortlog" % { - 'spec': urllib.quote(self.spec), - 'branch': urllib.quote(self.branch), - 'package': urllib.quote(self.spec[:-5]), + 'spec': urllib.parse.quote(self.spec), + 'branch': urllib.parse.quote(self.branch), + 'package': urllib.parse.quote(self.spec[:-5]), } desc = "%(src_rpm)s (%(spec)s -r %(branch)s%(rpmopts)s)" % { 'src_rpm': self.src_rpm, @@ -380,7 +380,7 @@ class Batch: tree_name = '-'.join(bld[:-1]) tree_arch = '-'.join(bld[-1:]) link_pre = "" \ - % (urllib.quote(tree_name), urllib.quote(tree_arch), urllib.quote(bl_name), urllib.quote(rid)) + % (urllib.parse.quote(tree_name), urllib.parse.quote(tree_arch), urllib.parse.quote(bl_name), urllib.parse.quote(rid)) link_post = "" def ftime(s): @@ -399,7 +399,7 @@ class Batch: 'color' : c, 'builder' : b, 'status' : s, - 'tooltip' : cgi.escape(tooltip, True), + 'tooltip' : html.escape(tooltip, True), } + link_post) f.write("%s]\n" % ' '.join(builders)) diff --git a/TLD_Builder/request_fetcher.py b/TLD_Builder/request_fetcher.py index fc9f49d..25b130f 100644 --- a/TLD_Builder/request_fetcher.py +++ b/TLD_Builder/request_fetcher.py @@ -6,12 +6,8 @@ import os import urllib import urllib2 import sys -if sys.version_info[0] == 2: - import StringIO -else: - from io import StringIO +from io import StringIO import gzip - import path import log import status diff --git a/TLD_Builder/request_handler.py b/TLD_Builder/request_handler.py index de48d9e..70a1866 100644 --- a/TLD_Builder/request_handler.py +++ b/TLD_Builder/request_handler.py @@ -5,12 +5,7 @@ import string import time import os import sys -if sys.version_info[0] == 2: - import StringIO -else: - from io import StringIO import fnmatch - import gpg import request import log diff --git a/TLD_Builder/rpm_builder.py b/TLD_Builder/rpm_builder.py index 239f5a3..e33a7e1 100644 --- a/TLD_Builder/rpm_builder.py +++ b/TLD_Builder/rpm_builder.py @@ -85,7 +85,7 @@ def check_skip_build(r, b): return False def fetch_src(r, b): - src_url = config.control_url + "/srpms/" + r.id + "/" + urllib.quote(b.src_rpm) + src_url = config.control_url + "/srpms/" + r.id + "/" + urllib.parse.quote(b.src_rpm) b.log_line("fetching %s" % src_url) start = time.time() good = False @@ -126,7 +126,7 @@ def fetch_src(r, b): print("error.reason exception %s" % e) raise - o = chroot.popen("cat > %s" % b.src_rpm, mode = "w") + o = chroot.popen("cat > %s" % b.src_rpm, mode = "wb") try: bytes = util.sendfile(f, o) diff --git a/TLD_Builder/srpm_builder.py b/TLD_Builder/srpm_builder.py index 11f793b..510b1f7 100644 --- a/TLD_Builder/srpm_builder.py +++ b/TLD_Builder/srpm_builder.py @@ -5,10 +5,6 @@ import string import time import os import sys -if sys.version_info[0] == 2: - import StringIO -else: - from io import StringIO import re import shutil import atexit diff --git a/TLD_Builder/util.py b/TLD_Builder/util.py index 05cf076..1d485a1 100644 --- a/TLD_Builder/util.py +++ b/TLD_Builder/util.py @@ -5,6 +5,7 @@ import sys import os import log import string +import codecs def uuid_python(): return str(uuid_random()) @@ -95,3 +96,19 @@ def cmp_to_key(mycmp): def __ne__(self, other): return mycmp(self.obj, other.obj) != 0 return K + +def to_bytes(s): + if type(s) is bytes: + return s + elif type(s) is str or (sys.version_info[0] < 3 and type(s) is unicode): + return codecs.encode(s, 'utf-8') + else: + raise TypeError("Expected bytes or string, but got %s." % type(s)) + +def to_str(s): + if type(s) is bytes: + return codecs.decode(s, 'utf-8') + elif type(s) is str or (sys.version_info[0] < 3 and type(s) is unicode): + return s + else: + raise TypeError("Expected bytes or string, but got %s." % type(s)) diff --git a/TLD_Builder/wrap.py b/TLD_Builder/wrap.py index 3d7c4a8..e9a24e0 100644 --- a/TLD_Builder/wrap.py +++ b/TLD_Builder/wrap.py @@ -4,10 +4,7 @@ import sys import log import traceback import sys -if sys.version_info[0] == 2: - import StringIO -else: - from io import StringIO +from io import StringIO import os import time