From 45d8f2d9258214696ed2d3bba02ceb0236d47fd3 Mon Sep 17 00:00:00 2001 From: Marcin Krol Date: Sat, 27 Feb 2021 14:41:56 +0100 Subject: [PATCH] - PLD merge --- builder.sh | 42 +++++++++-- rediff-patches.py | 169 +++++++++++++++++++++++++++++++++++++++++++ relup.sh | 15 +++- rpm-build-tools.spec | 2 +- sort-pkgs | 80 ++++++++++++++++++++ 5 files changed, 295 insertions(+), 13 deletions(-) create mode 100755 rediff-patches.py create mode 100755 sort-pkgs diff --git a/builder.sh b/builder.sh index bbeb418..a19da12 100755 --- a/builder.sh +++ b/builder.sh @@ -38,9 +38,10 @@ APPDIR=$(d=$0; [ -L "$d" ] && d=$(readlink -f "$d"); dirname "$d") VERSION="v0.35" VERSIONSTRING="\ Build package utility from PLD Linux Packages repository -$VERSION (C) 1999-2016 Free Penguins". +$VERSION (C) 1999-2020 Free Penguins". -CLEAN_PATH="/bin:/usr/bin:/usr/sbin:/sbin:/usr/X11R6/bin" +# Clean PATH without /usr/local or user paths +CLEAN_PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/X11R6/bin" # required rpm-build-macros RPM_MACROS_VER=1.534 @@ -496,6 +497,26 @@ Usage: builder [--all-branches] [-D|--debug] [-V|--version] [--short-version] [ " } +is_rpmorg() { + local v + + v=$(LC_ALL=C LANG=C rpm --version 2>&1) + v=${v#RPM version } # rpm 4 + v=${v#rpm \(RPM\) } # rpm 5 + + case "$v" in + 4.5|5.*) + return 1 + ;; + 4.*) + return 0; + ;; + *) + echo "ERROR: unsupported RPM version $v" >&2 + exit 1 + esac +} + # create tempfile. as secure as possible tempfile() { local prefix=builder.$PACKAGE_NAME${1:+.$1} @@ -709,7 +730,8 @@ EOF %_sourcedir ./ EOF fi - if rpm --version 2>&1 | grep -qE '5\.[0-9]+\.[0-9]+'; then + if ! is_rpmorg; then + local safe_macrofiles safe_macrofiles=$(rpm $TARGET_SWITCH --showrc | awk -F: '/^macrofiles/ { gsub(/^macrofiles[ \t]+:/, "", $0); print $0 } ') eval PATH=$CLEAN_PATH $RPMBUILD $TARGET_SWITCH --macros "$safe_macrofiles:$BUILDER_MACROS" $QUIET $RPMOPTS $RPMBUILDOPTS $BCOND $* 2>&1 else @@ -1079,7 +1101,10 @@ get_spec() { local _rev=$(get_pkgrev "$CVSTAG") echo "$_rev" | grep -q -E "^ERROR$" || CVSTAG="$_rev" if git rev-parse --verify -q "$CVSTAG" >/dev/null; then - git checkout "$CVSTAG" -- + # checkout only if differs, so this will not trash git reflog + if [ $(git rev-parse "$CVSTAG") != $(git rev-parse HEAD) ]; then + git checkout "$CVSTAG" -- + fi elif git rev-parse --verify -q "refs/remotes/${REMOTE_PLD}/$CVSTAG"; then git checkout -t "refs/remotes/${REMOTE_PLD}/$CVSTAG" > /dev/null fi @@ -1386,6 +1411,7 @@ get_files() { ${GETURI} "$target" "$url" fi + if [ -s "$target" ]; then cvsignore_df $target else @@ -1847,7 +1873,7 @@ set_bconds_values() { without_*) bcond=${opt#without_} case "$BCOND" in - *--without?${bcond}*) + *--without?${bcond}\ *|*--without?${bcond}) AVAIL_BCONDS_WITHOUT="$AVAIL_BCONDS_WITHOUT <$bcond>" ;; *) @@ -1858,7 +1884,7 @@ set_bconds_values() { with_*) bcond=${opt#with_} case "$BCOND" in - *--with?${bcond}*) + *--with?${bcond}\ *|*--with?${bcond}) AVAIL_BCONDS_WITH="$AVAIL_BCONDS_WITH <$bcond>" ;; *) @@ -1915,7 +1941,7 @@ run_sub_builder() { # this requires following sudo rules: # - poldek --noask --caplookup -ug poldek_install() { - LANG=C $POLDEK_CMD --noask --caplookup --uniq -ug "$@" + LC_ALL=C LANG=C $POLDEK_CMD --noask --caplookup --uniq -ug "$@" } # install packages @@ -2044,7 +2070,7 @@ _rpm_cnfl_check() { DEPS=$(cat) fi - LANG=C rpm -q --whatprovides $DEPS 2>/dev/null | awk '!/no package provides/ { print }' + LC_ALL=C LANG=C rpm -q --whatprovides $DEPS 2>/dev/null | awk '!/no package provides/ { print }' } # install deps via information from 'rpm-getdeps' or 'rpm --specsrpm' diff --git a/rediff-patches.py b/rediff-patches.py new file mode 100755 index 0000000..42897f4 --- /dev/null +++ b/rediff-patches.py @@ -0,0 +1,169 @@ +#!/usr/bin/python3 + +# rediff-patches.py name.spec + +import argparse +import collections +import logging +import os +import re +import rpm +import shutil +import subprocess +import sys +import tempfile + +RPMBUILD_ISPATCH = (1<<1) + +def prepare_spec(r, patch_nr, before=False): + tempspec = tempfile.NamedTemporaryFile() + re_patch = re.compile(r'^%patch(?P\d+)\w*') + for line in r.parsed.split('\n'): + m = re_patch.match(line) + if m: + patch_number = int(m.group('patch_number')) + if patch_nr == patch_number: + if before: + tempspec.write(b"exit 0\n# here was patch%d\n" % patch_nr) + else: + line = re.sub(r'#.*', "", line) + tempspec.write(b"%s\nexit 0\n" % line.encode('utf-8')) + continue + tempspec.write(b"%s\n" % line.encode('utf-8')) + tempspec.flush() + return tempspec + +def unpack(spec, appsourcedir, builddir): + cmd = [ 'rpmbuild', '-bp', + '--define', '_builddir %s' % builddir, + '--define', '_specdir %s' % appsourcedir, + '--define', '_sourcedir %s' % appsourcedir, + '--define', '_enable_debug_packages 0', + '--define', '_default_patch_fuzz 2', + spec ] + logging.debug("running %s" % repr(cmd)) + try: + res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True, + env={'LC_ALL': 'C.UTF-8'}, timeout=600) + except subprocess.CalledProcessError as err: + logging.error("unpacking exited with status code %d." % err.returncode) + logging.error("STDOUT:") + if err.stdout: + for line in err.stdout.decode('utf-8').split("\n"): + logging.error(line) + logging.error("STDERR:") + if err.stderr: + for line in err.stderr.decode('utf-8').split("\n"): + logging.error(line) + raise + else: + logging.debug("unpacking exited with status code %d." % res.returncode) + logging.debug("STDOUT/STDERR:") + if res.stdout: + for line in res.stdout.decode('utf-8').split("\n"): + logging.debug(line) + + +def diff(diffdir_org, diffdir, builddir, output): + diffdir_org = os.path.basename(diffdir_org) + diffdir = os.path.basename(diffdir) + + with open(output, 'wt') as f: + cmd = [ 'diff', '-urNp', '-x', '*.orig', diffdir_org, diffdir ] + logging.debug("running %s" % repr(cmd)) + try: + subprocess.check_call(cmd, cwd=builddir, stdout=f, stderr=sys.stderr, + env={'LC_ALL': 'C.UTF-8'}, timeout=600) + except subprocess.CalledProcessError as err: + if err.returncode != 1: + raise + logging.info("rediff generated as %s" % output) + +def diffstat(patch): + cmd = [ 'diffstat', patch ] + logging.info("running diffstat for: %s" % patch) + try: + subprocess.check_call(cmd, stdout=sys.stdout, stderr=sys.stderr, + env={'LC_ALL': 'C.UTF-8'}, timeout=60) + except subprocess.CalledProcessError as err: + logging.error("running diffstat failed: %s" % err) + except FileNotFoundError as err: + logging.error("running diffstat failed: %s, install diffstat package?" % err) + +def main(): + parser = parser = argparse.ArgumentParser(description='rediff patches to avoid fuzzy hunks') + parser.add_argument('spec', type=str, help='spec file name') + parser.add_argument('-p', '--patches', type=str, help='comma separated list of patch numbers to rediff') + parser.add_argument('-s', '--skip-patches', type=str, help='comma separated list of patch numbers to skip rediff') + parser.add_argument('-v', '--verbose', help='increase output verbosity', action='store_true') + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + rpm.setVerbosity(rpm.RPMLOG_ERR) + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + rpm.setVerbosity(rpm.RPMLOG_DEBUG) + + if args.patches: + args.patches = [int(x) for x in args.patches.split(',')] + + if args.skip_patches: + args.skip_patches = [int(x) for x in args.skip_patches.split(',')] + + specfile = args.spec + appsourcedir = os.path.dirname(os.path.abspath(specfile)) + + tempdir = tempfile.TemporaryDirectory(dir="/dev/shm") + topdir = tempdir.name + builddir = os.path.join(topdir, 'BUILD') + + rpm.addMacro("_builddir", builddir) + + r = rpm.spec(specfile) + + patches = {} + + for (name, nr, flags) in r.sources: + if flags & RPMBUILD_ISPATCH: + patches[nr] = name + + applied_patches = collections.OrderedDict() + re_patch = re.compile(r'^%patch(?P\d+)\w*(?P.*)') + for line in r.parsed.split('\n'): + m = re_patch.match(line) + if not m: + continue + patch_nr = int(m.group('patch_number')) + patch_args = m.group('patch_args') + applied_patches[patch_nr] = patch_args + + appbuilddir = rpm.expandMacro("%{_builddir}/%{?buildsubdir}") + + for patch_nr in applied_patches.keys(): + if args.patches and patch_nr not in args.patches: + continue + if args.skip_patches and patch_nr in args.skip_patches: + continue + patch_name = patches[patch_nr] + logging.info("*** patch %d: %s" % (patch_nr, patch_name)) + + tempspec = prepare_spec(r, patch_nr, before=True) + unpack(tempspec.name, appsourcedir, builddir) + tempspec.close() + os.rename(appbuilddir, appbuilddir + ".org") + + tempspec = prepare_spec(r, patch_nr, before=False) + unpack(tempspec.name, appsourcedir, builddir) + tempspec.close() + + diff(appbuilddir + ".org", appbuilddir, builddir, os.path.join(topdir, os.path.join(appsourcedir, patch_name + ".rediff"))) + + diffstat(os.path.join(topdir, os.path.join(appsourcedir, patch_name))) + diffstat(os.path.join(topdir, os.path.join(appsourcedir, patch_name + ".rediff"))) + + shutil.rmtree(builddir) + tempdir.cleanup() + +if __name__ == '__main__': + main() diff --git a/relup.sh b/relup.sh index 7557ba6..8267436 100755 --- a/relup.sh +++ b/relup.sh @@ -8,12 +8,19 @@ set -e get_dump() { local specfile="$1" - if ! out=$(rpm --specfile "$specfile" --define 'prep %dump' -q 2>&1); then - echo >&2 "$out" + local rpm_dump + local success="y" + if [ -x /usr/bin/rpm-specdump ]; then + rpm_dump=$(rpm-specdump "$specfile" 2>&1) || success="n" + else + rpm_dump=$(rpm --specfile "$specfile" --define 'prep %dump' -q 2>&1) || success="n" + fi + if [ "$success" != "y" ]; then + echo >&2 "$rpm_dump" echo >&2 "You need icon files being present in SOURCES." exit 1 fi - echo "$out" + echo "$rpm_dump" } usage="Usage: @@ -38,7 +45,7 @@ get_release() { rel=$(awk '/^%define[ ]+_?rel[ ]+/{print $NF}' $specfile) if [ -z "$rel" ]; then dump=$(get_dump "$specfile") - rel=$(echo "$dump" | awk '/PACKAGE_RELEASE/{print $NF; exit}') + rel=$(echo "$dump" | awk '$2~/^(PACKAGE_)?RELEASE$/{print $NF; exit}') fi echo $rel } diff --git a/rpm-build-tools.spec b/rpm-build-tools.spec index edf0a6c..6b4485c 100644 --- a/rpm-build-tools.spec +++ b/rpm-build-tools.spec @@ -6,7 +6,7 @@ Summary(ru.UTF-8): Скрипты и утилиты, необходимые дл Summary(uk.UTF-8): Скрипти та утиліти, необхідні для побудови пакетів Name: rpm-build-tools Version: 4.9 -Release: 7 +Release: 12 License: GPL Group: Applications/File Source0: builder.sh diff --git a/sort-pkgs b/sort-pkgs new file mode 100755 index 0000000..7b1fa41 --- /dev/null +++ b/sort-pkgs @@ -0,0 +1,80 @@ +#!/usr/bin/python3 + +""" +This script tries to set ordering in which packages ought to be sent to builders. +Input: file with names of packages (without the .spec suffix). One package name per line. +Output: sorted packages on stdout. + +If the script goes in a infinite loop, that means there is a cycle or other bug. +""" + +import os +import re +import sys + +BR_PATTERN = re.compile('BuildRequires:\s+(.*?)(\s|$)') +PACKAGE_PATTERN_WITH_N = re.compile('%package\s+-n\s+(.*)') +PACKAGE_PATTERN = re.compile('%package\s+(.*)') + +DIR = os.getenv("HOME") + '/rpm/packages' + +BUILD_REQUIRES = {} +PACKAGES = {} +SPECS = {} +VISITED = {} + + +def parse_spec(name): + global PACKAGES, SPECS, BUILD_REQUIRES, VISITED + res = [] + try: + with open(os.path.join(DIR, name, name + '.spec'), 'r') as f: + for line in f: + br = BR_PATTERN.match(line) + if br: + p = br.group(1) + res.append(p) + if line.startswith('%package'): + pn = PACKAGE_PATTERN_WITH_N.match(line) + if pn: + package = pn.group(1) + PACKAGES[package] = name + else: + pn = PACKAGE_PATTERN.match(line) + if pn: + ext = pn.group(1) + if ext: + package = name + '-' + ext + PACKAGES[package] = name + BUILD_REQUIRES[name] = res[:] + PACKAGES[name] = name + SPECS[name] = True + VISITED[name] = False + except: + pass + + +def print_spec(spec): + global PACKAGES, SPECS, BUILD_REQUIRES, VISITED + + if not VISITED[spec]: + VISITED[spec] = True + for br in BUILD_REQUIRES[spec]: + name = PACKAGES.get(br, '') + if name in SPECS: + if not VISITED[name]: + print_spec(name) + print(spec) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: %s filename" % sys.argv[0]) + sys.exit(1) + with open(sys.argv[1], 'r') as f: + for line in f: + spec = line.rstrip() + parse_spec(spec) + + for spec in SPECS: + print_spec(spec) -- 2.44.0