X-Git-Url: https://git.tld-linux.org/?a=blobdiff_plain;f=builder.sh;h=a6eb3d72279dc333a5ca7eff96c3ce7e5fd2e645;hb=HEAD;hp=d421351a16f63a20cadb110bb0d0c7df28c85f53;hpb=75843344628e426b1265152faa19bb72f018eb6f;p=packages%2Frpm-build-tools.git diff --git a/builder.sh b/builder.sh index d421351..b022e1f 100755 --- a/builder.sh +++ b/builder.sh @@ -35,13 +35,13 @@ PROGRAM=${0##*/} APPDIR=$(d=$0; [ -L "$d" ] && d=$(readlink -f "$d"); dirname "$d") -RCSID='$Id: builder,v 1.645 2011/02/13 17:54:10 glen Exp $' r=${RCSID#* * } rev=${r%% *} -VERSION="v0.35/$rev" +VERSION="v0.35" VERSIONSTRING="\ Build package utility from PLD Linux Packages repository -$VERSION (C) 1999-2013 Free Penguins". +$VERSION (C) 1999-2020 Free Penguins". -CLEAN_PATH="/bin:/usr/bin:/usr/sbin:/sbin:/usr/X11R6/bin" +# Clean PATH without /usr/local or user paths +CLEAN_PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/X11R6/bin" # required rpm-build-macros RPM_MACROS_VER=1.534 @@ -77,11 +77,12 @@ DATE=`date +%Y-%m-%d_%H-%M-%S` # target arch, can also be used for log file naming TARGET=$(rpm -E %{_target}) +# Note the *single* quotes, this allows using shell variables expanded at build time # Example: LOGFILE='../log.$PACKAGE_NAME' # Example: LOGFILE='../LOGS/log.$PACKAGE_NAME.$DATE' # Example: LOGFILE='$PACKAGE_NAME/$PACKAGE_NAME.$DATE.log' # Example: LOGFILE='$PACKAGE_NAME.$DATE.log' -# Yes, you can use variable name! Note _single_ quotes! +# Example: LOGFILE='.log.$PACKAGE_NAME-$PACKAGE_VERSION-$PACKAGE_RELEASE.$TARGET.$DATE' LOGFILE='' # use teeboth Perl wrapper @@ -96,6 +97,7 @@ LASTLOG_FILE="" CHMOD="no" CHMOD_MODE="0644" RPMOPTS="" +RPMUSERDEFS="" RPMBUILDOPTS="" BCOND="" GROUP_BCONDS="no" @@ -111,30 +113,33 @@ PACKAGE_VERSION="" PACKAGE_NAME="" ASSUMED_NAME="" PROTOCOL="http" +IPOPT="" # use lftp by default when available -USE_LFTP= -lftp --version > /dev/null 2>&1 && USE_LFTP=yes +test -z "${USE_LFTP+x}" && lftp --version > /dev/null 2>&1 && USE_LFTP=yes PARALLEL_DOWNLOADS=10 WGET_RETRIES=${MAX_WGET_RETRIES:-0} +# .distbcond URL +DISTBCOND_URL="https://src.tld-linux.org/.distbcond" + # rsync repository with git refs of packages -PKGREVS_URL="http://pkgrevs.tld-linux.org" +PKGREVS_URL="https://pkgrevs.tld-linux.org" +PKGREVS_PREFIX="tld/" +SETPKGREV="false" # TLD git/df config TLD_GIT_SERVER="git://git.tld-linux.org" -TLD_GIT_PUSH="git@git.tld-linux.org" +TLD_GIT_PUSH="git@git.tld-linux.org:7272" TLD_PACKAGES_DIR="packages" TLD_DISTFILES_SERVER="://df.tld-linux.org" -TLD_ATTICDISTFILES_SERVER="" # PLD git/df config PLD_GIT_SERVER="git://git.pld-linux.org" PLD_GIT_PUSH="git@git.pld-linux.org" PLD_PACKAGES_DIR="packages" PLD_DISTFILES_SERVER="://distfiles.pld-linux.org" -PLD_ATTICDISTFILES_SERVER="://attic-distfiles.pld-linux.org" CVS_FORCE="" CVSIGNORE_DF="yes" @@ -151,7 +156,6 @@ NEW_REPO="" RES_FILE="" DISTFILES_SERVER=${TLD_DISTFILES_SERVER} -ATTICDISTFILES_SERVER=${TLD_ATTICDISTFILES_SERVER} DEF_NICE_LEVEL=19 SCHEDTOOL="auto" @@ -197,8 +201,9 @@ fi #GROUP_BCONDS="yes" #LOGFILE='../LOGS/log.$PACKAGE_NAME.$DATE' #TITLECHANGE=no -# -SU_SUDO="" + +SU_SUDO="sudo" + if [ -n "$HOME_ETC" ]; then USER_CFG="$HOME_ETC/.builderrc" BUILDER_MACROS="$HOME_ETC/.builder-rpmmacros" @@ -218,37 +223,19 @@ if [ "$SCHEDTOOL" = "auto" ]; then fi if [ -n "$USE_PROZILLA" ]; then - GETURI="proz --no-getch -r -P ./ -t$WGET_RETRIES $PROZILLA_OPTS" - GETURI2="$GETURI" - OUTFILEOPT="-O" + GETURI=download_proz elif [ -n "$USE_AXEL" ]; then - GETURI="axel -a $AXEL_OPTS" - GETURI2="$GETURI" - OUTFILEOPT="-o" + GETURI=download_axel elif [ -n "$USE_LFTP" ]; then GETURI=download_lftp - GETURI2=$GETURI - OUTFILEOPT="" else - wget --help 2>&1 | grep -q -- ' --no-check-certificate ' && WGET_OPTS="$WGET_OPTS --no-check-certificate" - wget --help 2>&1 | grep -q -- ' --inet ' && WGET_OPTS="$WGET_OPTS --inet" - wget --help 2>&1 | grep -q -- ' --retry-connrefused ' && WGET_OPTS="$WGET_OPTS --retry-connrefused" - WGET_OPTS="$WGET_OPTS --user-agent=$USER_AGENT" - - GETURI="wget --passive-ftp -c -nd -t$WGET_RETRIES $WGET_OPTS" - GETURI2="wget -c -nd -t$WGET_RETRIES $WGET_OPTS" - OUTFILEOPT="-O" + GETURI=download_wget fi -GETLOCAL="cp -a" +GETLOCAL=${GETLOCAL:-cp -a} -if rpm --version 2>&1 | grep -q '4.0.[0-2]'; then - RPM="rpm" - RPMBUILD="rpm" -else - RPM="rpm" - RPMBUILD="rpmbuild" -fi +RPM="rpm" +RPMBUILD="rpmbuild" # # sanity checks @@ -260,9 +247,10 @@ if [ -d $HOME/rpm/SOURCES ]; then fi POLDEK_INDEX_DIR="$($RPM --eval %_rpmdir)/" -POLDEK_CMD="$SU_SUDO /usr/bin/poldek --noask" +POLDEK_CMD="$SU_SUDO /usr/bin/poldek" # TODO: add teeboth +# TODO: what this function does? run_poldek() { RES_FILE=$(tempfile) if [ -n "$LOGFILE" ]; then @@ -270,11 +258,11 @@ run_poldek() { if [ -n "$LASTLOG_FILE" ]; then echo "LASTLOG=$LOG" > $LASTLOG_FILE fi - (${NICE_COMMAND} ${POLDEK_CMD} `while test $# -gt 0; do echo "$1 ";shift;done` ; echo $? > ${RES_FILE})|tee -a $LOG + (${NICE_COMMAND} ${POLDEK_CMD} --noask `while test $# -gt 0; do echo "$1 ";shift;done` ; echo $? > ${RES_FILE})|tee -a $LOG # FIXME $exit_pldk undefined return $exit_pldk else - (${NICE_COMMAND} ${POLDEK_CMD} `while test $# -gt 0; do echo "$1 ";shift;done` ; echo $? > ${RES_FILE}) 1>&2 >/dev/null + (${NICE_COMMAND} ${POLDEK_CMD} --noask `while test $# -gt 0; do echo "$1 ";shift;done` ; echo $? > ${RES_FILE}) 1>&2 >/dev/null return `cat ${RES_FILE}` rm -rf ${RES_FILE} fi @@ -283,12 +271,54 @@ run_poldek() { #--------------------------------------------- # functions +download_prozilla() { + local outfile=$1 url=$2 retval + + proz --no-getch -r -P ./ -t$WGET_RETRIES $PROZILLA_OPTS -O "$outfile" "$url" + retval=$? + + return $retval +} + +download_axel() { + local outfile=$1 url=$2 retval + + axel -a $AXEL_OPTS -o "$outfile" "$url" + retval=$? + + return $retval +} + +download_wget() { + local outfile=$1 url=$2 retval wget_help + if [ -z "${WGET_OPTS_SET+x}" ]; then + wget_help="$(wget --help 2>&1)" + echo "$wget_help" | grep -q -- ' --no-check-certificate ' && WGET_OPTS="$WGET_OPTS --no-check-certificate" + echo "$wget_help" | grep -q -- ' --inet ' && WGET_OPTS="$WGET_OPTS --inet" + echo "$wget_help" | grep -q -- ' --retry-connrefused ' && WGET_OPTS="$WGET_OPTS --retry-connrefused" + echo "$wget_help" | grep -q -- ' --no-iri ' && WGET_OPTS="$WGET_OPTS --no-iri" + WGET_OPTS="-c -nd -t$WGET_RETRIES $WGET_OPTS --user-agent=$USER_AGENT $IPOPT --passive-ftp" + WGET_OPTS_SET=1 + fi + + wget $WGET_OPTS -O "$outfile" "$url" + retval=$? + if [ $retval -ne 0 ]; then + if [ "`echo $url | grep -E 'ftp://'`" ]; then + ${GETURI} -O "$outfile" "$url" + retval=$? + fi + fi + return $retval +} + download_lftp() { local outfile=$1 url=$2 retval tmpfile - # TODO: use mktemp - tmpfile=$outfile.tmp + tmpfile=$(tempfile) || exit 1 lftp -c " $([ "$DEBUG" = "yes" ] && echo "debug 5;") + $([ "$IPOPT" = "-4" ] && echo "set dns:order \"inet\";") + $([ "$IPOPT" = "-6" ] && echo "set dns:order \"inet6\";") set ssl:verify-certificate no; set net:max-retries $WGET_RETRIES; set http:user-agent \"$USER_AGENT\"; @@ -311,16 +341,14 @@ usage() { # if the line contains short and long option, it will take only the long option # but if you want both being completed, put the short option to separate line echo "\ -Usage: builder [--all-branches] [-D|--debug] [-V|--version] [--short-version] [-a|--add_cvs] [-b|-ba|--build] -[-bb|--build-binary] [-bs|--build-source] [-bc] [-bi] [-bl] [-u|--try-upgrade] -[{-cf|--cvs-force}] [{-B|--branch} ] [--depth ] -[-g|--get] [-h|--help] [--ftp] [--http] [{-l|--logtofile} ] [-m|--mr-proper] -[-q|--quiet] [--date [-r ] [{-T|--tag ] -[-Tvs|--tag-version-stable] [-Ts|--tag-stable] [-Tv|--tag-version] -[{-Tp|--tag-prefix} ] [{-tt|--test-tag}] +Usage: builder [--all-branches] [-D|--debug] [-V|--version] [--short-version] +[-a|--add_cvs] [-b|-ba|--build] [-bb|--build-binary] [-bs|--build-source] +[-bc] [-bi] [-bl] [-u|--try-upgrade] [{-cf|--cvs-force}] [{-B|--branch} ] +[--depth ] [-g|--get] [-h|--help] [--ftp] [--http] [{-l|--logtofile} ] +[-m|--mr-proper] [-q|--quiet] [--date ] [-r ] [-nu|--no-urls] [-v|--verbose] [--opts ] [--short-circuit] [--show-bconds] [--with/--without ] [--define ] -[--git-pld|--git-tld] [--pkgrev] +[--git-pld|--git-tld] [--pkgrev] [-lp] [.spec][:tag] -4 - force IPv4 when transferring files @@ -418,18 +446,6 @@ Usage: builder [--all-branches] [-D|--debug] [-V|--version] [--short-version] [ but icon file is absent), -su, --source-urls - list urls - urls to sources and patches intended for copying urls with spec with lots of macros in urls --T , --tag - - add git tag for files, --Tvs, --tag-version-stable - - add git tags STABLE and NAME-VERSION-RELEASE for files, --Ts, --tag-stable - - add git tag STABLE for files, --Tv, ---tag-version - add git tag NAME-VERSION-RELEASE for files, --Tp, --tag-prefix - - add to NAME-VERSION-RELEASE tags, --tt, --test-tag - - fail if tag is already present, -ir, --integer-release-only - allow only integer and snapshot releases -v, --verbose - be verbose, @@ -462,26 +478,50 @@ Usage: builder [--all-branches] [-D|--debug] [-V|--version] [--short-version] [ constructions. Set GROUP_BCONDS to yes to make use of it. --target , --target= - build for platform . ---init-rpm-dir - initialize ~/rpm directory structure +--init-rpm-dir, --init + - initialize ~/rpm directory structure --git-pld - force use of PLD git and distfiles --git-tld - force use of TLD git and distfiles (note: it will fall back to PLD git/df if package doesn't exist in TLD git) ---pkgrev - write git revision of package being built into rsync repo +--pkgrev - save git revision of package being built into pkgrevs (note: only official TLD source builders are allowed to do so) +-lp - list pkgrevs for specified package " } +is_rpmorg() { + local v + + v=$(LC_ALL=C LANG=C rpm --version 2>&1) + v=${v#RPM version } # rpm 4 + v=${v#rpm \(RPM\) } # rpm 5 + + case "$v" in + 4.5|5.*) + return 1 + ;; + 4.*) + return 0; + ;; + *) + echo "ERROR: unsupported RPM version $v" >&2 + exit 1 + esac +} + # create tempfile. as secure as possible tempfile() { - mktemp --tmpdir -t builder.$PACKAGE_NAME.XXXXXX || ${TMPDIR:-/tmp}/builder.$RANDOM.$$ + local prefix=builder.$PACKAGE_NAME${1:+.$1} + mktemp --tmpdir -t $prefix.XXXXXX || echo ${TMPDIR:-/tmp}/$prefix.$RANDOM.$$ } tempdir() { - mktemp --tmpdir -d builder.$PACKAGE_NAME.XXXXXX + local prefix=builder.$PACKAGE_NAME${1:+.$1} + mktemp --tmpdir -d $prefix.XXXXXX } # inserts git log instead of %changelog -# outputs name of modified file created by tempfile +# @output directory containing modified specfile insert_gitlog() { local SPECFILE=$1 specdir=$(tempdir) gitlog=$(tempfile) speclog=$(tempfile) @@ -501,7 +541,7 @@ insert_gitlog() { done > $gitlog # add link to full git logs - local giturl="http://git.tld-linux.org/?p=packages/$PACKAGE_NAME.git;a=log" + local giturl="http://git.tld-linux.org/?p=packages/${SPECFILE%.spec}.git;a=log" if [ -n "$CVSTAG" ]; then giturl="$giturl;h=$CVSTAG" fi @@ -549,14 +589,30 @@ teeboth() { # common changes: # - perl(Package::Name) -> perl-Package-Name depspecname() { - local package="$1" + local DEPS + + if [ $# -gt 0 ]; then + DEPS="$@" + else + DEPS=$(cat) + fi + + echo "$DEPS" | tr ' ' '\n' | sed -re ' + # perl virtual deps + /perl\(.*\)/{ + s/perl\((.*)\)/perl-\1/ + s/::/-/g + } + + s/apache\(EAPI\)-devel/apache-devel/ - package=$(echo "$package" | sed -e '/perl(.*)/{s,perl(\(.*\)),perl-\1,;s,::,-,g};' -e 's/-\(devel\|static\)$//' ) - echo "$package" + s/db-devel/db5.3-devel/ + s/libjpeg-devel/libjpeg-turbo-devel/ + ' } update_shell_title() { - [ -t 1 ] || return + [ -t 2 ] || return local len=${COLUMNS:-80} local msg="$(echo "$*" | cut -c-$len)" @@ -604,11 +660,6 @@ set_spec_target() { # runs rpm with minimal macroset minirpm() { - # we reset macros not to contain macros.build as all the %() macros are - # executed here, while none of them are actually needed. - # at the time of this writing macros.build + macros contained 70 "%(...)" macros. - safe_macrofiles=$(rpm $TARGET_SWITCH --showrc | awk -F: '/^macrofiles/ { gsub(/^macrofiles[ \t]+:/, "", $0); gsub(/:.*macros.build:/, ":", $0); print $0 } ') - # TODO: move these to /usr/lib/rpm/macros cat > $BUILDER_MACROS <<'EOF' %x8664 x86_64 amd64 ia32e @@ -661,7 +712,7 @@ minirpm() { %remove_etc_shells(p) %{p:} %lua_add_etc_shells() %{nil} %lua_remove_etc_shells() %{nil} -%required_jdk %{nil} +%required_jdk jdk %buildrequires_jdk %{nil} %pear_package_print_optionalpackages %{nil} EOF @@ -671,18 +722,30 @@ EOF %_sourcedir ./ EOF fi - eval PATH=$CLEAN_PATH $RPMBUILD $TARGET_SWITCH --macros "$safe_macrofiles:$BUILDER_MACROS" $QUIET $RPMOPTS $RPMBUILDOPTS $BCOND $* 2>&1 + if ! is_rpmorg; then + local safe_macrofiles + safe_macrofiles=$(rpm $TARGET_SWITCH --showrc | awk -F: '/^macrofiles/ { gsub(/^macrofiles[ \t]+:/, "", $0); print $0 } ') + eval PATH=$CLEAN_PATH $RPMBUILD $TARGET_SWITCH --macros "$safe_macrofiles:$BUILDER_MACROS" $QUIET $RPMOPTS $RPMUSERDEFS $RPMBUILDOPTS $BCOND $* 2>&1 + else + eval PATH=$CLEAN_PATH $RPMBUILD $TARGET_SWITCH --load "$BUILDER_MACROS" $QUIET $RPMOPTS $RPMUSERDEFS $RPMBUILDOPTS $BCOND $* 2>&1 + fi } cache_rpm_dump() { + local SPEC_PATH if [ -n "$DEBUG" ]; then set -x set -v fi + if [ ! -z "$1" ]; then + SPEC_PATH="$1" + else + SPEC_PATH="$PACKAGE_DIR/$SPECFILE" + fi if [ -x /usr/bin/rpm-specdump ]; then update_shell_title "cache_rpm_dump using rpm-specdump command" - rpm_dump_cache=$(rpm-specdump $TARGET_SWITCH $BCOND --define "_specdir $PACKAGE_DIR" --define "_sourcedir $PACKAGE_DIR" $PACKAGE_DIR/$SPECFILE) + rpm_dump_cache=$(eval rpm-specdump $TARGET_SWITCH $BCOND $RPMUSERDEFS --define \'_specdir $PACKAGE_DIR\' --define \'_sourcedir $PACKAGE_DIR\' $SPEC_PATH) else update_shell_title "cache_rpm_dump using rpmbuild command" local rpm_dump @@ -745,7 +808,7 @@ parse_spec() { get_icons cd $PACKAGE_DIR - cache_rpm_dump + cache_rpm_dump "$1" if rpm_dump | grep -qEi ":.*nosource.*1"; then FAIL_IF_NO_SOURCES="no" @@ -785,6 +848,13 @@ parse_spec() { update_shell_title "parse_spec: OK!" } +# aborts program abnormally +die() { + local rc=${2:-1} + echo >&2 "$PROGRAM: ERROR: $*" + exit $rc +} + Exit_error() { if [ -n "$DEBUG" ]; then set -x @@ -803,7 +873,11 @@ Exit_error() { exit 2 ;; "err_no_spec_in_repo" ) remove_build_requires - echo >&2 "Error: spec file not stored in PLD repo." + echo >&2 "Error: spec file not stored in repository." + if [ -n "$2" ]; then + echo >&2 "Tried: $2" + fi + exit 3 ;; "err_no_source_in_repo" ) remove_build_requires @@ -820,9 +894,9 @@ Exit_error() { remove_build_requires echo >&2 "Error: couldn't get out package name/version/release from spec file." exit 6 ;; - "err_tag_exists" ) + "err_pkgrev_exists" ) remove_build_requires - echo >&2 "Tag ${2} already exists" + echo >&2 "Error: package revision ${2} already exists" exit 9 ;; "err_fract_rel" ) remove_build_requires @@ -849,6 +923,9 @@ Exit_error() { "err_pkgrev_get" ) echo >&2 "Error: failed to get package revision for tag ${2}" exit 16 ;; + "err_distbcond_update" ) + echo >&2 "Error: failed to update .distbcond" + exit 17 ;; "err_not_implemented" ) remove_build_requires echo >&2 "Error: functionality not yet imlemented" @@ -865,7 +942,7 @@ init_builder() { fi if [ "$NOINIT" != "yes" ] ; then - TOP_DIR=$(eval $RPM $RPMOPTS --eval '%{_topdir}') + TOP_DIR=$(eval $RPM $RPMOPTS $RPMUSERDEFS --eval '%{_topdir}') local macros_ver=$(rpm -E %?rpm_build_macros) if [ -z "$macros_ver" ]; then @@ -880,8 +957,10 @@ init_builder() { PACKAGE_DIR=$REPO_DIR/$ASSUMED_NAME fi else - REPO_DIR="." - PACKAGE_DIR="." + TOP_DIR=$(pwd) + PACKAGE_DIR=$TOP_DIR + REPO_DIR=$PACKAGE_DIR + RPMBUILDOPTS="$RPMBUILDOPTS --define '_topdir $TOP_DIR' --define '_builddir %_topdir' --define '_rpmdir %_topdir' --define '_srcrpmdir %_topdir'" fi export GIT_WORK_TREE=$PACKAGE_DIR export GIT_DIR=$PACKAGE_DIR/.git @@ -920,10 +999,17 @@ create_git_repo() { fi [ -d "$ASSUMED_NAME/.git" ] || NEW_REPO=yes ssh $GIT_PUSH create ${ASSUMED_NAME} || Exit_error err_cvs_add_failed + ( + set -e git init - git remote add $REMOTE_PLD ${GIT_SERVER}/${PACKAGES_DIR}/${ASSUMED_NAME}.git && \ - git remote set-url --push $REMOTE_PLD ssh://${GIT_PUSH}/${PACKAGES_DIR}/${ASSUMED_NAME} \ - || Exit_error err_remote_problem $REMOTE_PLD + git remote add $REMOTE_PLD ${GIT_SERVER}/${PACKAGES_DIR}/${ASSUMED_NAME}.git + git remote set-url --push $REMOTE_PLD ssh://${GIT_PUSH}/${PACKAGES_DIR}/${ASSUMED_NAME} + + git config --local push.default current + git config --local branch.master.remote $REMOTE_PLD + git config --local branch.master.merge refs/heads/master + ) + test $? = 0 || Exit_error err_remote_problem $REMOTE_PLD } get_spec() { @@ -940,7 +1026,7 @@ get_spec() { if [ "$NOCVSSPEC" != "yes" ]; then if [ -z "$DEPTH" ]; then if [ -d "$PACKAGE_DIR/.git" ]; then - git fetch $REMOTE_PLD || Exit_error err_no_spec_in_repo + git fetch $IPOPT $REMOTE_PLD || Exit_error err_no_spec_in_repo elif [ "$ADD_PACKAGE_CVS" = "yes" ]; then if [ ! -r "$PACKAGE_DIR/$SPECFILE" ]; then echo "ERROR: No package to add ($PACKAGE_DIR/$SPECFILE)" >&2 @@ -950,13 +1036,16 @@ get_spec() { else ( unset GIT_WORK_TREE - git clone -o $REMOTE_PLD ${GIT_SERVER}/${PACKAGES_DIR}/${ASSUMED_NAME}.git || { + git clone $IPOPT -o $REMOTE_PLD ${GIT_SERVER}/${PACKAGES_DIR}/${ASSUMED_NAME}.git || { # softfail if new package, i.e not yet added to PLD rep [ ! -f "$PACKAGE_DIR/$SPECFILE" ] && Exit_error err_no_spec_in_repo - echo "Warning: package not in CVS - assuming new package" + echo "Warning: package not in Git - assuming new package" NOCVSSPEC="yes" } - git config --local --add "remote.$REMOTE_PLD.fetch" 'refs/notes/*:refs/notes/*' + git config --local --add "remote.$REMOTE_PLD.fetch" 'refs/notes/*:refs/notes/*' + git config --local --add "remote.$REMOTE_PLD.push" 'refs/notes/*:refs/notes/*' + git config --local --add "remote.$REMOTE_PLD.push" HEAD + git config --local push.default current git remote set-url --push $REMOTE_PLD ssh://${GIT_PUSH}/${PACKAGES_DIR}/${ASSUMED_NAME} ) fi @@ -967,7 +1056,10 @@ get_spec() { fi git init git remote add $REMOTE_PLD ${GIT_SERVER}/${PACKAGES_DIR}/${ASSUMED_NAME}.git - git config --local --add "remote.$REMOTE_PLD.fetch" 'refs/notes/*:refs/notes/*' + git config --local --add "remote.$REMOTE_PLD.fetch" 'refs/notes/*:refs/notes/*' + git config --local --add "remote.$REMOTE_PLD.push" 'refs/heads/*:refs/remotes/origin/*' + git config --local --add "remote.$REMOTE_PLD.push" HEAD + git config --local push.default current git remote set-url --push $REMOTE_PLD ssh://${GIT_PUSH}/${PACKAGES_DIR}/${ASSUMED_NAME} CVSTAG=${CVSTAG:-"master"} fi @@ -975,19 +1067,19 @@ get_spec() { if [ -z "$ALL_BRANCHES" ]; then refs="${CVSTAG}:remotes/${REMOTE_PLD}/${CVSTAG}" fi - git fetch $DEPTH $REMOTE_PLD $refs || { + git fetch $IPOPT $DEPTH $REMOTE_PLD $refs || { echo >&2 "Error: branch $CVSTAG does not exist" exit 3 } fi - git fetch $REMOTE_PLD 'refs/notes/*:refs/notes/*' + git fetch $IPOPT $REMOTE_PLD 'refs/notes/*:refs/notes/*' cvsignore_df .gitignore # add default log format to .gitignore if it is relative to package dir if [ -n "$LOGFILE" -a "$LOGFILE" = "${LOGFILE##*/}" ]; then # substitute known "macros" to glob - local logfile=$(echo "$LOGFILE" | sed -e 's,\$\(PACKAGE_NAME\|DATE\|TARGET\),*,g') + local logfile=$(echo "$LOGFILE" | sed -r -e 's,\$(PACKAGE_(NAME|VERSION|RELEASE)|DATE|TARGET),*,g') if [ "$logfile" ]; then cvsignore_df "$logfile" fi @@ -995,7 +1087,7 @@ get_spec() { # create symlinks for tools if [ "$SYMLINK_TOOLS" != "no" -a -d "$PACKAGE_DIR" ]; then - for a in dropin md5 adapter builder {relup,compile,repackage,rsync,pearize}.sh pldnotify.awk; do + for a in dropin md5 builder {relup,compile,repackage,rsync,pearize}.sh; do # skip tools that don't exist in top dir [ -f $a ] || continue # skip tools that already exist @@ -1009,15 +1101,19 @@ get_spec() { if [ -n "$CVSTAG" ]; then local _rev=$(get_pkgrev "$CVSTAG") echo "$_rev" | grep -q -E "^ERROR$" || CVSTAG="$_rev" - if git rev-parse --verify -q "$CVSTAG"; then - git checkout "$CVSTAG" -- + if git rev-parse --verify -q "$CVSTAG" >/dev/null; then + # checkout only if differs, so this will not trash git reflog + if [ $(git rev-parse "$CVSTAG") != $(git rev-parse HEAD) ]; then + git checkout "$CVSTAG" -- + fi elif git rev-parse --verify -q "refs/remotes/${REMOTE_PLD}/$CVSTAG"; then git checkout -t "refs/remotes/${REMOTE_PLD}/$CVSTAG" > /dev/null fi if [ $(git rev-parse "$CVSTAG") != $(git rev-parse HEAD) ]; then Exit_error "err_no_checkut" "$CVSTAG" fi - git merge --ff-only '@{u}' + + git merge --ff-only '@{u}' git symbolic-ref -q HEAD > /dev/null && [ "$NOCVSSPEC" != "yes" ] && if [ -n "$CVSDATE" ]; then git checkout $(git rev-list -n1 --before="'$CVSDATE'" $CVSTAG) || exit 1 @@ -1025,7 +1121,7 @@ get_spec() { fi if [ ! -f "$PACKAGE_DIR/$SPECFILE" ]; then - Exit_error err_no_spec_in_repo + Exit_error err_no_spec_in_repo "$PACKAGE_DIR/$SPECFILE" fi if [ "$CHMOD" = "yes" -a -n "$SPECFILE" ]; then @@ -1037,15 +1133,22 @@ get_spec() { set_spec_target } +# find mirrors in this order. first match wins: +# - package dir (~/rpm/packages/foo) +# - repository dir (~/rpm/packages) +# - tools dir dir (~/rpm/packages/rpm-build-tools) find_mirror() { - cd "$REPO_DIR" local url="$1" - if [ ! -f "mirrors" ] ; then - ln -s ../rpm-build-tools/mirrors . - fi - IFS="|" + update_shell_title "find_mirror[$url][$REPO_DIR]" + + # NOTE: as while loop runs in subshell, + # we use exit 2 to indicate that the match was found + # otherwise we end up outputing mirror url and origin url. + local origin mirror name rest ol prefix + IFS="|" + cat "$PACKAGE_DIR/mirrors" "$REPO_DIR/mirrors" "$REPO_DIR/../rpm-build-tools/mirrors" /dev/null 2>/dev/null | \ while read origin mirror name rest; do # skip comments and empty lines if [ -z "$origin" ] || [ "${origin#\#}" != "$origin" ]; then @@ -1056,10 +1159,9 @@ find_mirror() { if [ "$prefix" = "$origin" ] ; then suffix=$(echo "$url" | cut -b $((ol+1))-) echo -n "$mirror$suffix" - return 0 + exit 2 fi - done < mirrors - echo "$url" + done && echo "$url" } # Warning: unpredictable results if same URL used twice @@ -1120,11 +1222,7 @@ distfiles_path() { } distfiles_url() { - echo "$PROTOCOL$DISTFILES_SERVER/distfiles/$(distfiles_path "$1")" -} - -distfiles_attic_url() { - echo "$PROTOCOL$ATTICDISTFILES_SERVER/distfiles/Attic/$(distfiles_path "$1")" + echo "$PROTOCOL$DISTFILES_SERVER/$(distfiles_path "$1")" } good_md5() { @@ -1142,7 +1240,7 @@ cvsignore_df() { if [ "$CVSIGNORE_DF" != "yes" ]; then return fi - cvsignore=${PACKAGE_DIR}/.gitignore + local cvsignore=${PACKAGE_DIR}/.git/info/exclude # add only if not yet there if ! awk -vf="$1" -vc=1 '$0 == f { c = 0 } END { exit c }' $cvsignore 2>/dev/null; then @@ -1298,7 +1396,6 @@ get_files() { url=$(distfiles_url "$i") fi - url_attic=$(distfiles_attic_url "$i") FROM_DISTFILES=1 # is $url local file? if [[ "$url" = [./]* ]]; then @@ -1307,35 +1404,14 @@ get_files() { else local uri=${url} # make shorter message for distfiles urls - if [[ "$uri" = ${PROTOCOL}${DISTFILES_SERVER}* ]] || [[ "$uri" = ${PROTOCOL}${ATTICDISTFILES_SERVER}* ]]; then - uri=${uri#${PROTOCOL}${DISTFILES_SERVER}/distfiles/by-md5/?/?/*/} - uri=${uri#${PROTOCOL}${ATTICDISTFILES_SERVER}/distfiles/by-md5/?/?/*/} + if [[ "$uri" = ${PROTOCOL}${DISTFILES_SERVER}* ]]; then + uri=${uri#${PROTOCOL}${DISTFILES_SERVER}/by-md5/?/?/*/} uri="df: $uri" fi update_shell_title "${GETURI%% *}: $uri" - ${GETURI} ${OUTFILEOPT} "$target" "$url" || \ - if [ "`echo $url | grep -E 'ftp://'`" ]; then - update_shell_title "${GETURI2%% *}: $url" - ${GETURI2} ${OUTFILEOPT} "$target" "$url" - fi + ${GETURI} "$target" "$url" fi - # is it empty file? - if [ ! -s "$target" ]; then - rm -f "$target" - if [ `echo $url_attic | grep -E '^(\.|/)'` ]; then - update_shell_title "${GETLOCAL%% *}: $url_attic" - ${GETLOCAL} $url_attic $target - else - update_shell_title "${GETURI%% *}: $url_attic" - ${GETURI} ${OUTFILEOPT} "$target" "$url_attic" || \ - if [ "`echo $url_attic | grep -E 'ftp://'`" ]; then - update_shell_title "${GETURI2%% *}: $url_attic" - ${GETURI2} ${OUTFILEOPT} "$target" "$url_attic" - fi - test -s "$target" || rm -f "$target" - fi - fi if [ -s "$target" ]; then cvsignore_df $target @@ -1352,11 +1428,7 @@ get_files() { im="$i" fi update_shell_title "${GETURI%% *}: $im" - ${GETURI} ${OUTFILEOPT} "$target" "$im" || \ - if [ "`echo $im | grep -E 'ftp://'`" ]; then - update_shell_title "${GETURI2%% *}: $im" - ${GETURI2} ${OUTFILEOPT} "$target" "$im" - fi + ${GETURI} "$target" "$im" test -s "$target" || rm -f "$target" fi @@ -1381,20 +1453,7 @@ get_files() { FROM_DISTFILES=2 rm -f $target update_shell_title "${GETURI%% *}: $url" - ${GETURI} ${OUTFILEOPT} "$target" "$url" || \ - if [ "`echo $url | grep -E 'ftp://'`" ]; then - update_shell_title "${GETURI2%% *}: $url" - ${GETURI2} ${OUTFILEOPT} "$target" "$url" - fi - if [ ! -s "$target" ]; then - rm -f "$target" - update_shell_title "${GETURI%% *}: $url_attic" - ${GETURI} ${OUTFILEOPT} "$target" "$url_attic" || \ - if [ "`echo $url_attic | grep -E 'ftp://'`" ]; then - update_shell_title "${GETURI2%% *}: $url_attic" - ${GETURI2} ${OUTFILEOPT} "$target" "$url_attic" - fi - fi + ${GETURI} "$target" "$url" test -s "$target" || rm -f "$target" fi done @@ -1410,36 +1469,13 @@ get_files() { fi } -tag_exist() { -# If tag exists and points to other commit exit with error -# If it existsts and points to HEAD return 1 -# If it doesn't exist return 0 - local _tag="$1" - local sha1=$(git rev-parse HEAD) - echo "Searching for tag $_tag..." - if [ -n "$DEPTH" ]; then - local ref=$(git ls-remote $REMOTE_PLD "refs/tags/$_tag" | cut -c -40) - else - local ref=$(git show-ref -s "refs/tags/$_tag") - fi - [ -z "$ref" ] && return 0 - [ "$ref" = "$sha1" ] || Exit_error err_tag_exists "$_tag" - return 1 -} - -make_tagver() { +make_pkgrev() { if [ -n "$DEBUG" ]; then set -x set -v fi - # Check whether first character of PACKAGE_NAME is legal for tag name - if [ -z "${PACKAGE_NAME##[_0-9]*}" -a -z "$TAG_PREFIX" ]; then - TAG_PREFIX=tag_ - fi - - # NOTE: CVS tags may must not contain the characters `$,.:;@' - TAGVER=$(echo $TAG_PREFIX$PACKAGE_NAME-$PACKAGE_VERSION-$PACKAGE_RELEASE) + TAGVER=$(echo $PKGREVS_PREFIX$PACKAGE_NAME-$PACKAGE_VERSION-$PACKAGE_RELEASE) # Remove @kernel.version_release from TAGVER because tagging sources # could occur with different kernel-headers than kernel-headers used at build time. @@ -1452,37 +1488,6 @@ make_tagver() { echo -n "$TAGVER" } -tag_files() { - if [ -n "$DEBUG" ]; then - set -x - set -v - fi - - echo "Version: $PACKAGE_VERSION" - echo "Release: $PACKAGE_RELEASE" - - local _tag - if [ "$TAG_VERSION" = "yes" ]; then - _tag=`make_tagver` - fi - if [ -n "$TAG" ]; then - _tag="$TAG" - fi - echo "tag: $_tag" - - local OPTIONS="tag $CVS_FORCE" - - cd "$PACKAGE_DIR" - - if tag_exist $_tag || [ -n "$CVS_FORCE" ]; then - update_shell_title "tag sources: $_tag" - git $OPTIONS $_tag || exit - git push $CVS_FORCE $REMOTE_PLD tag $_tag || Exit_error err_remote_problem $REMOTE_PLD - else - echo "Tag $_tag already exists and points to the same commit" - fi -} - get_pkgrev() { [ -z "$1" ] && return 1 local _tmp=$(mktemp /tmp/.builder-XXXX) @@ -1495,12 +1500,8 @@ get_pkgrev() { set_pkgrev() { local _tag - if [ "$TAG_VERSION" = "yes" ]; then - _tag=`make_tagver` - fi - if [ -n "$TAG" ]; then - _tag="$TAG" - fi + parse_spec "$1" + _tag=`make_pkgrev` echo "Writing git revision for tag $_tag" local _tmp=$(mktemp /tmp/.builder-XXXX) rm $_tmp 2>/dev/null @@ -1508,11 +1509,20 @@ set_pkgrev() { $GETURI $OUTFILEOPT $_tmp $PKGREVS_URL/set/$_rev/$_tag 1>/dev/null 2>&1 || Exit_error err_pkgrev_get "$1" local result=$(cat $_tmp) rm $_tmp 2>/dev/null + [ "$(get_pkgrev "$_tag")" = "$CVSTAG" ] && return 0 echo "$result" | grep -q -E "^OK$" && return 0 - echo "$result" | grep -q -E "^EXISTS$" && Exit_error err_tag_exists "$_tag" + echo "$result" | grep -q -E "^EXISTS$" && Exit_error err_pkgrev_exists "$_tag" Exit_error err_pkgrev_set } +list_pkgrev() { + local _tmp=$(mktemp /tmp/.builder-XXXX) + rm $_tmp 2>/dev/null + $GETURI $OUTFILEOPT $_tmp $PKGREVS_URL/list/$PACKAGE_NAME 1>/dev/null 2>&1 + cat $_tmp + rm $_tmp 2>/dev/null +} + branch_files() { TAG=$1 echo "Git branch: $TAG" @@ -1572,50 +1582,61 @@ set_version() { " $specfile } +# try to upgrade .spec to new version +# if --upgrade-version is specified, use that as new version, otherwise invoke pldnotify to find new version +# +# return 1: if .spec was updated +# return 0: no changes to .spec +# exit 1 in case of error try_upgrade() { - if [ -n "$TRY_UPGRADE" ]; then - local TNOTIFY TNEWVER TOLDVER - update_shell_title "build_package: try_upgrade" + if [ -z "$TRY_UPGRADE" ]; then + return 0 + fi - cd "$PACKAGE_DIR" - - if [ "$UPGRADE_VERSION" ]; then - TNEWVER=$UPGRADE_VERSION - else - if [ -n "$FLOAT_VERSION" ]; then - TNOTIFY=$($APPDIR/pldnotify.awk ${BE_VERBOSE:+-vDEBUG=1} $SPECFILE -n) || exit 1 - else - TNOTIFY=$($APPDIR/pldnotify.awk ${BE_VERBOSE:+-vDEBUG=1} $SPECFILE) || exit 1 - fi + local TNOTIFY TNEWVER TOLDVER + update_shell_title "build_package: try_upgrade" - # pldnotify.awk does not set exit codes, but it has match for ERROR - # in output which means so. - if [[ "$TNOTIFY" = *ERROR* ]]; then - echo >&2 "$TNOTIFY" - exit 1 - fi + cd "$PACKAGE_DIR" - TNEWVER=$(echo $TNOTIFY | awk '{ match($4,/\[NEW\]/); print $5 }') + if [ "$UPGRADE_VERSION" ]; then + TNEWVER=$UPGRADE_VERSION + echo "Updating spec file to version $TNEWVER" + else + if [ -n "$FLOAT_VERSION" ]; then + TNOTIFY=$(pldnotify ${BE_VERBOSE:+-vDEBUG=1} $SPECFILE -n) || exit 1 + else + TNOTIFY=$(pldnotify ${BE_VERBOSE:+-vDEBUG=1} $SPECFILE) || exit 1 fi - if [ -n "$TNEWVER" ]; then - TOLDVER=`echo $TNOTIFY | awk '{ print $3; }'` - echo "New version found, updating spec file from $TOLDVER to version $TNEWVER" - if [ "$REVERT_BROKEN_UPGRADE" = "yes" ]; then - cp -f $SPECFILE $SPECFILE.bak - fi - chmod +w $SPECFILE - set_release $SPECFILE $PACKAGE_RELEASE 1 - set_version $SPECFILE $PACKAGE_VERSION $TNEWVER - parse_spec - if [ "$PACKAGE_VERSION" != "$TNEWVER" ]; then - echo >&2 "Upgrading version failed, you need to update spec yourself" - exit 1 - fi - return 1 + # pldnotify does not set exit codes, but it has match for ERROR + # in output which means so. + if [[ "$TNOTIFY" = *ERROR* ]]; then + echo >&2 "$TNOTIFY" + exit 1 fi + + TOLDVER=`echo $TNOTIFY | awk '{ print $3; }'` + echo "New version found, updating spec file from $TOLDVER to version $TNEWVER" + + TNEWVER=$(echo $TNOTIFY | awk '{ match($4,/\[NEW\]/); print $5 }') fi - return 0 + + if [ -z "$TNEWVER" ]; then + return 0 + fi + + if [ "$REVERT_BROKEN_UPGRADE" = "yes" ]; then + cp -f $SPECFILE $SPECFILE.bak + fi + chmod +w $SPECFILE + set_version $SPECFILE $PACKAGE_VERSION $TNEWVER + set_release $SPECFILE $PACKAGE_RELEASE 1 + parse_spec + if [ "$PACKAGE_VERSION" != "$TNEWVER" ]; then + echo >&2 "Upgrading version failed, you need to update spec yourself" + exit 1 + fi + return 1 } build_package() { @@ -1659,10 +1680,36 @@ build_package() { fi fi + # unset these, should not be exposed to builder shell! + unset GIT_WORK_TREE GIT_DIR + # these are set by jenkins + unset GIT_PREVIOUS_COMMIT GIT_URL GIT_PREVIOUS_SUCCESSFUL_COMMIT GIT_BRANCH GIT_COMMIT + # this may be set by user + unset GIT_SSH + # may be set by user + unset GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_TESTING_PORCELAIN_COMMAND_LIST + # fail if something still set + env | grep ^GIT_ && Exit_error err_build_fail + local specdir=$(insert_gitlog $SPECFILE) + ulimit -c unlimited + # If required exclude directories with systemd related files from package contents + if grep -q -E '(lib/(systemd|binfmt.d|sysctl.d|sysusers.d|tmpfiles.d)|systemd(unitdir|userunitdir|tmpfilesdir))' $specdir/$SPECFILE; then + sed -i -e '/^%exclude_systemd_files/d; /^%files/s/$/\n%exclude_systemd_files/g;' $specdir/$SPECFILE + fi + # Enable/disable distro wide bconds based on ~/.distbcond + process_distbcond "$specdir/$SPECFILE" + # Add %tld macro to release to allow release control + sed -i -r -e '/^Release:/s/%\{\?tld\}//g; s/^Release:\s+(.*)$/Release:\t\1%{?tld}/;' $specdir/$SPECFILE # FIXME: eval here is exactly why? - PATH=$CLEAN_PATH eval teeboth "'$logfile'" ${NICE_COMMAND} $RPMBUILD $TARGET_SWITCH $BUILD_SWITCH -v $QUIET $CLEAN $RPMOPTS $RPMBUILDOPTS $BCOND --define \'_specdir $PACKAGE_DIR\' --define \'_sourcedir $PACKAGE_DIR\' $specdir/$SPECFILE + PATH=$CLEAN_PATH eval teeboth "'$logfile'" ${TIME_COMMAND} ${NICE_COMMAND} $RPMBUILD $TARGET_SWITCH $BUILD_SWITCH -v $QUIET $CLEAN $RPMOPTS $RPMUSERDEFS $RPMBUILDOPTS $BCOND --define \'_specdir $PACKAGE_DIR\' --define \'_sourcedir $PACKAGE_DIR\' $specdir/$SPECFILE retval=$? + + # Set pkgrev if requested and build status is OK + if [ ! -z "$SETPKGREV" ] && [ "$SETPKGREV" = "true" ] && [ "$retval" -eq "0" ]; then + set_pkgrev "$specdir/$SPECFILE" + fi + rm -r $specdir if [ -n "$logfile" ] && [ -n "$LOGDIROK" ] && [ -n "$LOGDIRFAIL" ]; then @@ -1757,6 +1804,57 @@ process_bcondrc() { update_shell_title "parse ~/.bcondrc: DONE!" } +process_distbcond() { + # apply bconds from ~/.distbcond to spec + # The file structure is like gentoo's package.use: + # --- + # * -selinux + # samba -mysql -pgsql + # w32codec-installer license_agreement + # php +mysqli + # --- + + local distbcond=$HOME/.distbcond + + echo "Updating .distbcond" + wget -q --no-check-certificate $DISTBCOND_URL -O "$distbcond" + if [ $? -ne 0 ]; then + Exit_error err_distbcond_update + fi + DISTBCOND_MD5=$(grep -E "^# MD5 " "$distbcond" | cut -d ' ' -f 3) + DISTBCOND_MD5_LOCAL=$(grep -v -E "^#" "$distbcond" | md5sum | cut -d ' ' -f 1) + if [ "$DISTBCOND_MD5" != "$DISTBCOND_MD5_LOCAL" ]; then + rm -f "$distbcond" + Exit_error err_distbcond_update + fi + + SN=${SPECFILE%%\.spec} + + local bcond_avail=$(find_spec_bcond $SPECFILE) + + while read pkg flags; do + # ignore comments + [[ "$pkg" == \#* ]] && continue + + # any package or current package? + if [ "$pkg" = "*" ] || [ "$pkg" = "$PACKAGE_NAME" ] || [ "$pkg" = "$SN" ]; then + for flag in $flags; do + local opt=${flag#[+-]} + + # use only flags which are in this package. + if [[ $bcond_avail = *${opt}* ]]; then + if [[ $flag = -* ]]; then + sed -i -r -e '/^%bcond_(with|without)\s+'$opt'/s/^%bcond_(with|without)/%bcond_with/g;' "$1" + elif [[ $flag = +* ]]; then + sed -i -r -e '/^%bcond_(with|without)\s+'$opt'/s/^%bcond_(with|without)/%bcond_without/g;' "$1" + fi + fi + done + fi + done < $distbcond + update_shell_title "parse ~/.distbcond: DONE!" +} + set_bconds_values() { update_shell_title "set bcond values" @@ -1783,7 +1881,7 @@ set_bconds_values() { without_*) bcond=${opt#without_} case "$BCOND" in - *--without?${bcond}*) + *--without?${bcond}\ *|*--without?${bcond}) AVAIL_BCONDS_WITHOUT="$AVAIL_BCONDS_WITHOUT <$bcond>" ;; *) @@ -1794,7 +1892,7 @@ set_bconds_values() { with_*) bcond=${opt#with_} case "$BCOND" in - *--with?${bcond}*) + *--with?${bcond}\ *|*--with?${bcond}) AVAIL_BCONDS_WITH="$AVAIL_BCONDS_WITH <$bcond>" ;; *) @@ -1814,13 +1912,13 @@ run_sub_builder() { package_name="${1}" update_shell_title "run_sub_builder $package_name" # - # No i tutaj bym chcia³ zrobiæ sztuczn± inteligencjê, która spróbuje tego - # pakieta zbudowaæ. Aktualnie niewiele dziala, bo generalnie nie widze do + # No i tutaj bym chciał zrobić sztuczną inteligencję, która spróbuje tego + # pakieta zbudować. Aktualnie niewiele dziala, bo generalnie nie widze do # konca algorytmu... Ale damy rade. :) Na razie po prostu sie wyjebie tak samo # jakby nie bylo tego kawalka kodu. # - # Update: Poprawi³em parê rzeczy i zaczê³o generowaæ pakiety spoza zadanej listy. - # Jednym s³owem budowanie niespoldkowanych zale¿no¶ci dzia³a w paru przypadkach. + # Update: Poprawiłem parę rzeczy i zaczęło generować pakiety spoza zadanej listy. + # Jednym słowem budowanie niespoldkowanych zależności działa w paru przypadkach. # # # y0shi. @@ -1845,6 +1943,46 @@ run_sub_builder() { NOT_INSTALLED_PACKAGES="$NOT_INSTALLED_PACKAGES $package_name" } +# install package with poldek +# @return exit code from poldek +# +# this requires following sudo rules: +# - poldek --noask --caplookup -ug +poldek_install() { + LC_ALL=C LANG=C $POLDEK_CMD --noask --caplookup --uniq -ug "$@" +} + +# install packages +# +# this requires following sudo rules: +# - poldek -q --update --upa +install_packages() { + # sync poldek indexes once per invocation + if [ -z "$package_indexes_updated" ]; then + update_shell_title "poldek: update indexes" + $POLDEK_CMD -q --update --upa --mo=nodesc + package_indexes_updated=true + fi + + update_shell_title "install packages: $*" + poldek_install "$@" && return + + # retry install, install packages one by one + # this is slower one + local rc=0 package + for package in $*; do + package=$(depspecname $package) + update_shell_title "install package: $package" + poldek_install "$package" || rc=$? + done + return $rc +} + +uninstall_packages() { + update_shell_title "uninstall packages: $*" + $POLDEK_CMD --noask --nofollow -ev "$@" +} + spawn_sub_builder() { package_name="${1}" update_shell_title "spawn_sub_builder $package_name" @@ -1907,25 +2045,25 @@ display_branches() { git branch -r 2>/dev/null | grep "^ ${REMOTE_PLD}" | grep -v ${REMOTE_PLD}/HEAD | sed "s#^ *${REMOTE_PLD}/##" | xargs } -# checks a given list of packages/files/provides agains current rpmdb. +# checks a given list of packages/files/provides against current rpmdb. # outputs all dependencies which current rpmdb doesn't satisfy. # input can be either STDIN or parameters _rpm_prov_check() { - local DEPS + local deps out if [ $# -gt 0 ]; then - DEPS="$@" + deps="$@" else - DEPS=$(cat) + deps=$(cat) fi - DEPS=$(LANG=C rpm -q --whatprovides $DEPS 2>&1 | awk '/^(error:|no package provides)/ { print }') + out=$(LC_ALL=C rpm -q --whatprovides $deps 2>&1) # packages - echo "$DEPS" | awk '/^no package provides/ { print $NF }' + echo "$out" | awk '/^no package provides/ { print $NF }' # other deps (files) - echo "$DEPS" | awk -F: '/^error:.*No such file/{o = $2; gsub("^ file ", "", o); print o}' + echo "$out" | sed -rne 's/file (.*): No such file or directory/\1/p' } # checks if given package/files/provides exists in rpmdb. @@ -1940,49 +2078,31 @@ _rpm_cnfl_check() { DEPS=$(cat) fi - LANG=C rpm -q --whatprovides $DEPS 2>/dev/null | awk '!/no package provides/ { print }' + LC_ALL=C LANG=C rpm -q --whatprovides $DEPS 2>/dev/null | awk '!/no package provides/ { print }' } # install deps via information from 'rpm-getdeps' or 'rpm --specsrpm' install_build_requires_rpmdeps() { + local DEPS CNFL if [ "$FETCH_BUILD_REQUIRES_RPMGETDEPS" = "yes" ]; then # TODO: Conflicts list doesn't check versions - local CNFL=$(rpm-getdeps $BCOND $SPECFILE 2> /dev/null | awk '/^\-/ { print $3 } ' | _rpm_cnfl_check | xargs) - local DEPS=$(rpm-getdeps $BCOND $SPECFILE 2> /dev/null | awk '/^\+/ { print $3 } ' | _rpm_prov_check | xargs) + CNFL=$(eval rpm-getdeps $BCOND $RPMOPTS $RPMUSERDEFS $SPECFILE 2> /dev/null | awk '/^\-/ { print $3 } ' | _rpm_cnfl_check | xargs) + DEPS=$(eval rpm-getdeps $BCOND $RPMOPTS $RPMUSERDEFS $SPECFILE 2> /dev/null | awk '/^\+/ { print $3 } ' | _rpm_prov_check | xargs) fi if [ "$FETCH_BUILD_REQUIRES_RPMSPECSRPM" = "yes" ]; then - local CNFL=$(rpm -q --specsrpm --conflicts $BCOND $SPECFILE | awk '{print $1}' | _rpm_cnfl_check | xargs) - local DEPS=$(rpm -q --specsrpm --requires $BCOND $SPECFILE | awk '{print $1}' | _rpm_prov_check | xargs) + CNFL=$(eval rpm -q --specsrpm --conflicts $BCOND $RPMOPTS $RPMUSERDEFS $SPECFILE | awk '{print $1}' | _rpm_cnfl_check | xargs) + DEPS=$(eval rpm -q --specsrpm --requires $BCOND $RPMOPTS $RPM_USERDEFS $SPECFILE | awk '{print $1}' | _rpm_prov_check | xargs) fi - if [ -n "$CNFL" ] || [ -n "$DEPS" ]; then - echo "fetch BuildRequires: install [$DEPS]; remove [$CNFL]" - update_shell_title "poldek: install [$DEPS]; remove [$CNFL]" - $SU_SUDO /usr/bin/poldek -q --update || $SU_SUDO /usr/bin/poldek -q --upa - fi if [ -n "$CNFL" ]; then - update_shell_title "uninstall conflicting packages: $CNFL" - echo "Trying to uninstall conflicting packages ($CNFL):" - $SU_SUDO /usr/bin/poldek --noask --nofollow -ev $CNFL - fi - - while [ "$DEPS" ]; do - update_shell_title "install deps: $DEPS" - echo "Trying to install dependencies ($DEPS):" - local log=.${SPECFILE}_poldek.log - LANG=C $SU_SUDO /usr/bin/poldek --noask --caplookup -uGqQ $DEPS | tee $log - failed=$(awk '/^error:/{a=$2; sub(/^error: /, "", a); sub(/:$/, "", a); print a}' $log) - rm -f $log - local ok - if [ -n "$failed" ]; then - for package in $failed; do - spawn_sub_builder -bb $(depspecname $package) && ok="$ok $package" - done - DEPS="$ok" - else - DEPS="" - fi - done + echo "Uninstall conflicting packages: $CNFL" + uninstall_packages $CNFL + fi + + if [ -n "$DEPS" ]; then + echo "Install dependencies: $DEPS" + install_packages $DEPS + fi } fetch_build_requires() @@ -1997,166 +2117,34 @@ fetch_build_requires() return fi - # XXX is this ugliest code written in human history still needed? - echo "All packages installed by fetch_build_requires() are written to:" - echo "`pwd`/.${SPECFILE}_INSTALLED_PACKAGES" - echo "" - echo "If anything fails, you may get rid of them by executing:" - echo "poldek -e \`cat `pwd`/.${SPECFILE}_INSTALLED_PACKAGES\`" - echo "" - echo > `pwd`/.${SPECFILE}_INSTALLED_PACKAGES - for package_item in $(cat $SPECFILE | grep -B100000 ^%changelog|grep -v ^#|grep BuildRequires|grep -v ^-|sed -e "s/^.*BuildRequires://g"|awk '{print $1}'|sed -e s,perl\(,perl-,g -e s,::,-,g -e s,\(.*\),,g -e s,%{,,g -e s,},,g|grep -v OpenGL-devel|sed -e s,sh-utils,coreutils,g -e s,fileutils,coreutils,g -e s,textutils,coreutils,g -e s,kgcc_package,gcc,g -e s,\),,g) - do - package_item=$(echo $package_item|sed -e s,rpmbuild,rpm-build,g |sed -e s,__perl,perl,g |sed -e s,gasp,binutils-gasp,g -e s,binutils-binutils,binutils,g -e s,apxs,apache,g|sed -e s,apache\(EAPI\)-devel,apache-devel,g -e s,kernel-headers\(netfilter\),kernel-headers,g -e s,awk,mawk,g -e s,mmawk,mawk,g -e s,motif,openmotif,g -e s,openopenmotif,openmotif,g) - GO="yes" - package=$(basename "$package_item"|sed -e "s/}$//g") - COND_ARCH_TST=$(cat $SPECFILE|grep -B1 BuildRequires|grep -B1 $package|grep ifarch|sed -e "s/^.*ifarch//g") - mach=$(uname -m) - - COND_TST=`cat $SPECFILE|grep BuildRequires|grep "$package"` - if `echo $COND_TST|grep -q '^BuildRequires:'`; then - if [ "$COND_ARCH_TST" != "" ] && [ "`echo $COND_ARCH_TST|sed -e "s/i.86/ix86/g"`" != "`echo $mach|sed -e "s/i.86/ix86/g"`" ]; then - GO="yes" - fi - # bcond: - else - COND_NAME=`echo $COND_TST|sed -e s,:BuildRequires:.*$,,g` - GO="" - # %{without} - if `echo $COND_TST|grep -q 'without_'`; then - COND_NAME=`echo $COND_NAME|sed -e s,^.*without_,,g` - if `echo $COND_TST|grep -q !`; then - COND_STATE="with" - else - COND_STATE="wout" - fi - COND_WITH=`echo $AVAIL_BCONDS_WITH|grep "<$COND_NAME>"` - COND_WITHOUT=`echo $AVAIL_BCONDS_WITHOUT|grep "<$COND_NAME>"` - if [ -n "$COND_WITHOUT" ] || [ -z "$COND_WITH" ]; then - COND_ARGV="wout" - else - COND_ARGV="with" - fi - # %{with} - elif `echo $COND_TST|grep -q 'with_'`; then - COND_NAME=`echo $COND_NAME|sed -e s,^.*with_,,g` - if `echo $COND_TST|grep -q !`; then - COND_STATE="wout" - else - COND_STATE="with" - fi - COND_WITH=`echo $AVAIL_BCONDS_WITH|grep "<$COND_NAME>"` - COND_WITHOUT=`echo $AVAIL_BCONDS_WITHOUT|grep "<$COND_NAME>"` - if [ -n "$COND_WITH" ] || [ -z "$COND_WITHOUT" ]; then - COND_ARGV="with" - else - COND_ARGV="wout" - fi - fi - RESULT="${COND_STATE}-${COND_ARGV}" - case "$RESULT" in - "with-wout" | "wout-with" ) - GO="" - ;; - "wout-wout" | "with-with" ) - GO="yes" - ;; - * ) - echo "Action '$RESULT' was not defined for package '$package_item'" - GO="yes" - ;; - esac - fi + die "need rpm-getdeps tool" +} - if [ "$GO" = "yes" ]; then - if [ "`rpm -q $package|sed -e "s/$package.*/$package/g"`" != "$package" ]; then - echo "Testing if $package has subrequirements..." - run_poldek -t -i $package --dumpn=".$package-req.txt" - if [ -f ".$package-req.txt" ]; then - for package_name in `cat ".$package-req.txt"|grep -v ^#` - do - if [ "$package_name" = "$package" ]; then - echo "Installing BuildRequired package:\t$package_name" - update_shell_title "Installing BuildRequired package: ${package_name}" - install_required_packages $package - else - echo "Installing (sub)Required package:\t$package_name" - update_shell_title "Installing (sub)Required package: ${package_name}" - install_required_packages $package_name - fi - case $? in - 0) - INSTALLED_PACKAGES="$package_name $INSTALLED_PACKAGES" - echo $package_name >> `pwd`/.${SPECFILE}_INSTALLED_PACKAGES - ;; - *) - echo "Attempting to run spawn sub - builder..." - echo "Package installation failed:\t$package_name" - run_sub_builder $package_name - if [ $? -eq 0 ]; then - install_required_packages $package_name - case $? in - 0) - INSTALLED_PACKAGES="$package_name $INSTALLED_PACKAGES" - echo $package_name >> `pwd`/.${SPECFILE}_INSTALLED_PACKAGES - ;; - *) - NOT_INSTALLED_PACKAGES="$package_name $NOT_INSTALLED_PACKAGES" - ;; - esac - fi - ;; - esac - done - rm -f ".$package-req.txt" - else - echo "Attempting to run spawn sub - builder..." - echo "Package installation failed:\t$package" - run_sub_builder $package - if [ $? -eq 0 ]; then - install_required_packages $package - case $? in - 0) - INSTALLED_PACKAGES="$package_name $INSTALLED_PACKAGES" - echo $package_name >> `pwd`/.${SPECFILE}_INSTALLED_PACKAGES - ;; - *) - NOT_INSTALLED_PACKAGES="$package_name $NOT_INSTALLED_PACKAGES" - ;; - esac - fi - fi - else - echo "Package $package is already installed. BuildRequirement satisfied." - fi - fi - done - if [ "$NOT_INSTALLED_PACKAGES" != "" ]; then - echo >&2 "Unable to install following packages and their dependencies:" - for pkg in "$NOT_INSTALLED_PACKAGES" - do - echo $pkg - done - remove_build_requires - exit 8 - fi +init_repository() { + local remoterepo=$1 + local localrepo=$2 + + if [ ! -e $localrepo ]; then + git clone $IPOPT -o $REMOTE_PLD ${GIT_SERVER}/$remoterepo $localrepo + git --git-dir=$localrepo/.git remote set-url --push $REMOTE_PLD ssh://${GIT_PUSH}/$remoterepo + fi } init_rpm_dir() { - local TOP_DIR=$(eval $RPM $RPMOPTS --eval '%{_topdir}') - local rpmdir=$(eval $RPM $RPMOPTS --eval '%{_rpmdir}') - local buildir=$(eval $RPM $RPMOPTS --eval '%{_builddir}') - local srpmdir=$(eval $RPM $RPMOPTS --eval '%{_srcrpmdir}') + local TOP_DIR=$(eval $RPM $RPMOPTS $RPMUSERDEFS --eval '%{_topdir}') + local rpmdir=$(eval $RPM $RPMOPTS $RPMUSERDEFS --eval '%{_rpmdir}') + local buildir=$(eval $RPM $RPMOPTS $RPMUSERDEFS --eval '%{_builddir}') + local srpmdir=$(eval $RPM $RPMOPTS $RPMUSERDEFS --eval '%{_srcrpmdir}') + local TEMPLATES=template-specs local tmp echo "Initializing rpm directories to $TOP_DIR from $GIT_SERVER" mkdir -p $TOP_DIR $rpmdir $buildir $srpmdir cd "$TOP_DIR" - if [ ! -e ../rpm-build-tools ]; then - git clone ${GIT_SERVER}/${PACKAGES_DIR}/rpm-build-tools.git ../rpm-build-tools - fi - for a in adapter builder fetchsrc_request compile repackage; do + init_repository ${PACKAGES_DIR}/rpm-build-tools.git ../rpm-build-tools + init_repository projects/$TEMPLATES ../$TEMPLATES + for a in builder fetchsrc_request compile repackage; do ln -sf ../rpm-build-tools/${a}.sh $a done for a in md5; do @@ -2185,12 +2173,15 @@ if [ $# = 0 ]; then exit 1 fi +# stuff global $BUILDER_OPTS from env as args +if [ "$BUILDER_OPTS" ]; then + set -- "$BUILDER_OPTS" "$@" +fi + while [ $# -gt 0 ]; do case "${1}" in -4|-6) - # NOTE: we should be fetcher specific, like fille WGET_OPTS, but - # unfortunately $GETURI is already formed - GETURI="$GETURI $1" + IPOPT="${1}" shift ;; -5 | --update-md5) @@ -2379,33 +2370,6 @@ while [ $# -gt 0 ]; do -su | --source-urls) COMMAND="list-sources-urls" shift ;; - -Tvs | --tag-version-stable ) - COMMAND="tag" - TAG="STABLE" - TAG_VERSION="yes" - shift;; - -Ts | --tag-stable ) - COMMAND="tag" - TAG="STABLE" - TAG_VERSION="no" - shift;; - -Tv | --tag-version ) - COMMAND="tag" - TAG="" - TAG_VERSION="yes" - shift;; - -Tp | --tag-prefix ) - TAG_PREFIX="$2" - shift 2;; - -tt | --test-tag ) - TEST_TAG="yes" - shift;; - -T | --tag ) - COMMAND="tag" - shift - TAG="$1" - TAG_VERSION="no" - shift;; -ir | --integer-release-only ) INTEGER_RELEASE="yes" shift;; @@ -2418,7 +2382,7 @@ while [ $# -gt 0 ]; do -Upi | --update-poldek-indexes ) UPDATE_POLDEK_INDEXES="yes" shift ;; - --init-rpm-dir) + --init-rpm-dir|--init) COMMAND="init_rpm_dir" shift ;; -u | --try-upgrade ) @@ -2434,16 +2398,16 @@ while [ $# -gt 0 ]; do MACRO="${1}" shift if echo "${MACRO}" | grep -q '\W'; then - RPMOPTS="${RPMOPTS} --define \"${MACRO}\"" + RPMUSERDEFS="${RPMUSERDEFS} --define \"${MACRO}\"" else VALUE="${1}" shift - RPMOPTS="${RPMOPTS} --define \"${MACRO} ${VALUE}\"" + RPMUSERDEFS="${RPMUSERDEFS} --define \"${MACRO} ${VALUE}\"" fi ;; --alt_kernel) shift - RPMOPTS="${RPMOPTS} --define \"alt_kernel $1\"" + RPMOPTS="${RPMOPTS} --define \"alt_kernel $1\" --define \"build_kernels $1\"" shift ;; --short-circuit) @@ -2475,7 +2439,6 @@ while [ $# -gt 0 ]; do GIT_PUSH=${PLD_GIT_PUSH} PACKAGES_DIR=${PLD_PACKAGES_DIR} DISTFILES_SERVER=${PLD_DISTFILES_SERVER} - ATTICDISTFILES_SERVER=${PLD_ATTICDISTFILES_SERVER} ;; --git-tld) shift @@ -2483,10 +2446,13 @@ while [ $# -gt 0 ]; do GIT_PUSH=${TLD_GIT_PUSH} PACKAGES_DIR=${TLD_PACKAGES_DIR} DISTFILES_SERVER=${TLD_DISTFILES_SERVER} - ATTICDISTFILES_SERVER=${TLD_ATTICDISTFILES_SERVER} ;; --pkgrev) - COMMAND="set_pkgrev" + SETPKGREV="true" + COMMAND="build-source" + shift;; + -lp) + COMMAND="list_pkgrev" shift;; -*) Exit_error err_invalid_cmdline "$1" @@ -2511,7 +2477,6 @@ if ! git ls-remote --heads ${GIT_SERVER}/${PACKAGES_DIR}/${ASSUMED_NAME} 1>/dev/ GIT_PUSH=${PLD_GIT_PUSH} PACKAGES_DIR=${PLD_PACKAGES_DIR} DISTFILES_SERVER=${PLD_DISTFILES_SERVER} - ATTICDISTFILES_SERVER=${PLD_ATTICDISTFILES_SERVER} fi if [ "$CVSTAG" ]; then @@ -2544,6 +2509,9 @@ else NICE_COMMAND="nice -n ${DEF_NICE_LEVEL}" fi +# see time(1) for output format that could be used +TIME_COMMAND="time -p" + update_shell_title "$COMMAND" case "$COMMAND" in "show_bconds") @@ -2623,35 +2591,6 @@ case "$COMMAND" in fi fi - # ./builder -bs test.spec -r AC-branch -Tp auto-ac- -tt - if [ -n "$TEST_TAG" ]; then - local TAGVER=` - make_tagver` - tag_exist $TAGVER || [ $TAGVER = $CVSTAG ] || Exit_error err_tag_exists $TAGVER - # check also tags created in CVS - local TAGVER_CVS=$(echo $TAGVER | tr '[.@]' '[_#]') - local CVSTAG_CVS=$(echo $CVSTAG | tr '[.@]' '[_#]') - tag_exist $TAGVER_CVS || [ $TAGVER_CVS = $CVSTAG_CVS ] \ - || Exit_error err_tag_exists $TAGVER_CVS - # - do not allow to build from HEAD when XX-branch exists - TREE_PREFIX=$(echo "$TAG_PREFIX" | sed -e 's#^auto/\([a-zA-Z]\+\)/.*#\1#g') - if [ "$TAGVER" != "$CVSTAG" -a "$TAGVER_CVS" != "$CVSTAG" -a "$TREE_PREFIX" != "$TAG_PREFIX" ]; then - TAG_BRANCH="${TREE_PREFIX}-branch" - if [ -n "$DEPTH" ]; then - cmd_branches="git ls-remote --heads" - ref_prefix=refs/heads - else - cmd_branches="git show-ref" - ref_prefix=refs/remotes/${REMOTE_PLD} - fi - TAG_STATUS=$($cmd_branches | grep -i "${ref_prefix}/$TAG_BRANCH$" | cut -c'-40') - if [ -n "$TAG_STATUS" -a "$TAG_STATUS" != $(git rev-parse "$CVSTAG") ]; then - Exit_error err_branch_exists "$TAG_STATUS" - fi - fi - - fi - if [ -n "$NOSOURCE0" ] ; then SOURCES=`echo $SOURCES | xargs | sed -e 's/[^ ]*//'` fi @@ -2719,6 +2658,7 @@ case "$COMMAND" in fi get_files $SOURCES $PATCHES check_md5 $SOURCES + fetch_build_requires ;; "update_md5" ) init_builder @@ -2734,21 +2674,6 @@ case "$COMMAND" in fi update_md5 $SOURCES $PATCHES ;; - "tag" ) - NOURLS=1 - NODIST="yes" - init_builder - if [ -z "$SPECFILE" ]; then - Exit_error err_no_spec_in_cmdl - fi - - parse_spec - if [ ! -d .git ]; then - echo "No git reposiotory" >&2 - exit 101 - fi - tag_files - ;; "mr-proper" ) mr_proper ;; @@ -2831,16 +2756,14 @@ case "$COMMAND" in "version" ) echo "$VERSIONSTRING" ;; - "set_pkgrev" ) + "list_pkgrev" ) init_builder if [ -z "$SPECFILE" ]; then Exit_error err_no_spec_in_cmdl fi get_spec > /dev/null parse_spec - set_bconds_values - display_bconds - set_pkgrev + list_pkgrev ;; esac if [ -f "`pwd`/.${SPECFILE}_INSTALLED_PACKAGES" -a "$REMOVE_BUILD_REQUIRES" != "" ]; then