client/make-request.sh

   1 #!/bin/sh
   2 # vim:noet:ts=4:sw=4
   3 VERSION=1.89
   4
   5 # prevent "*" from being expanded in builders var
   6 set -f
   7
   8 builders=
   9 with=
  10 without=
  11 flags=
  12 command=
  13 command_flags=
  14 gpg_opts=
  15 default_branch='HEAD'
  16 dist=
  17 url=
  18 no_depend=no
  19 verbose=no
  20 pkgrevision=no
  21 requester_override=
  22 relup=no
  23
  24 if [ -x /usr/bin/python3 ]; then
  25         send_mode="python"
  26 else
  27         echo "No python3 present, aborting"
  28         exit 1
  29 fi
  30
  31 if [ -n "$HOME_ETC" ]; then
  32         USER_CFG=$HOME_ETC/.requestrc
  33 else
  34         USER_CFG=$HOME/.requestrc
  35 fi
  36
  37 if [ ! -f "$USER_CFG" ]; then
  38         echo "Creating config file $USER_CFG. You *must* edit it."
  39         cat > $USER_CFG <<EOF
  40 priority=2
  41 requester=deviloper@tld-linux.org
  42 default_key=deviloper@tld-linux.org
  43 send_mode="$send_mode"
  44 url="$url"
  45 gpg_opts=""
  46 dist=ti
  47 url="https://giant.tld-linux.org:1241/"
  48
  49 # defaults:
  50 f_upgrade=yes
  51 EOF
  52 exit
  53 fi
  54
  55 if [ -f "$USER_CFG" ]; then
  56         . $USER_CFG
  57         # legacy fallback
  58         if [ "${distro:+set}" = "set" ]; then
  59                 dist=$distro
  60         fi
  61 fi
  62
  63 # internal options, not to be overriden
  64 specs=
  65 upgrade_macros=no
  66 upgrade_scripts=no
  67 cr=$(printf "\r")
  68
  69 # Set colors
  70 c_star=$(tput setaf 2)
  71 c_red=$(tput setaf 1)
  72 c_norm=$(tput op)
  73 msg() {
  74         echo >&2 "${c_star}*${c_norm} $*"
  75 }
  76 red() {
  77         echo "${c_red}$*${c_norm}"
  78 }
  79
  80 die() {
  81         echo >&2 "$0: $*"
  82         exit 1
  83 }
  84
  85 send_request() {
  86         if [ -n "$wait" ]; then
  87                 msg "Waiting $wait seconds before sending request"
  88                 sleep $wait
  89                 msg "Wait has ended, proceeding!"
  90         fi
  91
  92         msg "Sending using HTTP mode to $url"
  93         cat - | python3 -c '
  94 import sys, socket, ssl
  95 from urllib import request
  96
  97 try:
  98     _create_unverified_https_context = ssl._create_unverified_context
  99 except AttributeError:
 100     pass
 101 else:
 102     ssl._create_default_https_context = _create_unverified_https_context
 103
 104 try:
 105         data = sys.stdin.read().encode("utf-8")
 106         url = sys.argv[1]
 107         socket.setdefaulttimeout(30)
 108         req = request.Request(url, data=data)
 109         f = request.urlopen(req)
 110         f.close()
 111 except Exception as e:
 112         print("Problem while sending request via HTTP: %s: %s" % (url, e), file=sys.stderr)
 113         sys.exit(1)
 114 print("Request queued via HTTP.", file=sys.stdout)
 115 ' "$url"
 116 }
 117
 118 # htmlspecialchars: escape <, > and &
 119 hsc() {
 120         local input=$1
 121         printf "%s\n" "$input" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
 122 }
 123
 124 # get package revision for specs
 125 get_pkgrevision() {
 126         local pkgrev pkg spec rpmdir
 127
 128         rpmdir=$(rpm -E %_topdir)
 129         for pkg in "$@"; do
 130                 cd $rpmdir
 131                 # strip branches
 132                 pkg=${pkg%:*}
 133                 # strip .spec extension
 134                 pkg=${pkg%.spec}
 135                 wget http://pkgrevs.tld-linux.org/list/$pkg -O /tmp/$pkg.tags 1>/dev/null 2>&1
 136                 pkgrev=$(head -n 1 /tmp/$pkg.tags)
 137                 rm -f /tmp/$pkg.tags
 138                 if [ "x$pkgrev" = "x" ]; then
 139                         echo "$pkg.spec"
 140                 else
 141                         echo "$pkg.spec:$pkgrev"
 142                 fi
 143         done
 144 }
 145
 146 relup() {
 147         local script=$(dirname $(rpm -E %_topdir))/rpm-build-tools/relup.sh
 148         $script -u -i "$@"
 149 }
 150
 151 usage() {
 152         cat <<EOF
 153 Usage: make-request.sh [OPTION] ... [SPECFILE] ....
 154
 155 Mandatory arguments to long options are mandatory for short options too.
 156
 157       --config-file /path/to/config/file
 158             Source additional config file (after $USER_CFG), useful when
 159             when sending build requests to Ac/Th from the same account
 160       -a
 161             Try to use latest auto-tag for the spec when building
 162             WARNING: This will checkout new files to your packages dir
 163       -b 'BUILDER BUILDER ...',  --builder='BUILDER BUILDER ...'
 164             Sends request to given builders (in 'version-arch' format)
 165       --with VALUE, --without VALUE
 166             Build package with(out) a given bcond
 167       --kernel VALUE
 168             set alt_kernel to VALUE
 169       --target VALUE
 170             set --target to VALUE
 171       -D "NAME VALUE"|--define "NAME VALUE"
 172             define macro named NAME with value VALUE
 173       -s BUILD_ID, --skip BUILD_ID[,BUILD_ID][,BUILD_ID]
 174             mark build ids on src builder to be skipped (instructs srcbuilder to create 'skipme' file)
 175       --branch VALUE
 176             specify default branch for specs in request
 177       -t, --test-build
 178             Performs a 'test-build'. Package will be uploaded to hidden .test-builds/
 179             ftp tree and won't be upgraded on builders.
 180       -r, --ready-build
 181             Preforms a 'ready' build. Package will be built and uploaded to test/ ftp tree
 182             (and later moved by release manager staff to ready/ and main ftp tree)
 183       -u, --upgrade
 184             Forces package upgrade (for use with -c or -q, not -t)
 185       --relup
 186             Bump package release, see also --relup
 187       -m, --message
 188             Set commit message for relup
 189       -n, --no-upgrade
 190             Disables package upgrade (for use with -r)
 191       -ni, --no-install-br
 192             Do not install missing BuildRequires (--nodeps)
 193       -nd, --no-depend
 194             Do not add dependency of build jobs, each job in batch runs itself
 195       -j, --jobs
 196             Number of parallel jobs for single build
 197       -f, --flag
 198       -d, --dist DISTRIBUTION_ID
 199             Specify value for \$dist
 200       -df,  --distfiles-fetch[-request] PACKAGE
 201             Send distfiles request to fetch sources for PACKAGE
 202       -cf, --command-flag
 203             Not yet documented
 204       -c, --command
 205             Executes a given command on builders (prepended to build jobs if build jobs included)
 206       -C, --post-command
 207             Executes a given command on builders (appended to build jobs if build jobs included)
 208       --test-remove-pkg
 209             shortcut for --command poldek -evt ARGS
 210       --remove-pkg
 211             shortcut for --command poldek -ev --noask ARGS
 212       --upgrade-pkg
 213             shortcut for --command poldek --up -Uv ARGS
 214       --pull
 215             Updates builders infrastructure (outside chroot)
 216       --update-macros
 217             Updates rpm-build-macros on src builder
 218       --update-scripts
 219             Updates ~/rpm/rpm-build-tools on builder
 220       --requester username
 221             Override the requester
 222       -q
 223             shortcut for --command rpm -q ARGS
 224       -g, --gpg-opts "opts"
 225             Pass additional options to gpg binary
 226       -p, --priority VALUE
 227             sets request priority (default 2)
 228       -w SECONDS
 229             Wait SECONDS before sending actual request. Note: gpg passphrase is still asked immediately.
 230             This may be useful if you just commited package and want to send it
 231             for test build after distfiles has fetched the file.
 232       -h, --help
 233             Displays this help message
 234       -v
 235             Verbose. Print some more debug on screen
 236 EOF
 237         exit 0
 238 }
 239
 240 # validate distro, set $dist
 241 set_dist() {
 242         case "$1" in
 243         ti)
 244                 ;;
 245         ti-dev)
 246                 ;;
 247         *)
 248                 die "dist \`$1' not known"
 249                 ;;
 250         esac
 251
 252         dist=$1
 253 }
 254
 255 while [ $# -gt 0 ]; do
 256         case "$1" in
 257                 -d | --dist | --distro)
 258                         set_dist $2
 259                         shift
 260                         ;;
 261
 262                 --config-file)
 263                         [ -f "$2" ] && . $2 || die "Config file not found"
 264                         shift
 265                         ;;
 266
 267                 --builder | -b)
 268                         for b in $2; do
 269                                 builders="$builders ${b%:*}"
 270                         done
 271                         shift
 272                         ;;
 273
 274                 -a)
 275                         pkgrevision=yes
 276                         ;;
 277
 278                 -m)
 279                         shift
 280                         message=$1
 281                         ;;
 282
 283                 --relup)
 284                         relup=yes
 285                         ;;
 286
 287                 --with)
 288                         with="$with $(echo "$2" | tr ',' ' ')"
 289                         shift
 290                         ;;
 291
 292                 --without)
 293                         without="$without $(echo "$2" | tr ',' ' ')"
 294                         shift
 295                         ;;
 296
 297                 --test-build | -t)
 298                         build_mode=test
 299                         f_upgrade=no
 300                         ;;
 301
 302                 --kernel)
 303                         kernel=$2
 304                         shift
 305                         ;;
 306
 307                 --target)
 308                         target=$2
 309                         shift
 310                         ;;
 311
 312                 -D|--define)
 313                         value=${2#* }
 314                         name=${2%% *}
 315                         define="$define$cr$name=$value"
 316                         shift
 317                         ;;
 318
 319                 -s|--skip)
 320                         skip="$2"
 321                         shift
 322                         ;;
 323
 324                 --branch)
 325                         branch=$2
 326                         shift
 327                         ;;
 328
 329                 --priority | -p)
 330                         priority=$2
 331                         shift
 332                         ;;
 333
 334                 --ready-build | -r)
 335                         build_mode=ready
 336                         ;;
 337
 338                 --upgrade | -u)
 339                         f_upgrade=yes
 340                         ;;
 341
 342                 --no-upgrade | -n)
 343                         f_upgrade=no
 344                         ;;
 345
 346                 --no-depend | -nd)
 347                         no_depend=yes
 348                         ;;
 349
 350                 --no-install-br | -ni)
 351                         flags="$flags no-install-br"
 352                         ;;
 353
 354                 -j | --jobs)
 355                         jobs="$2"
 356                         shift
 357                         ;;
 358
 359                 -j*)
 360                         jobs="${1#-j}"
 361                         ;;
 362
 363                 -w)
 364                         wait="$2"
 365                         shift
 366                         ;;
 367
 368                 -v)
 369                         verbose=yes
 370                         ;;
 371
 372                 --flag | -f)
 373                         flags="$flags $2"
 374                         shift
 375                         ;;
 376
 377                 --command-flags | -cf)
 378                         command_flags="$2"
 379                         shift
 380                         ;;
 381
 382                 --command | -c)
 383                         command="$2"
 384                         if [ "$command" = - ]; then
 385                                 echo >&2 "Reading command from STDIN"
 386                                 echo >&2 "---"
 387                                 command=$(cat)
 388                                 echo >&2 "---"
 389                         fi
 390                         shift
 391                         ;;
 392                 --post-command | -C)
 393                         post_command="$2"
 394                         if [ "$post_command" = - ]; then
 395                                 echo >&2 "Reading post_command from STDIN"
 396                                 echo >&2 "---"
 397                                 post_command=$(cat)
 398                                 echo >&2 "---"
 399                         fi
 400                         shift
 401                         ;;
 402                 --test-remove-pkg)
 403                         command="poldek -evt $2"
 404                         f_upgrade=no
 405                         shift
 406                         ;;
 407                 --remove-pkg)
 408                         command="for a in $2; do poldek -ev --noask \$a; done"
 409                         f_upgrade=no
 410                         shift
 411                         ;;
 412                 --upgrade-pkg|-Uhv)
 413                         command="poldek --up; poldek -uv $2"
 414                         f_upgrade=no
 415                         shift
 416                         ;;
 417                 -q)
 418                         command="rpm -q $2"
 419                         f_upgrade=no
 420                         shift
 421                         ;;
 422
 423                 --pull)
 424                         command_flags="no-chroot"
 425                         command="git pull"
 426                         f_upgrade=no
 427                         ;;
 428
 429                 --update-macros)
 430                         upgrade_macros="yes"
 431                         ;;
 432
 433                 --update-scripts)
 434                         upgrade_scripts='yes'
 435                         ;;
 436
 437                 --gpg-opts | -g)
 438                         gpg_opts="$2"
 439                         shift
 440                         ;;
 441
 442                 --help | -h)
 443                         usage
 444                         ;;
 445
 446                 --requester)
 447                         requester_override="$2"
 448                         shift
 449                         ;;
 450
 451                 -*)
 452                         die "unknown knob: $1"
 453                         ;;
 454
 455                 *:* | *)
 456                         specs="$specs $1"
 457                         ;;
 458         esac
 459         shift
 460 done
 461
 462 case "$dist" in
 463 ti)
 464         default_builders="ti-*"
 465         url="https://giant.tld-linux.org:1241/"
 466         control_url="http://src.tld-linux.org/~builderti"
 467         ;;
 468 ti-dev)
 469         default_builders="ti-dev-*"
 470         url="https://giant.tld-linux.org:1242/"
 471         control_url="http://src.tld-linux.org/~buildertidev"
 472         ;;
 473 *)
 474         die "dist \`$dist' not known"
 475         ;;
 476 esac
 477
 478 # need to do this after dist selection
 479 if [ "$skip" ]; then
 480         skip=$(skip="$skip" control_url="$control_url" python3 -c '
 481 import sys
 482 import gzip
 483 import re
 484 import os
 485 import string
 486 from urllib import request
 487 from io import BytesIO
 488 from xml.dom import minidom
 489
 490 skip = os.environ.get("skip").split(",");
 491 control_url = os.environ.get("control_url")
 492
 493 print("* Check queue_id-s against %s" % control_url, file=sys.stderr)
 494
 495 try:
 496         headers = { "Cache-Control": "no-cache", "Pragma": "no-cache" }
 497         req = request.Request(url=control_url + "/queue.gz", headers=headers)
 498         f = request.urlopen(req)
 499 except Exception as e:
 500         print("Fetch error %s: %s" % (control_url + "/queue.gz", e), file=sys.stderr)
 501         sys.exit(1)
 502
 503 sio = BytesIO()
 504 sio.write(f.read())
 505 f.close()
 506 sio.seek(0)
 507 f = gzip.GzipFile(fileobj = sio)
 508
 509 xml = re.compile("(<queue>.*?</queue>)", re.DOTALL).match(f.read().decode("utf-8")).group(1)
 510 d = minidom.parseString(xml)
 511
 512 q = []
 513 for c in d.documentElement.childNodes:
 514         if c.nodeName != "group":
 515                 continue
 516         q.append(c.attributes["id"].value)
 517
 518 err = 0
 519 for s in skip:
 520         if s not in q:
 521                 print("- Check %s: ERROR: Not valid queue-id" % s, file=sys.stderr)
 522                 err = 1
 523         else:
 524                 print("- Check %s: OK" % s, file=sys.stderr)
 525 if err == 1:
 526         sys.exit(1)
 527 print(",".join(skip))
 528 ') || exit $?
 529         f_upgrade=no
 530         build_mode=test
 531         priority=-1
 532         command="skip:$skip"
 533         command_flags="no-chroot"
 534         builders="$dist-src"
 535 fi
 536
 537 branch=${branch:-$default_branch}
 538
 539 specs=`for s in $specs; do
 540         case "$s" in
 541         ^)
 542                 # skip marker - pass it along
 543                 echo $s
 544                 ;;
 545         *:*) # package name with branch
 546                 _specfile=$(basename ${s%:*})
 547                 echo ${_specfile%.spec}.spec:${s##*:}
 548                 ;;
 549         *) # just package name
 550                 echo $(basename ${s%.spec}).spec:$branch
 551                 ;;
 552         esac
 553 done`
 554
 555 if [ "$relup" = "yes" ]; then
 556         msg "Auto relup enabled"
 557         relup ${message:+-m "$message"} $specs
 558 fi
 559
 560 if [ "$pkgrevision" = "yes" ]; then
 561         msg "Package revision build enabled"
 562         specs=$(get_pkgrevision $specs)
 563 fi
 564
 565 if [ "$upgrade_macros" = "yes" ]; then
 566         command="poldek --up; poldek -uv rpm-build-macros"
 567         builders="$dist-src"
 568         f_upgrade=no
 569         build_mode=test
 570 fi
 571
 572 if [ "$upgrade_scripts" = "yes" ]; then
 573         command="cd ~/rpm/rpm-build-tools && git pull"
 574         command_flags=as-builder
 575         builders="$dist-src"
 576         f_upgrade=no
 577         build_mode=test
 578 fi
 579
 580 if [[ "$requester" != *@* ]] ; then
 581         requester="$requester@tld-linux.org"
 582 fi
 583
 584 if [ -z "$builders" ] ; then
 585         builders="$default_builders"
 586 fi
 587
 588 if [ "$f_upgrade" = "yes" ] ; then
 589         flags="$flags upgrade"
 590 fi
 591
 592 if [ "$build_mode" = "test" ] ; then
 593         if [ "$f_upgrade" = "yes" ] ; then
 594                 die "--upgrade and --test-build are mutually exclusive"
 595         fi
 596         flags="$flags test-build"
 597 fi
 598
 599 if [ -z "$build_mode" ] ; then
 600         # missing build mode, builders go crazy when you proceed"
 601         die "please specify build mode"
 602 fi
 603
 604
 605 ok=
 606 for s in $specs; do
 607         ok=1
 608 done
 609
 610 if [ -z "$specs" -a -z "$command" ]; then
 611         die "no packages to build or command to invoke specified"
 612 fi
 613
 614 id=$(uuidgen)
 615
 616 gen_req() {
 617         echo "<group id='$id' no='0' flags='$flags'>"
 618         echo "  <time>$(date +%s)</time>"
 619         msg "Using priority $priority"
 620         echo "  <priority>$priority</priority>"
 621         if [ -n "$jobs" ]; then
 622                 msg "Using jobs $jobs"
 623                 echo "  <maxjobs>$jobs</maxjobs>"
 624         fi
 625         msg "Using URL $url"
 626
 627         if [ "$build_mode" = "ready" ]; then
 628                 msg "Build mode: $(tput setaf 2)$build_mode$c_norm"
 629         else
 630                 msg "Build mode: $(tput setaf 3)$build_mode$c_norm"
 631         fi
 632
 633         if [ -n "$requester_override" ] ; then
 634                 echo "  <requester>$requester_override</requester>"
 635         fi
 636
 637         msg "Queue-ID: $id"
 638         echo
 639
 640         # job to depend on
 641         local depend=
 642         local b i=1 val
 643         local name branch builders_xml
 644
 645         for b in $builders; do
 646                 msg "Builder: $(red $b)"
 647                 builders_xml="$builders_xml <builder>$b</builder>"
 648         done
 649
 650         if [ "$command" ]; then
 651                 bid=$(uuidgen)
 652                 printf "%s\n" "* Command: $command" >&2
 653                 echo "  <batch id='$bid' depends-on=''>"
 654                 echo "           <command flags='$command_flags'>"
 655                 hsc "$command"
 656                 echo "</command>"
 657                 echo "           <info></info>"
 658                 echo "$builders_xml"
 659                 echo "  </batch>"
 660                 depend=$bid
 661         fi
 662
 663         if [ "$f_upgrade" = "yes" ] ; then
 664                 msg "Upgrade mode: $f_upgrade"
 665         fi
 666
 667         for s in $specs; do
 668                 # skip marker
 669                 if [ "$s" = "^" ]; then
 670                         depend=
 671                         continue
 672                 fi
 673                 if [ "$no_depend" = yes ]; then
 674                         depend=
 675                 fi
 676                 bid=$(uuidgen)
 677                 echo "  <batch id='$bid' depends-on='$depend'>"
 678
 679                 name=$(echo "$s" | sed -e 's|:.*||')
 680                 branch=$(echo "$s" | sed -e 's|.*:||')
 681                 msg "Adding #$i $name:$branch${kernel:+ alt_kernel=$kernel}${target:+ target=$target}${depend:+ depends on $depend}"
 682                 echo "           <spec>$name</spec>"
 683                 echo "           <branch>$branch</branch>"
 684                 echo "           ${kernel:+<kernel>$kernel</kernel>}"
 685                 echo "           ${target:+<target>$target</target>}"
 686
 687                 oIFS=$IFS; IFS=$cr
 688                 for b in $define; do
 689                         [ -z "$b" ] && continue
 690                         value=${b#*=}
 691                         b=${b%%=*}
 692                         echo "           <define name='$(hsc $b)'>$(hsc "$value")</define>"
 693                         msg "- define $b=$value"
 694                 done
 695                 IFS=$oIFS
 696
 697                 echo "           <info></info>"
 698                 echo
 699                 for b in $with; do
 700                         echo "           <with>$b</with>"
 701                 done
 702                 for b in $without; do
 703                         echo "           <without>$b</without>"
 704                 done
 705                 echo
 706                 echo "$builders_xml"
 707                 echo "  </batch>"
 708                 i=$((i+1))
 709
 710                 # let next job depend on previous
 711                 depend=$bid
 712         done
 713
 714         if [ "$post_command" ]; then
 715                 bid=$(uuidgen)
 716                 if [ "$no_depend" = yes ]; then
 717                         depend=
 718                 fi
 719                 printf "%s\n" "* Post-Command: $post_command" >&2
 720                 echo "  <batch id='$bid' depends-on='$depend'>"
 721                 echo "           <command flags='$command_flags'>"
 722                 hsc "$post_command"
 723                 echo "</command>"
 724                 echo "           <info></info>"
 725                 echo "$builders_xml"
 726                 echo "  </batch>"
 727                 depend=$bid
 728         fi
 729
 730         echo "</group>"
 731 }
 732
 733 gen_email () {
 734         # make request first, so the STDERR/STDOUT streams won't be mixed
 735         local tmp req
 736         tmp=$(mktemp)
 737         gen_req > $tmp
 738
 739         if [ "$verbose" = "yes" ]; then
 740                 cat $tmp >&2
 741         fi
 742
 743         cat <<-EOF
 744         From: $requester
 745         To: $builder_email
 746         Subject: build request
 747         Message-Id: <$id@$(hostname)>
 748         X-New-TLD-Builder: request
 749         X-Requester-Version: $VERSION
 750
 751         EOF
 752
 753         gpg --clearsign --default-key $default_key $gpg_opts --output=- $tmp
 754         rm -f $tmp
 755 }
 756
 757 gen_email | send_request