]> TLD Linux GIT Repositories - tld-builder.git/blob - client/make-request.sh
- use builtin urllib instead of urllib3
[tld-builder.git] / client / make-request.sh
1 #!/bin/sh
2 # vim:noet:ts=4:sw=4
3 VERSION=1.89
4
5 # prevent "*" from being expanded in builders var
6 set -f
7
8 builders=
9 with=
10 without=
11 flags=
12 command=
13 command_flags=
14 gpg_opts=
15 default_branch='HEAD'
16 dist=
17 url=
18 no_depend=no
19 verbose=no
20 pkgrevision=no
21 requester_override=
22 relup=no
23
24 if [ -x /usr/bin/python3 ]; then
25         send_mode="python"
26 else
27         echo "No python present, using mail mode"
28         send_mode="mail"
29 fi
30
31 if [ -n "$HOME_ETC" ]; then
32         USER_CFG=$HOME_ETC/.requestrc
33 else
34         USER_CFG=$HOME/.requestrc
35 fi
36
37 if [ ! -f "$USER_CFG" ]; then
38         echo "Creating config file $USER_CFG. You *must* edit it."
39         cat > $USER_CFG <<EOF
40 priority=2
41 requester=deviloper@tld-linux.org
42 default_key=deviloper@tld-linux.org
43 send_mode="$send_mode"
44 url="$url"
45 mailer="/usr/lib/sendmail -t"
46 gpg_opts=""
47 dist=ti
48 url="https://src.tld-linux.org:1241/"
49
50 # defaults:
51 f_upgrade=yes
52 EOF
53 exit
54 fi
55
56 if [ -f "$USER_CFG" ]; then
57         . $USER_CFG
58         # legacy fallback
59         if [ "${distro:+set}" = "set" ]; then
60                 dist=$distro
61         fi
62 fi
63
64 # internal options, not to be overriden
65 specs=
66 df_fetch=no
67 upgrade_macros=no
68 upgrade_scripts=no
69 cr=$(printf "\r")
70
71 # Set colors
72 c_star=$(tput setaf 2)
73 c_red=$(tput setaf 1)
74 c_norm=$(tput op)
75 msg() {
76         echo >&2 "${c_star}*${c_norm} $*"
77 }
78 red() {
79         echo "${c_red}$*${c_norm}"
80 }
81
82 die() {
83         echo >&2 "$0: $*"
84         exit 1
85 }
86
87 send_request() {
88         # switch to mail mode, if no url set
89         [ -z "$url" ] && send_mode="mail"
90
91         if [ -n "$wait" ]; then
92                 msg "Waiting $wait seconds before sending request"
93                 sleep $wait
94                 msg "Wait has ended, proceeding!"
95         fi
96
97         case "$send_mode" in
98         "mail")
99                 msg "Sending using mail mode"
100                 cat - | $mailer
101                 ;;
102         *)
103                 msg "Sending using HTTP mode to $url"
104                 cat - | python3 -c '
105 import sys, socket, ssl
106 from urllib import request
107
108 try:
109     _create_unverified_https_context = ssl._create_unverified_context
110 except AttributeError:
111     pass
112 else:
113     ssl._create_default_https_context = _create_unverified_https_context
114
115 try:
116         data = sys.stdin.read().encode("utf-8")
117         url = sys.argv[1]
118         socket.setdefaulttimeout(30)
119         req = request.Request(url, data=data)
120         f = request.urlopen(req)
121         f.close()
122 except Exception as e:
123         print("Problem while sending request via HTTP: %s: %s" % (url, e), file=sys.stderr)
124         sys.exit(1)
125         print("Request queued via HTTP.", file=sys.stdout)
126 ' "$url"
127                 ;;
128         esac
129 }
130
131 # htmlspecialchars: escape <, > and &
132 hsc() {
133         local input=$1
134         printf "%s\n" "$input" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
135 }
136
137 # simple df_fetcher, based on packages/fetchsrc_request
138 # TODO: tcp (smtp) mode
139 # TODO: adjust for ~/.requestrc config
140 df_fetch() {
141         local specs="$@"
142
143         # Sending by
144         local MAILER='/usr/lib/sendmail'
145         # MAILER='/usr/bin/msmtp'
146         # Sending via
147         local VIA="SENDMAIL"
148         #VIA="localhost"
149         local VIA_ARGS=""
150         #VIA_ARGS="some additional flags"
151         # e.g. for msmtp:
152         # VIA_ARGS='-a gmail'
153         #
154         # DISTFILES EMAIL
155         local DMAIL="df@tld-linux.org"
156
157         local HOST=$(hostname -f)
158         local LOGIN=${requester%@*}
159
160         local SPEC BRANCH
161         for spec in $specs; do
162                 SPEC=$(echo "$spec" | sed -e 's|:.*||')
163                 SPEC=${SPEC%.spec}
164                 BRANCH=$(echo "$spec" | sed -e 's|.*:||')
165                 echo >&2 "Distfiles Request: $SPEC:$BRANCH via $MAILER ${VIA_ARGS:+ ($VIA_ARGS)}"
166                 cat <<-EOF | "$MAILER" -t -i $VIA_ARGS
167                         To: $DMAIL
168                         From: $LOGIN <$LOGIN@$HOST>
169                         Subject: fetchsrc_request notify
170                         X-distfiles-request: yes
171                         X-Login: $LOGIN
172                         X-Package: $SPEC
173                         X-Branch: $BRANCH
174                         X-Flags: force-reply
175
176                         .
177                         EOF
178         done
179 }
180
181 # get package revision for specs
182 get_pkgrevision() {
183         local pkgrev pkg spec rpmdir
184
185         rpmdir=$(rpm -E %_topdir)
186         for pkg in "$@"; do
187                 cd $rpmdir
188                 # strip branches
189                 pkg=${pkg%:*}
190                 # strip .spec extension
191                 pkg=${pkg%.spec}
192                 wget http://pkgrevs.tld-linux.org/list/$pkg -O /tmp/$pkg.tags 1>/dev/null 2>&1
193                 pkgrev=$(head -n 1 /tmp/$pkg.tags)
194                 rm -f /tmp/$pkg.tags
195                 if [ "x$pkgrev" = "x" ]; then
196                         echo "$pkg.spec"
197                 else
198                         echo "$pkg.spec:$pkgrev"
199                 fi
200         done
201 }
202
203 relup() {
204         local script=$(dirname $(rpm -E %_topdir))/rpm-build-tools/relup.sh
205         $script -u -i "$@"
206 }
207
208 usage() {
209         cat <<EOF
210 Usage: make-request.sh [OPTION] ... [SPECFILE] ....
211
212 Mandatory arguments to long options are mandatory for short options too.
213
214       --config-file /path/to/config/file
215             Source additional config file (after $USER_CFG), useful when
216             when sending build requests to Ac/Th from the same account
217       -a
218             Try to use latest auto-tag for the spec when building
219             WARNING: This will checkout new files to your packages dir
220       -b 'BUILDER BUILDER ...',  --builder='BUILDER BUILDER ...'
221             Sends request to given builders (in 'version-arch' format)
222       --with VALUE, --without VALUE
223             Build package with(out) a given bcond
224       --kernel VALUE
225             set alt_kernel to VALUE
226       --target VALUE
227             set --target to VALUE
228       -D "NAME VALUE"|--define "NAME VALUE"
229             define macro named NAME with value VALUE
230       -s BUILD_ID, --skip BUILD_ID[,BUILD_ID][,BUILD_ID]
231             mark build ids on src builder to be skipped (instructs srcbuilder to create 'skipme' file)
232       --branch VALUE
233             specify default branch for specs in request
234       -t, --test-build
235             Performs a 'test-build'. Package will be uploaded to hidden .test-builds/
236             ftp tree and won't be upgraded on builders.
237       -r, --ready-build
238             Preforms a 'ready' build. Package will be built and uploaded to test/ ftp tree
239             (and later moved by release manager staff to ready/ and main ftp tree)
240       -u, --upgrade
241             Forces package upgrade (for use with -c or -q, not -t)
242       --relup
243             Bump package release, see also --relup
244       -m, --message
245             Set commit message for relup
246       -n, --no-upgrade
247             Disables package upgrade (for use with -r)
248       -ni, --no-install-br
249             Do not install missing BuildRequires (--nodeps)
250       -nd, --no-depend
251             Do not add dependency of build jobs, each job in batch runs itself
252       -j, --jobs
253             Number of parallel jobs for single build
254       -f, --flag
255       -d, --dist DISTRIBUTION_ID
256             Specify value for \$dist
257       -df,  --distfiles-fetch[-request] PACKAGE
258             Send distfiles request to fetch sources for PACKAGE
259       -cf, --command-flag
260             Not yet documented
261       -c, --command
262             Executes a given command on builders (prepended to build jobs if build jobs included)
263       -C, --post-command
264             Executes a given command on builders (appended to build jobs if build jobs included)
265       --test-remove-pkg
266             shortcut for --command poldek -evt ARGS
267       --remove-pkg
268             shortcut for --command poldek -ev --noask ARGS
269       --upgrade-pkg
270             shortcut for --command poldek --up -Uv ARGS
271       --pull
272             Updates builders infrastructure (outside chroot)
273       --update-macros
274             Updates rpm-build-macros on src builder
275       --update-scripts
276             Updates ~/rpm/rpm-build-tools on builder
277       --requester username
278             Override the requester
279       -q
280             shortcut for --command rpm -q ARGS
281       -g, --gpg-opts "opts"
282             Pass additional options to gpg binary
283       -p, --priority VALUE
284             sets request priority (default 2)
285       -w SECONDS
286             Wait SECONDS before sending actual request. Note: gpg passphrase is still asked immediately.
287             This may be useful if you just commited package and want to send it
288             for test build after distfiles has fetched the file.
289       -h, --help
290             Displays this help message
291       -v
292             Verbose. Print some more debug on screen
293 EOF
294         exit 0
295 }
296
297 # validate distro, set $dist
298 set_dist() {
299         case "$1" in
300         ti)
301                 ;;
302         ti-dev)
303                 ;;
304         *)
305                 die "dist \`$1' not known"
306                 ;;
307         esac
308
309         dist=$1
310 }
311
312 while [ $# -gt 0 ]; do
313         case "$1" in
314                 -d | --dist | --distro)
315                         set_dist $2
316                         shift
317                         ;;
318
319                 --config-file)
320                         [ -f "$2" ] && . $2 || die "Config file not found"
321                         shift
322                         ;;
323
324                 --builder | -b)
325                         for b in $2; do
326                                 builders="$builders ${b%:*}"
327                         done
328                         shift
329                         ;;
330
331                 -a)
332                         pkgrevision=yes
333                         ;;
334
335                 -m)
336                         shift
337                         message=$1
338                         ;;
339
340                 --relup)
341                         relup=yes
342                         ;;
343
344                 --with)
345                         with="$with $(echo "$2" | tr ',' ' ')"
346                         shift
347                         ;;
348
349                 --without)
350                         without="$without $(echo "$2" | tr ',' ' ')"
351                         shift
352                         ;;
353
354                 --test-build | -t)
355                         build_mode=test
356                         f_upgrade=no
357                         ;;
358
359                 --kernel)
360                         kernel=$2
361                         shift
362                         ;;
363
364                 --target)
365                         target=$2
366                         shift
367                         ;;
368
369                 -D|--define)
370                         value=${2#* }
371                         name=${2%% *}
372                         define="$define$cr$name=$value"
373                         shift
374                         ;;
375
376                 -s|--skip)
377                         skip="$2"
378                         shift
379                         ;;
380
381                 --branch)
382                         branch=$2
383                         shift
384                         ;;
385
386                 --priority | -p)
387                         priority=$2
388                         shift
389                         ;;
390
391                 --ready-build | -r)
392                         build_mode=ready
393                         ;;
394
395                 --upgrade | -u)
396                         f_upgrade=yes
397                         ;;
398
399                 --no-upgrade | -n)
400                         f_upgrade=no
401                         ;;
402
403                 --no-depend | -nd)
404                         no_depend=yes
405                         ;;
406
407                 --no-install-br | -ni)
408                         flags="$flags no-install-br"
409                         ;;
410
411                 -j | --jobs)
412                         jobs="$2"
413                         shift
414                         ;;
415
416                 -j*)
417                         jobs="${1#-j}"
418                         ;;
419
420                 -w)
421                         wait="$2"
422                         shift
423                         ;;
424
425                 -v)
426                         verbose=yes
427                         ;;
428
429                 --flag | -f)
430                         flags="$flags $2"
431                         shift
432                         ;;
433
434                 --command-flags | -cf)
435                         command_flags="$2"
436                         shift
437                         ;;
438
439                 --command | -c)
440                         command="$2"
441                         if [ "$command" = - ]; then
442                                 echo >&2 "Reading command from STDIN"
443                                 echo >&2 "---"
444                                 command=$(cat)
445                                 echo >&2 "---"
446                         fi
447                         shift
448                         ;;
449                 --post-command | -C)
450                         post_command="$2"
451                         if [ "$post_command" = - ]; then
452                                 echo >&2 "Reading post_command from STDIN"
453                                 echo >&2 "---"
454                                 post_command=$(cat)
455                                 echo >&2 "---"
456                         fi
457                         shift
458                         ;;
459                 --test-remove-pkg)
460                         command="poldek -evt $2"
461                         f_upgrade=no
462                         shift
463                         ;;
464                 --remove-pkg)
465                         command="for a in $2; do poldek -ev --noask \$a; done"
466                         f_upgrade=no
467                         shift
468                         ;;
469                 --upgrade-pkg|-Uhv)
470                         command="poldek --up; poldek -uv $2"
471                         f_upgrade=no
472                         shift
473                         ;;
474                 -q)
475                         command="rpm -q $2"
476                         f_upgrade=no
477                         shift
478                         ;;
479
480                 --pull)
481                         command_flags="no-chroot"
482                         command="git pull"
483                         f_upgrade=no
484                         ;;
485
486                 --update-macros)
487                         upgrade_macros="yes"
488                         ;;
489
490                 --update-scripts)
491                         upgrade_scripts='yes'
492                         ;;
493
494                 -df | --distfiles-fetch | --distfiles-fetch-request)
495                         df_fetch=yes
496                         ;;
497
498                 --gpg-opts | -g)
499                         gpg_opts="$2"
500                         shift
501                         ;;
502
503                 --help | -h)
504                         usage
505                         ;;
506
507                 --requester)
508                         requester_override="$2"
509                         shift
510                         ;;
511
512                 -*)
513                         die "unknown knob: $1"
514                         ;;
515
516                 *:* | *)
517                         specs="$specs $1"
518                         ;;
519         esac
520         shift
521 done
522
523 case "$dist" in
524 ti)
525         builder_email="builderti@src.tld-linux.org"
526         default_builders="ti-*"
527         url="https://src.tld-linux.org:1241/"
528         control_url="http://src.tld-linux.org/~builderti"
529         ;;
530 ti-dev)
531         builder_email="buildertidev@src.tld-linux.org"
532         default_builders="ti-dev-*"
533         url="https://src.tld-linux.org:1242/"
534         control_url="http://src.tld-linux.org/~buildertidev"
535         ;;
536 *)
537         die "dist \`$dist' not known"
538         ;;
539 esac
540
541 # need to do this after dist selection
542 if [ "$skip" ]; then
543         skip=$(skip="$skip" control_url="$control_url" python3 -c '
544 import sys
545 import gzip
546 import re
547 import os
548 import string
549 from urllib import request
550 from io import BytesIO
551 from xml.dom import minidom
552
553 skip = os.environ.get("skip").split(",");
554 control_url = os.environ.get("control_url")
555
556 print("* Check queue_id-s against %s" % control_url, file=sys.stderr)
557
558 try:
559         headers = { "Cache-Control": "no-cache", "Pragma": "no-cache" }
560         req = request.Request(url=control_url + "/queue.gz", headers=headers)
561         f = request.urlopen(req)
562 except Exception as e:
563         print("Fetch error %s: %s" % (control_url + "/queue.gz", e), file=sys.stderr)
564         sys.exit(1)
565
566 sio = BytesIO()
567 sio.write(f.read())
568 f.close()
569 sio.seek(0)
570 f = gzip.GzipFile(fileobj = sio)
571
572 xml = re.compile("(<queue>.*?</queue>)", re.DOTALL).match(f.read().decode("utf-8")).group(1)
573 d = minidom.parseString(xml)
574
575 q = []
576 for c in d.documentElement.childNodes:
577         if c.nodeName != "group":
578                 continue
579         q.append(c.attributes["id"].value)
580
581 err = 0
582 for s in skip:
583         if s not in q:
584                 print("- Check %s: ERROR: Not valid queue-id" % s, file=sys.stderr)
585                 err = 1
586         else:
587                 print("- Check %s: OK" % s, file=sys.stderr)
588 if err == 1:
589         sys.exit(1)
590 print(",".join(skip))
591 ') || exit $?
592         f_upgrade=no
593         build_mode=test
594         priority=-1
595         command="skip:$skip"
596         command_flags="no-chroot"
597         builders="$dist-src"
598 fi
599
600 branch=${branch:-$default_branch}
601
602 specs=`for s in $specs; do
603         case "$s" in
604         ^)
605                 # skip marker - pass it along
606                 echo $s
607                 ;;
608         *:*) # package name with branch
609                 _specfile=$(basename ${s%:*})
610                 echo ${_specfile%.spec}.spec:${s##*:}
611                 ;;
612         *) # just package name
613                 echo $(basename ${s%.spec}).spec:$branch
614                 ;;
615         esac
616 done`
617
618 if [ "$relup" = "yes" ]; then
619         msg "Auto relup enabled"
620         relup ${message:+-m "$message"} $specs
621 fi
622
623 if [ "$pkgrevision" = "yes" ]; then
624         msg "Package revision build enabled"
625         specs=$(get_pkgrevision $specs)
626 fi
627
628 if [ "$df_fetch" = "yes" ]; then
629         df_fetch $specs
630         exit 0
631 fi
632
633 if [ "$upgrade_macros" = "yes" ]; then
634         command="poldek --up; poldek -uv rpm-build-macros"
635         builders="$dist-src"
636         f_upgrade=no
637         build_mode=test
638 fi
639
640 if [ "$upgrade_scripts" = "yes" ]; then
641         command="cd ~/rpm/rpm-build-tools && git pull"
642         command_flags=as-builder
643         builders="$dist-src"
644         f_upgrade=no
645         build_mode=test
646 fi
647
648 if [[ "$requester" != *@* ]] ; then
649         requester="$requester@tld-linux.org"
650 fi
651
652 if [ -z "$builders" ] ; then
653         builders="$default_builders"
654 fi
655
656 if [ "$f_upgrade" = "yes" ] ; then
657         flags="$flags upgrade"
658 fi
659
660 if [ "$build_mode" = "test" ] ; then
661         if [ "$f_upgrade" = "yes" ] ; then
662                 die "--upgrade and --test-build are mutually exclusive"
663         fi
664         flags="$flags test-build"
665 fi
666
667 if [ -z "$build_mode" ] ; then
668         # missing build mode, builders go crazy when you proceed"
669         die "please specify build mode"
670 fi
671
672
673 ok=
674 for s in $specs; do
675         ok=1
676 done
677
678 if [ -z "$specs" -a -z "$command" ]; then
679         die "no packages to build or command to invoke specified"
680 fi
681
682 id=$(uuidgen)
683
684 gen_req() {
685         echo "<group id='$id' no='0' flags='$flags'>"
686         echo "  <time>$(date +%s)</time>"
687         msg "Using priority $priority"
688         echo "  <priority>$priority</priority>"
689         if [ -n "$jobs" ]; then
690                 msg "Using jobs $jobs"
691                 echo "  <maxjobs>$jobs</maxjobs>"
692         fi
693         if [ -z "$url" ]; then
694                 msg "Using email $builder_email"
695         else
696                 msg "Using URL $url"
697         fi
698
699         if [ "$build_mode" = "ready" ]; then
700                 msg "Build mode: $(tput setaf 2)$build_mode$c_norm"
701         else
702                 msg "Build mode: $(tput setaf 3)$build_mode$c_norm"
703         fi
704
705         if [ -n "$requester_override" ] ; then
706                 echo "  <requester>$requester_override</requester>"
707         fi
708
709         msg "Queue-ID: $id"
710         echo
711
712         # job to depend on
713         local depend=
714         local b i=1 val
715         local name branch builders_xml
716
717         for b in $builders; do
718                 msg "Builder: $(red $b)"
719                 builders_xml="$builders_xml <builder>$b</builder>"
720         done
721
722         if [ "$command" ]; then
723                 bid=$(uuidgen)
724                 printf "%s\n" "* Command: $command" >&2
725                 echo "  <batch id='$bid' depends-on=''>"
726                 echo "           <command flags='$command_flags'>"
727                 hsc "$command"
728                 echo "</command>"
729                 echo "           <info></info>"
730                 echo "$builders_xml"
731                 echo "  </batch>"
732                 depend=$bid
733         fi
734
735         if [ "$f_upgrade" = "yes" ] ; then
736                 msg "Upgrade mode: $f_upgrade"
737         fi
738
739         for s in $specs; do
740                 # skip marker
741                 if [ "$s" = "^" ]; then
742                         depend=
743                         continue
744                 fi
745                 if [ "$no_depend" = yes ]; then
746                         depend=
747                 fi
748                 bid=$(uuidgen)
749                 echo "  <batch id='$bid' depends-on='$depend'>"
750
751                 name=$(echo "$s" | sed -e 's|:.*||')
752                 branch=$(echo "$s" | sed -e 's|.*:||')
753                 msg "Adding #$i $name:$branch${kernel:+ alt_kernel=$kernel}${target:+ target=$target}${depend:+ depends on $depend}"
754                 echo "           <spec>$name</spec>"
755                 echo "           <branch>$branch</branch>"
756                 echo "           ${kernel:+<kernel>$kernel</kernel>}"
757                 echo "           ${target:+<target>$target</target>}"
758
759                 oIFS=$IFS; IFS=$cr
760                 for b in $define; do
761                         [ -z "$b" ] && continue
762                         value=${b#*=}
763                         b=${b%%=*}
764                         echo "           <define name='$(hsc $b)'>$(hsc "$value")</define>"
765                         msg "- define $b=$value"
766                 done
767                 IFS=$oIFS
768
769                 echo "           <info></info>"
770                 echo
771                 for b in $with; do
772                         echo "           <with>$b</with>"
773                 done
774                 for b in $without; do
775                         echo "           <without>$b</without>"
776                 done
777                 echo
778                 echo "$builders_xml"
779                 echo "  </batch>"
780                 i=$((i+1))
781
782                 # let next job depend on previous
783                 depend=$bid
784         done
785
786         if [ "$post_command" ]; then
787                 bid=$(uuidgen)
788                 if [ "$no_depend" = yes ]; then
789                         depend=
790                 fi
791                 printf "%s\n" "* Post-Command: $post_command" >&2
792                 echo "  <batch id='$bid' depends-on='$depend'>"
793                 echo "           <command flags='$command_flags'>"
794                 hsc "$post_command"
795                 echo "</command>"
796                 echo "           <info></info>"
797                 echo "$builders_xml"
798                 echo "  </batch>"
799                 depend=$bid
800         fi
801
802         echo "</group>"
803 }
804
805 gen_email () {
806         # make request first, so the STDERR/STDOUT streams won't be mixed
807         local tmp req
808         tmp=$(mktemp)
809         gen_req > $tmp
810
811         if [ "$verbose" = "yes" ]; then
812                 cat $tmp >&2
813         fi
814
815         cat <<-EOF
816         From: $requester
817         To: $builder_email
818         Subject: build request
819         Message-Id: <$id@$(hostname)>
820         X-New-TLD-Builder: request
821         X-Requester-Version: $VERSION
822
823         EOF
824
825         gpg --clearsign --default-key $default_key $gpg_opts --output=- $tmp
826         rm -f $tmp
827 }
828
829 gen_email | send_request