]> TLD Linux GIT Repositories - tld-builder.git/blob - client/make-request.sh
fdb750b506a28727301954c3645c9bf1b8eba93f
[tld-builder.git] / client / make-request.sh
1 #!/bin/sh
2 # vim:noet:ts=4:sw=4
3 VERSION=1.89
4
5 # prevent "*" from being expanded in builders var
6 set -f
7
8 builders=
9 with=
10 without=
11 flags=
12 command=
13 command_flags=
14 gpg_opts=
15 default_branch='HEAD'
16 dist=
17 url=
18 no_depend=no
19 verbose=no
20 pkgrevision=no
21 requester_override=
22 relup=no
23
24 if [ -x /usr/bin/python3 ]; then
25         send_mode="python"
26 else
27         echo "No python present, using mail mode"
28         send_mode="mail"
29 fi
30
31 if [ -n "$HOME_ETC" ]; then
32         USER_CFG=$HOME_ETC/.requestrc
33 else
34         USER_CFG=$HOME/.requestrc
35 fi
36
37 if [ ! -f "$USER_CFG" ]; then
38         echo "Creating config file $USER_CFG. You *must* edit it."
39         cat > $USER_CFG <<EOF
40 priority=2
41 requester=deviloper@tld-linux.org
42 default_key=deviloper@tld-linux.org
43 send_mode="$send_mode"
44 url="$url"
45 mailer="/usr/lib/sendmail -t"
46 gpg_opts=""
47 dist=ti
48 url="https://src.tld-linux.org:1241/"
49
50 # defaults:
51 f_upgrade=yes
52 EOF
53 exit
54 fi
55
56 if [ -f "$USER_CFG" ]; then
57         . $USER_CFG
58         # legacy fallback
59         if [ "${distro:+set}" = "set" ]; then
60                 dist=$distro
61         fi
62 fi
63
64 # internal options, not to be overriden
65 specs=
66 df_fetch=no
67 upgrade_macros=no
68 upgrade_scripts=no
69 cr=$(printf "\r")
70
71 # Set colors
72 c_star=$(tput setaf 2)
73 c_red=$(tput setaf 1)
74 c_norm=$(tput op)
75 msg() {
76         echo >&2 "${c_star}*${c_norm} $*"
77 }
78 red() {
79         echo "${c_red}$*${c_norm}"
80 }
81
82 die() {
83         echo >&2 "$0: $*"
84         exit 1
85 }
86
87 send_request() {
88         # switch to mail mode, if no url set
89         [ -z "$url" ] && send_mode="mail"
90
91         if [ -n "$wait" ]; then
92                 msg "Waiting $wait seconds before sending request"
93                 sleep $wait
94                 msg "Wait has ended, proceeding!"
95         fi
96
97         case "$send_mode" in
98         "mail")
99                 msg "Sending using mail mode"
100                 cat - | $mailer
101                 ;;
102         *)
103                 msg "Sending using HTTP mode to $url"
104                 cat - | python3 -c '
105 import sys, socket, urllib3, ssl
106
107 try:
108     _create_unverified_https_context = ssl._create_unverified_context
109 except AttributeError:
110     pass
111 else:
112     ssl._create_default_https_context = _create_unverified_https_context
113
114 try:
115         data = sys.stdin.read()
116         url = sys.argv[1]
117         socket.setdefaulttimeout(30)
118         http = urllib3.PoolManager(cert_reqs="CERT_NONE")
119         http.request("POST",url, body=data)
120 except Exception as e:
121         print("Problem while sending request via HTTP: %s: %s" % (url, e), file=sys.stderr)
122         sys.exit(1)
123         print("Request queued via HTTP.", file=sys.stdout)
124 ' "$url"
125                 ;;
126         esac
127 }
128
129 # htmlspecialchars: escape <, > and &
130 hsc() {
131         local input=$1
132         printf "%s\n" "$input" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
133 }
134
135 # simple df_fetcher, based on packages/fetchsrc_request
136 # TODO: tcp (smtp) mode
137 # TODO: adjust for ~/.requestrc config
138 df_fetch() {
139         local specs="$@"
140
141         # Sending by
142         local MAILER='/usr/lib/sendmail'
143         # MAILER='/usr/bin/msmtp'
144         # Sending via
145         local VIA="SENDMAIL"
146         #VIA="localhost"
147         local VIA_ARGS=""
148         #VIA_ARGS="some additional flags"
149         # e.g. for msmtp:
150         # VIA_ARGS='-a gmail'
151         #
152         # DISTFILES EMAIL
153         local DMAIL="df@tld-linux.org"
154
155         local HOST=$(hostname -f)
156         local LOGIN=${requester%@*}
157
158         local SPEC BRANCH
159         for spec in $specs; do
160                 SPEC=$(echo "$spec" | sed -e 's|:.*||')
161                 SPEC=${SPEC%.spec}
162                 BRANCH=$(echo "$spec" | sed -e 's|.*:||')
163                 echo >&2 "Distfiles Request: $SPEC:$BRANCH via $MAILER ${VIA_ARGS:+ ($VIA_ARGS)}"
164                 cat <<-EOF | "$MAILER" -t -i $VIA_ARGS
165                         To: $DMAIL
166                         From: $LOGIN <$LOGIN@$HOST>
167                         Subject: fetchsrc_request notify
168                         X-distfiles-request: yes
169                         X-Login: $LOGIN
170                         X-Package: $SPEC
171                         X-Branch: $BRANCH
172                         X-Flags: force-reply
173
174                         .
175                         EOF
176         done
177 }
178
179 # get package revision for specs
180 get_pkgrevision() {
181         local pkgrev pkg spec rpmdir
182
183         rpmdir=$(rpm -E %_topdir)
184         for pkg in "$@"; do
185                 cd $rpmdir
186                 # strip branches
187                 pkg=${pkg%:*}
188                 # strip .spec extension
189                 pkg=${pkg%.spec}
190                 wget http://pkgrevs.tld-linux.org/list/$pkg -O /tmp/$pkg.tags 1>/dev/null 2>&1
191                 pkgrev=$(head -n 1 /tmp/$pkg.tags)
192                 rm -f /tmp/$pkg.tags
193                 if [ "x$pkgrev" = "x" ]; then
194                         echo "$pkg.spec"
195                 else
196                         echo "$pkg.spec:$pkgrev"
197                 fi
198         done
199 }
200
201 relup() {
202         local script=$(dirname $(rpm -E %_topdir))/rpm-build-tools/relup.sh
203         $script -u -i "$@"
204 }
205
206 usage() {
207         cat <<EOF
208 Usage: make-request.sh [OPTION] ... [SPECFILE] ....
209
210 Mandatory arguments to long options are mandatory for short options too.
211
212       --config-file /path/to/config/file
213             Source additional config file (after $USER_CFG), useful when
214             when sending build requests to Ac/Th from the same account
215       -a
216             Try to use latest auto-tag for the spec when building
217             WARNING: This will checkout new files to your packages dir
218       -b 'BUILDER BUILDER ...',  --builder='BUILDER BUILDER ...'
219             Sends request to given builders (in 'version-arch' format)
220       --with VALUE, --without VALUE
221             Build package with(out) a given bcond
222       --kernel VALUE
223             set alt_kernel to VALUE
224       --target VALUE
225             set --target to VALUE
226       -D "NAME VALUE"|--define "NAME VALUE"
227             define macro named NAME with value VALUE
228       -s BUILD_ID, --skip BUILD_ID[,BUILD_ID][,BUILD_ID]
229             mark build ids on src builder to be skipped (instructs srcbuilder to create 'skipme' file)
230       --branch VALUE
231             specify default branch for specs in request
232       -t, --test-build
233             Performs a 'test-build'. Package will be uploaded to hidden .test-builds/
234             ftp tree and won't be upgraded on builders.
235       -r, --ready-build
236             Preforms a 'ready' build. Package will be built and uploaded to test/ ftp tree
237             (and later moved by release manager staff to ready/ and main ftp tree)
238       -u, --upgrade
239             Forces package upgrade (for use with -c or -q, not -t)
240       --relup
241             Bump package release, see also --relup
242       -m, --message
243             Set commit message for relup
244       -n, --no-upgrade
245             Disables package upgrade (for use with -r)
246       -ni, --no-install-br
247             Do not install missing BuildRequires (--nodeps)
248       -nd, --no-depend
249             Do not add dependency of build jobs, each job in batch runs itself
250       -j, --jobs
251             Number of parallel jobs for single build
252       -f, --flag
253       -d, --dist DISTRIBUTION_ID
254             Specify value for \$dist
255       -df,  --distfiles-fetch[-request] PACKAGE
256             Send distfiles request to fetch sources for PACKAGE
257       -cf, --command-flag
258             Not yet documented
259       -c, --command
260             Executes a given command on builders (prepended to build jobs if build jobs included)
261       -C, --post-command
262             Executes a given command on builders (appended to build jobs if build jobs included)
263       --test-remove-pkg
264             shortcut for --command poldek -evt ARGS
265       --remove-pkg
266             shortcut for --command poldek -ev --noask ARGS
267       --upgrade-pkg
268             shortcut for --command poldek --up -Uv ARGS
269       --pull
270             Updates builders infrastructure (outside chroot)
271       --update-macros
272             Updates rpm-build-macros on src builder
273       --update-scripts
274             Updates ~/rpm/rpm-build-tools on builder
275       --requester username
276             Override the requester
277       -q
278             shortcut for --command rpm -q ARGS
279       -g, --gpg-opts "opts"
280             Pass additional options to gpg binary
281       -p, --priority VALUE
282             sets request priority (default 2)
283       -w SECONDS
284             Wait SECONDS before sending actual request. Note: gpg passphrase is still asked immediately.
285             This may be useful if you just commited package and want to send it
286             for test build after distfiles has fetched the file.
287       -h, --help
288             Displays this help message
289       -v
290             Verbose. Print some more debug on screen
291 EOF
292         exit 0
293 }
294
295 # validate distro, set $dist
296 set_dist() {
297         case "$1" in
298         ti)
299                 ;;
300         ti-dev)
301                 ;;
302         *)
303                 die "dist \`$1' not known"
304                 ;;
305         esac
306
307         dist=$1
308 }
309
310 while [ $# -gt 0 ]; do
311         case "$1" in
312                 -d | --dist | --distro)
313                         set_dist $2
314                         shift
315                         ;;
316
317                 --config-file)
318                         [ -f "$2" ] && . $2 || die "Config file not found"
319                         shift
320                         ;;
321
322                 --builder | -b)
323                         for b in $2; do
324                                 builders="$builders ${b%:*}"
325                         done
326                         shift
327                         ;;
328
329                 -a)
330                         pkgrevision=yes
331                         ;;
332
333                 -m)
334                         shift
335                         message=$1
336                         ;;
337
338                 --relup)
339                         relup=yes
340                         ;;
341
342                 --with)
343                         with="$with $(echo "$2" | tr ',' ' ')"
344                         shift
345                         ;;
346
347                 --without)
348                         without="$without $(echo "$2" | tr ',' ' ')"
349                         shift
350                         ;;
351
352                 --test-build | -t)
353                         build_mode=test
354                         f_upgrade=no
355                         ;;
356
357                 --kernel)
358                         kernel=$2
359                         shift
360                         ;;
361
362                 --target)
363                         target=$2
364                         shift
365                         ;;
366
367                 -D|--define)
368                         value=${2#* }
369                         name=${2%% *}
370                         define="$define$cr$name=$value"
371                         shift
372                         ;;
373
374                 -s|--skip)
375                         skip="$2"
376                         shift
377                         ;;
378
379                 --branch)
380                         branch=$2
381                         shift
382                         ;;
383
384                 --priority | -p)
385                         priority=$2
386                         shift
387                         ;;
388
389                 --ready-build | -r)
390                         build_mode=ready
391                         ;;
392
393                 --upgrade | -u)
394                         f_upgrade=yes
395                         ;;
396
397                 --no-upgrade | -n)
398                         f_upgrade=no
399                         ;;
400
401                 --no-depend | -nd)
402                         no_depend=yes
403                         ;;
404
405                 --no-install-br | -ni)
406                         flags="$flags no-install-br"
407                         ;;
408
409                 -j | --jobs)
410                         jobs="$2"
411                         shift
412                         ;;
413
414                 -j*)
415                         jobs="${1#-j}"
416                         ;;
417
418                 -w)
419                         wait="$2"
420                         shift
421                         ;;
422
423                 -v)
424                         verbose=yes
425                         ;;
426
427                 --flag | -f)
428                         flags="$flags $2"
429                         shift
430                         ;;
431
432                 --command-flags | -cf)
433                         command_flags="$2"
434                         shift
435                         ;;
436
437                 --command | -c)
438                         command="$2"
439                         if [ "$command" = - ]; then
440                                 echo >&2 "Reading command from STDIN"
441                                 echo >&2 "---"
442                                 command=$(cat)
443                                 echo >&2 "---"
444                         fi
445                         shift
446                         ;;
447                 --post-command | -C)
448                         post_command="$2"
449                         if [ "$post_command" = - ]; then
450                                 echo >&2 "Reading post_command from STDIN"
451                                 echo >&2 "---"
452                                 post_command=$(cat)
453                                 echo >&2 "---"
454                         fi
455                         shift
456                         ;;
457                 --test-remove-pkg)
458                         command="poldek -evt $2"
459                         f_upgrade=no
460                         shift
461                         ;;
462                 --remove-pkg)
463                         command="for a in $2; do poldek -ev --noask \$a; done"
464                         f_upgrade=no
465                         shift
466                         ;;
467                 --upgrade-pkg|-Uhv)
468                         command="poldek --up; poldek -uv $2"
469                         f_upgrade=no
470                         shift
471                         ;;
472                 -q)
473                         command="rpm -q $2"
474                         f_upgrade=no
475                         shift
476                         ;;
477
478                 --pull)
479                         command_flags="no-chroot"
480                         command="git pull"
481                         f_upgrade=no
482                         ;;
483
484                 --update-macros)
485                         upgrade_macros="yes"
486                         ;;
487
488                 --update-scripts)
489                         upgrade_scripts='yes'
490                         ;;
491
492                 -df | --distfiles-fetch | --distfiles-fetch-request)
493                         df_fetch=yes
494                         ;;
495
496                 --gpg-opts | -g)
497                         gpg_opts="$2"
498                         shift
499                         ;;
500
501                 --help | -h)
502                         usage
503                         ;;
504
505                 --requester)
506                         requester_override="$2"
507                         shift
508                         ;;
509
510                 -*)
511                         die "unknown knob: $1"
512                         ;;
513
514                 *:* | *)
515                         specs="$specs $1"
516                         ;;
517         esac
518         shift
519 done
520
521 case "$dist" in
522 ti)
523         builder_email="builderti@src.tld-linux.org"
524         default_builders="ti-*"
525         url="https://src.tld-linux.org:1241/"
526         control_url="http://src.tld-linux.org/~builderti"
527         ;;
528 ti-dev)
529         builder_email="buildertidev@src.tld-linux.org"
530         default_builders="ti-dev-*"
531         url="https://src.tld-linux.org:1242/"
532         control_url="http://src.tld-linux.org/~buildertidev"
533         ;;
534 *)
535         die "dist \`$dist' not known"
536         ;;
537 esac
538
539 # need to do this after dist selection
540 if [ "$skip" ]; then
541         skip=$(skip="$skip" control_url="$control_url" python3 -c '
542 import urllib2
543 import sys
544 import StringIO
545 import gzip
546 import re
547 import os
548 import string
549 from xml.dom import minidom
550
551 skip = os.environ.get("skip").split(",");
552 control_url = os.environ.get("control_url")
553
554 print >> sys.stderr, "* Check queue_id-s against %s" % control_url
555
556 try:
557         headers = { "Cache-Control": "no-cache", "Pragma": "no-cache" }
558         req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
559         f = urllib2.urlopen(req)
560 except Exception, e:
561         print >> sys.stderr, "Fetch error %s: %s" % (control_url + "/queue.gz", e)
562         sys.exit(1)
563
564 sio = StringIO.StringIO()
565 sio.write(f.read())
566 f.close()
567 sio.seek(0)
568 f = gzip.GzipFile(fileobj = sio)
569
570 xml = re.compile("(<queue>.*?</queue>)", re.DOTALL).match(f.read()).group(1)
571 d = minidom.parseString(xml)
572
573 q = []
574 for c in d.documentElement.childNodes:
575         if c.nodeName != "group":
576                 continue
577         q.append(c.attributes["id"].value)
578
579 err = 0
580 for s in skip:
581         if s not in q:
582                 print >> sys.stderr, "- Check %s: ERROR: Not valid queue-id" % s
583                 err = 1
584         else:
585                 print >> sys.stderr, "- Check %s: OK" % s
586 if err == 1:
587         sys.exit(1)
588 print string.join(skip, ",")
589 ') || exit $?
590         f_upgrade=no
591         build_mode=test
592         priority=-1
593         command="skip:$skip"
594         command_flags="no-chroot"
595         builders="$dist-src"
596 fi
597
598 branch=${branch:-$default_branch}
599
600 specs=`for s in $specs; do
601         case "$s" in
602         ^)
603                 # skip marker - pass it along
604                 echo $s
605                 ;;
606         *:*) # package name with branch
607                 _specfile=$(basename ${s%:*})
608                 echo ${_specfile%.spec}.spec:${s##*:}
609                 ;;
610         *) # just package name
611                 echo $(basename ${s%.spec}).spec:$branch
612                 ;;
613         esac
614 done`
615
616 if [ "$relup" = "yes" ]; then
617         msg "Auto relup enabled"
618         relup ${message:+-m "$message"} $specs
619 fi
620
621 if [ "$pkgrevision" = "yes" ]; then
622         msg "Package revision build enabled"
623         specs=$(get_pkgrevision $specs)
624 fi
625
626 if [ "$df_fetch" = "yes" ]; then
627         df_fetch $specs
628         exit 0
629 fi
630
631 if [ "$upgrade_macros" = "yes" ]; then
632         command="poldek --up; poldek -uv rpm-build-macros"
633         builders="$dist-src"
634         f_upgrade=no
635         build_mode=test
636 fi
637
638 if [ "$upgrade_scripts" = "yes" ]; then
639         command="cd ~/rpm/rpm-build-tools && git pull"
640         command_flags=as-builder
641         builders="$dist-src"
642         f_upgrade=no
643         build_mode=test
644 fi
645
646 if [[ "$requester" != *@* ]] ; then
647         requester="$requester@tld-linux.org"
648 fi
649
650 if [ -z "$builders" ] ; then
651         builders="$default_builders"
652 fi
653
654 if [ "$f_upgrade" = "yes" ] ; then
655         flags="$flags upgrade"
656 fi
657
658 if [ "$build_mode" = "test" ] ; then
659         if [ "$f_upgrade" = "yes" ] ; then
660                 die "--upgrade and --test-build are mutually exclusive"
661         fi
662         flags="$flags test-build"
663 fi
664
665 if [ -z "$build_mode" ] ; then
666         # missing build mode, builders go crazy when you proceed"
667         die "please specify build mode"
668 fi
669
670
671 ok=
672 for s in $specs; do
673         ok=1
674 done
675
676 if [ -z "$specs" -a -z "$command" ]; then
677         die "no packages to build or command to invoke specified"
678 fi
679
680 id=$(uuidgen)
681
682 gen_req() {
683         echo "<group id='$id' no='0' flags='$flags'>"
684         echo "  <time>$(date +%s)</time>"
685         msg "Using priority $priority"
686         echo "  <priority>$priority</priority>"
687         if [ -n "$jobs" ]; then
688                 msg "Using jobs $jobs"
689                 echo "  <maxjobs>$jobs</maxjobs>"
690         fi
691         if [ -z "$url" ]; then
692                 msg "Using email $builder_email"
693         else
694                 msg "Using URL $url"
695         fi
696
697         if [ "$build_mode" = "ready" ]; then
698                 msg "Build mode: $(tput setaf 2)$build_mode$c_norm"
699         else
700                 msg "Build mode: $(tput setaf 3)$build_mode$c_norm"
701         fi
702
703         if [ -n "$requester_override" ] ; then
704                 echo "  <requester>$requester_override</requester>"
705         fi
706
707         msg "Queue-ID: $id"
708         echo
709
710         # job to depend on
711         local depend=
712         local b i=1 val
713         local name branch builders_xml
714
715         for b in $builders; do
716                 msg "Builder: $(red $b)"
717                 builders_xml="$builders_xml <builder>$b</builder>"
718         done
719
720         if [ "$command" ]; then
721                 bid=$(uuidgen)
722                 printf "%s\n" "* Command: $command" >&2
723                 echo "  <batch id='$bid' depends-on=''>"
724                 echo "           <command flags='$command_flags'>"
725                 hsc "$command"
726                 echo "</command>"
727                 echo "           <info></info>"
728                 echo "$builders_xml"
729                 echo "  </batch>"
730                 depend=$bid
731         fi
732
733         if [ "$f_upgrade" = "yes" ] ; then
734                 msg "Upgrade mode: $f_upgrade"
735         fi
736
737         for s in $specs; do
738                 # skip marker
739                 if [ "$s" = "^" ]; then
740                         depend=
741                         continue
742                 fi
743                 if [ "$no_depend" = yes ]; then
744                         depend=
745                 fi
746                 bid=$(uuidgen)
747                 echo "  <batch id='$bid' depends-on='$depend'>"
748
749                 name=$(echo "$s" | sed -e 's|:.*||')
750                 branch=$(echo "$s" | sed -e 's|.*:||')
751                 msg "Adding #$i $name:$branch${kernel:+ alt_kernel=$kernel}${target:+ target=$target}${depend:+ depends on $depend}"
752                 echo "           <spec>$name</spec>"
753                 echo "           <branch>$branch</branch>"
754                 echo "           ${kernel:+<kernel>$kernel</kernel>}"
755                 echo "           ${target:+<target>$target</target>}"
756
757                 oIFS=$IFS; IFS=$cr
758                 for b in $define; do
759                         [ -z "$b" ] && continue
760                         value=${b#*=}
761                         b=${b%%=*}
762                         echo "           <define name='$(hsc $b)'>$(hsc "$value")</define>"
763                         msg "- define $b=$value"
764                 done
765                 IFS=$oIFS
766
767                 echo "           <info></info>"
768                 echo
769                 for b in $with; do
770                         echo "           <with>$b</with>"
771                 done
772                 for b in $without; do
773                         echo "           <without>$b</without>"
774                 done
775                 echo
776                 echo "$builders_xml"
777                 echo "  </batch>"
778                 i=$((i+1))
779
780                 # let next job depend on previous
781                 depend=$bid
782         done
783
784         if [ "$post_command" ]; then
785                 bid=$(uuidgen)
786                 if [ "$no_depend" = yes ]; then
787                         depend=
788                 fi
789                 printf "%s\n" "* Post-Command: $post_command" >&2
790                 echo "  <batch id='$bid' depends-on='$depend'>"
791                 echo "           <command flags='$command_flags'>"
792                 hsc "$post_command"
793                 echo "</command>"
794                 echo "           <info></info>"
795                 echo "$builders_xml"
796                 echo "  </batch>"
797                 depend=$bid
798         fi
799
800         echo "</group>"
801 }
802
803 gen_email () {
804         # make request first, so the STDERR/STDOUT streams won't be mixed
805         local tmp req
806         tmp=$(mktemp)
807         gen_req > $tmp
808
809         if [ "$verbose" = "yes" ]; then
810                 cat $tmp >&2
811         fi
812
813         cat <<-EOF
814         From: $requester
815         To: $builder_email
816         Subject: build request
817         Message-Id: <$id@$(hostname)>
818         X-New-TLD-Builder: request
819         X-Requester-Version: $VERSION
820
821         EOF
822
823         gpg --clearsign --default-key $default_key $gpg_opts --output=- $tmp
824         rm -f $tmp
825 }
826
827 gen_email | send_request