]> TLD Linux GIT Repositories - tld-builder.git/blob - client/make-request.sh
- cosmetics and cleanup
[tld-builder.git] / client / make-request.sh
1 #!/bin/sh
2 # vim:noet:ts=4:sw=4
3 VERSION=1.89
4
5 # prevent "*" from being expanded in builders var
6 set -f
7
8 builders=
9 with=
10 without=
11 flags=
12 command=
13 command_flags=
14 gpg_opts=
15 default_branch='HEAD'
16 dist=
17 url=
18 no_depend=no
19 verbose=no
20 pkgrevision=no
21 requester_override=
22 relup=no
23
24 if [ -x /usr/bin/python ]; then
25         send_mode="python"
26 else
27         echo "No python present, using mail mode"
28         send_mode="mail"
29 fi
30
31 if [ -n "$HOME_ETC" ]; then
32         USER_CFG=$HOME_ETC/.requestrc
33 else
34         USER_CFG=$HOME/.requestrc
35 fi
36
37 if [ ! -f "$USER_CFG" ]; then
38         echo "Creating config file $USER_CFG. You *must* edit it."
39         cat > $USER_CFG <<EOF
40 priority=2
41 requester=deviloper@tld-linux.org
42 default_key=deviloper@tld-linux.org
43 send_mode="$send_mode"
44 url="$url"
45 mailer="/usr/lib/sendmail -t"
46 gpg_opts=""
47 dist=ti
48 url="https://src.tld-linux.org:1241/"
49
50 # defaults:
51 f_upgrade=yes
52 EOF
53 exit
54 fi
55
56 if [ -f "$USER_CFG" ]; then
57         . $USER_CFG
58         # legacy fallback
59         if [ "${distro:+set}" = "set" ]; then
60                 dist=$distro
61         fi
62 fi
63
64 # internal options, not to be overriden
65 specs=
66 df_fetch=no
67 upgrade_macros=no
68 upgrade_scripts=no
69 cr=$(printf "\r")
70
71 # Set colors
72 c_star=$(tput setaf 2)
73 c_red=$(tput setaf 1)
74 c_norm=$(tput op)
75 msg() {
76         echo >&2 "${c_star}*${c_norm} $*"
77 }
78 red() {
79         echo "${c_red}$*${c_norm}"
80 }
81
82 die() {
83         echo >&2 "$0: $*"
84         exit 1
85 }
86
87 send_request() {
88         # switch to mail mode, if no url set
89         [ -z "$url" ] && send_mode="mail"
90
91         if [ -n "$wait" ]; then
92                 msg "Waiting $wait seconds before sending request"
93                 sleep $wait
94                 msg "Wait has ended, proceeding!"
95         fi
96
97         case "$send_mode" in
98         "mail")
99                 msg "Sending using mail mode"
100                 cat - | $mailer
101                 ;;
102         *)
103                 msg "Sending using HTTP mode to $url"
104                 cat - | python -c '
105 import sys, socket, urllib2, ssl
106
107 try:
108     _create_unverified_https_context = ssl._create_unverified_context
109 except AttributeError:
110     pass
111 else:
112     ssl._create_default_https_context = _create_unverified_https_context
113
114 try:
115         data = sys.stdin.read()
116         url = sys.argv[1]
117         socket.setdefaulttimeout(30)
118         req = urllib2.Request(url, data)
119         f = urllib2.urlopen(req)
120         f.close()
121 except Exception, e:
122         print >> sys.stderr, "Problem while sending request via HTTP: %s: %s" % (url, e)
123         sys.exit(1)
124 print >> sys.stdout, "Request queued via HTTP."
125 ' "$url"
126                 ;;
127         esac
128 }
129
130 # htmlspecialchars: escape <, > and &
131 hsc() {
132         local input=$1
133         echo -E "$input" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
134 }
135
136 # simple df_fetcher, based on packages/fetchsrc_request
137 # TODO: tcp (smtp) mode
138 # TODO: adjust for ~/.requestrc config
139 df_fetch() {
140         local specs="$@"
141
142         # Sending by
143         local MAILER='/usr/lib/sendmail'
144         # MAILER='/usr/bin/msmtp'
145         # Sending via
146         local VIA="SENDMAIL"
147         #VIA="localhost"
148         local VIA_ARGS=""
149         #VIA_ARGS="some additional flags"
150         # e.g. for msmtp:
151         # VIA_ARGS='-a gmail'
152         #
153         # DISTFILES EMAIL
154         local DMAIL="df@tld-linux.org"
155
156         local HOST=$(hostname -f)
157         local LOGIN=${requester%@*}
158
159         local SPEC BRANCH
160         for spec in $specs; do
161                 SPEC=$(echo "$spec" | sed -e 's|:.*||')
162                 SPEC=${SPEC%.spec}
163                 BRANCH=$(echo "$spec" | sed -e 's|.*:||')
164                 echo >&2 "Distfiles Request: $SPEC:$BRANCH via $MAILER ${VIA_ARGS:+ ($VIA_ARGS)}"
165                 cat <<-EOF | "$MAILER" -t -i $VIA_ARGS
166                         To: $DMAIL
167                         From: $LOGIN <$LOGIN@$HOST>
168                         Subject: fetchsrc_request notify
169                         X-distfiles-request: yes
170                         X-Login: $LOGIN
171                         X-Package: $SPEC
172                         X-Branch: $BRANCH
173                         X-Flags: force-reply
174
175                         .
176                         EOF
177         done
178 }
179
180 # get package revision for specs
181 get_pkgrevision() {
182         local pkgrev pkg spec rpmdir
183
184         rpmdir=$(rpm -E %_topdir)
185         for pkg in "$@"; do
186                 cd $rpmdir
187                 # strip branches
188                 pkg=${pkg%:*}
189                 # strip .spec extension
190                 pkg=${pkg%.spec}
191                 wget http://pkgrevs.tld-linux.org/list/$pkg -O /tmp/$pkg.tags 1>/dev/null 2>&1
192                 pkgrev=$(head -n 1 /tmp/$pkg.tags)
193                 rm -f /tmp/$pkg.tags
194                 if [ "x$pkgrev" = "x" ]; then
195                         echo "$pkg.spec"
196                 else
197                         echo "$pkg.spec:$pkgrev"
198                 fi
199         done
200 }
201
202 relup() {
203         local script=$(dirname $(rpm -E %_topdir))/rpm-build-tools/relup.sh
204         $script -u -i "$@"
205 }
206
207 usage() {
208         cat <<EOF
209 Usage: make-request.sh [OPTION] ... [SPECFILE] ....
210
211 Mandatory arguments to long options are mandatory for short options too.
212
213       --config-file /path/to/config/file
214             Source additional config file (after $USER_CFG), useful when
215             when sending build requests to Ac/Th from the same account
216       -a
217             Try to use latest auto-tag for the spec when building
218             WARNING: This will checkout new files to your packages dir
219       -b 'BUILDER BUILDER ...',  --builder='BUILDER BUILDER ...'
220             Sends request to given builders (in 'version-arch' format)
221       --with VALUE, --without VALUE
222             Build package with(out) a given bcond
223       --kernel VALUE
224             set alt_kernel to VALUE
225       --target VALUE
226             set --target to VALUE
227       -D "NAME VALUE"|--define "NAME VALUE"
228             define macro named NAME with value VALUE
229       -s BUILD_ID, --skip BUILD_ID[,BUILD_ID][,BUILD_ID]
230             mark build ids on src builder to be skipped (instructs srcbuilder to create 'skipme' file)
231       --branch VALUE
232             specify default branch for specs in request
233       -t, --test-build
234             Performs a 'test-build'. Package will be uploaded to hidden .test-builds/
235             ftp tree and won't be upgraded on builders.
236       -r, --ready-build
237             Preforms a 'ready' build. Package will be built and uploaded to test/ ftp tree
238             (and later moved by release manager staff to ready/ and main ftp tree)
239       -u, --upgrade
240             Forces package upgrade (for use with -c or -q, not -t)
241       --relup
242             Bump package release, see also --relup
243       -m, --message
244             Set commit message for relup
245       -n, --no-upgrade
246             Disables package upgrade (for use with -r)
247       -ni, --no-install-br
248             Do not install missing BuildRequires (--nodeps)
249       -nd, --no-depend
250             Do not add dependency of build jobs, each job in batch runs itself
251       -j, --jobs
252             Number of parallel jobs for single build
253       -f, --flag
254       -d, --dist DISTRIBUTION_ID
255             Specify value for \$dist
256       -df,  --distfiles-fetch[-request] PACKAGE
257             Send distfiles request to fetch sources for PACKAGE
258       -cf, --command-flag
259             Not yet documented
260       -c, --command
261             Executes a given command on builders (prepended to build jobs if build jobs included)
262       -C, --post-command
263             Executes a given command on builders (appended to build jobs if build jobs included)
264       --test-remove-pkg
265             shortcut for --command poldek -evt ARGS
266       --remove-pkg
267             shortcut for --command poldek -ev --noask ARGS
268       --upgrade-pkg
269             shortcut for --command poldek --up -Uv ARGS
270       --pull
271             Updates builders infrastructure (outside chroot)
272       --update-macros
273             Updates rpm-build-macros on src builder
274       --update-scripts
275             Updates ~/rpm/rpm-build-tools on builder
276       --requester username
277             Override the requester
278       -q
279             shortcut for --command rpm -q ARGS
280       -g, --gpg-opts "opts"
281             Pass additional options to gpg binary
282       -p, --priority VALUE
283             sets request priority (default 2)
284       -w SECONDS
285             Wait SECONDS before sending actual request. Note: gpg passphrase is still asked immediately.
286             This may be useful if you just commited package and want to send it
287             for test build after distfiles has fetched the file.
288       -h, --help
289             Displays this help message
290       -v
291             Verbose. Print some more debug on screen
292 EOF
293         exit 0
294 }
295
296 # validate distro, set $dist
297 set_dist() {
298         case "$1" in
299         ti)
300                 ;;
301         ti-dev)
302                 ;;
303         *)
304                 die "dist \`$1' not known"
305                 ;;
306         esac
307
308         dist=$1
309 }
310
311 while [ $# -gt 0 ]; do
312         case "$1" in
313                 -d | --dist | --distro)
314                         set_dist $2
315                         shift
316                         ;;
317
318                 --config-file)
319                         [ -f "$2" ] && . $2 || die "Config file not found"
320                         shift
321                         ;;
322
323                 --builder | -b)
324                         for b in $2; do
325                                 builders="$builders ${b%:*}"
326                         done
327                         shift
328                         ;;
329
330                 -a)
331                         pkgrevision=yes
332                         ;;
333
334                 -m)
335                         shift
336                         message=$1
337                         ;;
338
339                 --relup)
340                         relup=yes
341                         ;;
342
343                 --with)
344                         with="$with $(echo "$2" | tr ',' ' ')"
345                         shift
346                         ;;
347
348                 --without)
349                         without="$without $(echo "$2" | tr ',' ' ')"
350                         shift
351                         ;;
352
353                 --test-build | -t)
354                         build_mode=test
355                         f_upgrade=no
356                         ;;
357
358                 --kernel)
359                         kernel=$2
360                         shift
361                         ;;
362
363                 --target)
364                         target=$2
365                         shift
366                         ;;
367
368                 -D|--define)
369                         value=${2#* }
370                         name=${2%% *}
371                         define="$define$cr$name=$value"
372                         shift
373                         ;;
374
375                 -s|--skip)
376                         skip="$2"
377                         shift
378                         ;;
379
380                 --branch)
381                         branch=$2
382                         shift
383                         ;;
384
385                 --priority | -p)
386                         priority=$2
387                         shift
388                         ;;
389
390                 --ready-build | -r)
391                         build_mode=ready
392                         ;;
393
394                 --upgrade | -u)
395                         f_upgrade=yes
396                         ;;
397
398                 --no-upgrade | -n)
399                         f_upgrade=no
400                         ;;
401
402                 --no-depend | -nd)
403                         no_depend=yes
404                         ;;
405
406                 --no-install-br | -ni)
407                         flags="$flags no-install-br"
408                         ;;
409
410                 -j | --jobs)
411                         jobs="$2"
412                         shift
413                         ;;
414
415                 -j*)
416                         jobs="${1#-j}"
417                         ;;
418
419                 -w)
420                         wait="$2"
421                         shift
422                         ;;
423
424                 -v)
425                         verbose=yes
426                         ;;
427
428                 --flag | -f)
429                         flags="$flags $2"
430                         shift
431                         ;;
432
433                 --command-flags | -cf)
434                         command_flags="$2"
435                         shift
436                         ;;
437
438                 --command | -c)
439                         command="$2"
440                         if [ "$command" = - ]; then
441                                 echo >&2 "Reading command from STDIN"
442                                 echo >&2 "---"
443                                 command=$(cat)
444                                 echo >&2 "---"
445                         fi
446                         shift
447                         ;;
448                 --post-command | -C)
449                         post_command="$2"
450                         if [ "$post_command" = - ]; then
451                                 echo >&2 "Reading post_command from STDIN"
452                                 echo >&2 "---"
453                                 post_command=$(cat)
454                                 echo >&2 "---"
455                         fi
456                         shift
457                         ;;
458                 --test-remove-pkg)
459                         command="poldek -evt $2"
460                         f_upgrade=no
461                         shift
462                         ;;
463                 --remove-pkg)
464                         command="for a in $2; do poldek -ev --noask \$a; done"
465                         f_upgrade=no
466                         shift
467                         ;;
468                 --upgrade-pkg|-Uhv)
469                         command="poldek --up; poldek -uv $2"
470                         f_upgrade=no
471                         shift
472                         ;;
473                 -q)
474                         command="rpm -q $2"
475                         f_upgrade=no
476                         shift
477                         ;;
478
479                 --pull)
480                         command_flags="no-chroot"
481                         command="git pull"
482                         f_upgrade=no
483                         ;;
484
485                 --update-macros)
486                         upgrade_macros="yes"
487                         ;;
488
489                 --update-scripts)
490                         upgrade_scripts='yes'
491                         ;;
492
493                 -df | --distfiles-fetch | --distfiles-fetch-request)
494                         df_fetch=yes
495                         ;;
496
497                 --gpg-opts | -g)
498                         gpg_opts="$2"
499                         shift
500                         ;;
501
502                 --help | -h)
503                         usage
504                         ;;
505
506                 --requester)
507                         requester_override="$2"
508                         shift
509                         ;;
510
511                 -*)
512                         die "unknown knob: $1"
513                         ;;
514
515                 *:* | *)
516                         specs="$specs $1"
517                         ;;
518         esac
519         shift
520 done
521
522 case "$dist" in
523 ti)
524         builder_email="builderti@src.tld-linux.org"
525         default_builders="ti-*"
526         url="https://src.tld-linux.org:1241/"
527         control_url="http://src.tld-linux.org/~builderti"
528         ;;
529 ti-dev)
530         builder_email="buildertidev@src.tld-linux.org"
531         default_builders="ti-dev-*"
532         url="https://src.tld-linux.org:1242/"
533         control_url="http://src.tld-linux.org/~buildertidev"
534         ;;
535 *)
536         die "dist \`$dist' not known"
537         ;;
538 esac
539
540 # need to do this after dist selection
541 if [ "$skip" ]; then
542         skip=$(skip="$skip" control_url="$control_url" python -c '
543 import urllib2
544 import sys
545 import StringIO
546 import gzip
547 import re
548 import os
549 import string
550 from xml.dom import minidom
551
552 skip = os.environ.get("skip").split(",");
553 control_url = os.environ.get("control_url")
554
555 print >> sys.stderr, "* Check queue_id-s against %s" % control_url
556
557 try:
558         headers = { "Cache-Control": "no-cache", "Pragma": "no-cache" }
559         req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
560         f = urllib2.urlopen(req)
561 except Exception, e:
562         print >> sys.stderr, "Fetch error %s: %s" % (control_url + "/queue.gz", e)
563         sys.exit(1)
564
565 sio = StringIO.StringIO()
566 sio.write(f.read())
567 f.close()
568 sio.seek(0)
569 f = gzip.GzipFile(fileobj = sio)
570
571 xml = re.compile("(<queue>.*?</queue>)", re.DOTALL).match(f.read()).group(1)
572 d = minidom.parseString(xml)
573
574 q = []
575 for c in d.documentElement.childNodes:
576         if c.nodeName != "group":
577                 continue
578         q.append(c.attributes["id"].value)
579
580 err = 0
581 for s in skip:
582         if s not in q:
583                 print >> sys.stderr, "- Check %s: ERROR: Not valid queue-id" % s
584                 err = 1
585         else:
586                 print >> sys.stderr, "- Check %s: OK" % s
587 if err == 1:
588         sys.exit(1)
589 print string.join(skip, ",")
590 ') || exit $?
591         f_upgrade=no
592         build_mode=test
593         priority=-1
594         command="skip:$skip"
595         command_flags="no-chroot"
596         builders="$dist-src"
597 fi
598
599 branch=${branch:-$default_branch}
600
601 specs=`for s in $specs; do
602         case "$s" in
603         ^)
604                 # skip marker - pass it along
605                 echo $s
606                 ;;
607         *:*) # package name with branch
608                 _specfile=$(basename ${s%:*})
609                 echo ${_specfile%.spec}.spec:${s##*:}
610                 ;;
611         *) # just package name
612                 echo $(basename ${s%.spec}).spec:$branch
613                 ;;
614         esac
615 done`
616
617 if [ "$relup" = "yes" ]; then
618         msg "Auto relup enabled"
619         relup ${message:+-m "$message"} $specs
620 fi
621
622 if [ "$pkgrevision" = "yes" ]; then
623         msg "Package revision build enabled"
624         specs=$(get_pkgrevision $specs)
625 fi
626
627 if [ "$df_fetch" = "yes" ]; then
628         df_fetch $specs
629         exit 0
630 fi
631
632 if [ "$upgrade_macros" = "yes" ]; then
633         command="poldek --up; poldek -uv rpm-build-macros"
634         builders="$dist-src"
635         f_upgrade=no
636         build_mode=test
637 fi
638
639 if [ "$upgrade_scripts" = "yes" ]; then
640         command="cd ~/rpm/rpm-build-tools && git pull"
641         command_flags=as-builder
642         builders="$dist-src"
643         f_upgrade=no
644         build_mode=test
645 fi
646
647 if [[ "$requester" != *@* ]] ; then
648         requester="$requester@tld-linux.org"
649 fi
650
651 if [ -z "$builders" ] ; then
652         builders="$default_builders"
653 fi
654
655 if [ "$f_upgrade" = "yes" ] ; then
656         flags="$flags upgrade"
657 fi
658
659 if [ "$build_mode" = "test" ] ; then
660         if [ "$f_upgrade" = "yes" ] ; then
661                 die "--upgrade and --test-build are mutually exclusive"
662         fi
663         flags="$flags test-build"
664 fi
665
666 if [ -z "$build_mode" ] ; then
667         # missing build mode, builders go crazy when you proceed"
668         die "please specify build mode"
669 fi
670
671
672 ok=
673 for s in $specs; do
674         ok=1
675 done
676
677 if [ -z "$specs" -a -z "$command" ]; then
678         die "no packages to build or command to invoke specified"
679 fi
680
681 id=$(uuidgen)
682
683 gen_req() {
684         echo "<group id='$id' no='0' flags='$flags'>"
685         echo "  <time>$(date +%s)</time>"
686         msg "Using priority $priority"
687         echo "  <priority>$priority</priority>"
688         if [ -n "$jobs" ]; then
689                 msg "Using jobs $jobs"
690                 echo "  <maxjobs>$jobs</maxjobs>"
691         fi
692         if [ -z "$url" ]; then
693                 msg "Using email $builder_email"
694         else
695                 msg "Using URL $url"
696         fi
697
698         if [ "$build_mode" = "ready" ]; then
699                 msg "Build mode: $(tput setaf 2)$build_mode$c_norm"
700         else
701                 msg "Build mode: $(tput setaf 3)$build_mode$c_norm"
702         fi
703
704         if [ -n "$requester_override" ] ; then
705                 echo "  <requester>$requester_override</requester>"
706         fi
707
708         msg "Queue-ID: $id"
709         echo
710
711         # job to depend on
712         local depend=
713         local b i=1 val
714         local name branch builders_xml
715
716         for b in $builders; do
717                 msg "Builder: $(red $b)"
718                 builders_xml="$builders_xml <builder>$b</builder>"
719         done
720
721         if [ "$command" ]; then
722                 bid=$(uuidgen)
723                 echo -E >&2 "* Command: $command"
724                 echo "  <batch id='$bid' depends-on=''>"
725                 echo "           <command flags='$command_flags'>"
726                 hsc "$command"
727                 echo "</command>"
728                 echo "           <info></info>"
729                 echo "$builders_xml"
730                 echo "  </batch>"
731                 depend=$bid
732         fi
733
734         if [ "$f_upgrade" = "yes" ] ; then
735                 msg "Upgrade mode: $f_upgrade"
736         fi
737
738         for s in $specs; do
739                 # skip marker
740                 if [ "$s" = "^" ]; then
741                         depend=
742                         continue
743                 fi
744                 if [ "$no_depend" = yes ]; then
745                         depend=
746                 fi
747                 bid=$(uuidgen)
748                 echo "  <batch id='$bid' depends-on='$depend'>"
749
750                 name=$(echo "$s" | sed -e 's|:.*||')
751                 branch=$(echo "$s" | sed -e 's|.*:||')
752                 msg "Adding #$i $name:$branch${kernel:+ alt_kernel=$kernel}${target:+ target=$target}${depend:+ depends on $depend}"
753                 echo "           <spec>$name</spec>"
754                 echo "           <branch>$branch</branch>"
755                 echo "           ${kernel:+<kernel>$kernel</kernel>}"
756                 echo "           ${target:+<target>$target</target>}"
757
758                 oIFS=$IFS; IFS=$cr
759                 for b in $define; do
760                         [ -z "$b" ] && continue
761                         value=${b#*=}
762                         b=${b%%=*}
763                         echo "           <define name='$(hsc $b)'>$(hsc "$value")</define>"
764                         msg "- define $b=$value"
765                 done
766                 IFS=$oIFS
767
768                 echo "           <info></info>"
769                 echo
770                 for b in $with; do
771                         echo "           <with>$b</with>"
772                 done
773                 for b in $without; do
774                         echo "           <without>$b</without>"
775                 done
776                 echo
777                 echo "$builders_xml"
778                 echo "  </batch>"
779                 i=$((i+1))
780
781                 # let next job depend on previous
782                 depend=$bid
783         done
784
785         if [ "$post_command" ]; then
786                 bid=$(uuidgen)
787                 if [ "$no_depend" = yes ]; then
788                         depend=
789                 fi
790                 echo -E >&2 "* Post-Command: $post_command"
791                 echo "  <batch id='$bid' depends-on='$depend'>"
792                 echo "           <command flags='$command_flags'>"
793                 hsc "$post_command"
794                 echo "</command>"
795                 echo "           <info></info>"
796                 echo "$builders_xml"
797                 echo "  </batch>"
798                 depend=$bid
799         fi
800
801         echo "</group>"
802 }
803
804 gen_email () {
805         # make request first, so the STDERR/STDOUT streams won't be mixed
806         local tmp req
807         tmp=$(mktemp)
808         gen_req > $tmp
809
810         if [ "$verbose" = "yes" ]; then
811                 cat $tmp >&2
812         fi
813
814         cat <<-EOF
815         From: $requester
816         To: $builder_email
817         Subject: build request
818         Message-Id: <$id@$(hostname)>
819         X-New-TLD-Builder: request
820         X-Requester-Version: $VERSION
821
822         EOF
823
824         gpg --clearsign --default-key $default_key $gpg_opts --output=- $tmp
825         rm -f $tmp
826 }
827
828 gen_email | send_request