]> TLD Linux GIT Repositories - tld-builder.git/blob - client/make-request.sh
- cleanup
[tld-builder.git] / client / make-request.sh
1 #!/bin/sh
2 # vim:noet:ts=4:sw=4
3 VERSION=1.89
4
5 # prevent "*" from being expanded in builders var
6 set -f
7
8 builders=
9 with=
10 without=
11 flags=
12 command=
13 command_flags=
14 gpg_opts=
15 default_branch='HEAD'
16 dist=
17 url=
18 no_depend=no
19 verbose=no
20 autotag=no
21 requester_override=
22 relup=no
23
24 if [ -x /usr/bin/python ]; then
25         send_mode="python"
26 else
27         echo "No python present, using mail mode"
28         send_mode="mail"
29 fi
30
31 if [ -n "$HOME_ETC" ]; then
32         USER_CFG=$HOME_ETC/.requestrc
33 else
34         USER_CFG=$HOME/.requestrc
35 fi
36
37 if [ ! -f "$USER_CFG" ]; then
38         echo "Creating config file $USER_CFG. You *must* edit it."
39         cat > $USER_CFG <<EOF
40 priority=2
41 requester=deviloper@tld-linux.org
42 default_key=deviloper@tld-linux.org
43 send_mode="$send_mode"
44 url="$url"
45 mailer="/usr/lib/sendmail -t"
46 gpg_opts=""
47 dist=ti
48 url="http://kraz.tld-linux.org:1231/"
49
50 # defaults:
51 f_upgrade=yes
52 EOF
53 exit
54 fi
55
56 if [ -f "$USER_CFG" ]; then
57         . $USER_CFG
58         # legacy fallback
59         if [ "${distro:+set}" = "set" ]; then
60                 dist=$distro
61         fi
62 fi
63
64 # internal options, not to be overriden
65 specs=
66 df_fetch=no
67 upgrade_macros=no
68 upgrade_scripts=no
69 cr=$(printf "\r")
70
71 # Set colors
72 c_star=$(tput setaf 2)
73 c_red=$(tput setaf 1)
74 c_norm=$(tput op)
75 msg() {
76         echo >&2 "${c_star}*${c_norm} $*"
77 }
78 red() {
79         echo "${c_red}$*${c_norm}"
80 }
81
82 die() {
83         echo >&2 "$0: $*"
84         exit 1
85 }
86
87 send_request() {
88         # switch to mail mode, if no url set
89         [ -z "$url" ] && send_mode="mail"
90
91         if [ -n "$wait" ]; then
92                 msg "Waiting $wait seconds before sending request"
93                 sleep $wait
94                 msg "Wait has ended, proceeding!"
95         fi
96
97         case "$send_mode" in
98         "mail")
99                 msg "Sending using mail mode"
100                 cat - | $mailer
101                 ;;
102         *)
103                 msg "Sending using HTTP mode to $url"
104                 cat - | python -c '
105 import sys, socket, urllib2
106
107 try:
108         data = sys.stdin.read()
109         url = sys.argv[1]
110         socket.setdefaulttimeout(30)
111         req = urllib2.Request(url, data)
112         f = urllib2.urlopen(req)
113         f.close()
114 except Exception, e:
115         print >> sys.stderr, "Problem while sending request via HTTP: %s: %s" % (url, e)
116         sys.exit(1)
117 print >> sys.stdout, "Request queued via HTTP."
118 ' "$url"
119                 ;;
120         esac
121 }
122
123 # htmlspecialchars: escape <, > and &
124 hsc() {
125         local input=$1
126         echo -E "$input" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
127 }
128
129 # simple df_fetcher, based on packages/fetchsrc_request
130 # TODO: tcp (smtp) mode
131 # TODO: adjust for ~/.requestrc config
132 df_fetch() {
133         local specs="$@"
134
135         # Sending by
136         local MAILER='/usr/lib/sendmail'
137         # MAILER='/usr/bin/msmtp'
138         # Sending via
139         local VIA="SENDMAIL"
140         #VIA="localhost"
141         local VIA_ARGS=""
142         #VIA_ARGS="some additional flags"
143         # e.g. for msmtp:
144         # VIA_ARGS='-a gmail'
145         #
146         # DISTFILES EMAIL
147         local DMAIL="df@tld-linux.org"
148
149         local HOST=$(hostname -f)
150         local LOGIN=${requester%@*}
151
152         local SPEC BRANCH
153         for spec in $specs; do
154                 SPEC=$(echo "$spec" | sed -e 's|:.*||')
155                 SPEC=${SPEC%.spec}
156                 BRANCH=$(echo "$spec" | sed -e 's|.*:||')
157                 echo >&2 "Distfiles Request: $SPEC:$BRANCH via $MAILER ${VIA_ARGS:+ ($VIA_ARGS)}"
158                 cat <<-EOF | "$MAILER" -t -i $VIA_ARGS
159                         To: $DMAIL
160                         From: $LOGIN <$LOGIN@$HOST>
161                         Subject: fetchsrc_request notify
162                         X-distfiles-request: yes
163                         X-Login: $LOGIN
164                         X-Package: $SPEC
165                         X-Branch: $BRANCH
166                         X-Flags: force-reply
167
168                         .
169                         EOF
170         done
171 }
172
173 # autotag from rpm-build-macros
174 # displays latest used tag for a specfile
175 autotag() {
176         local out s
177         for s in "$@"; do
178                 # strip branches
179                 s=${s%:*}
180                 # ensure package ends with .spec
181                 s=${s%.spec}.spec
182                 git fetch --tags
183                 out=$(git for-each-ref --count=1 --sort=-authordate refs/tags/auto/$dist \
184                         --format='%(refname:short)')
185                 echo "$s:$out"
186         done
187 }
188
189 # get autotag for specs
190 # WARNING: This may checkout some files from VCS
191 get_autotag() {
192         local pkg spec rpmdir
193
194         rpmdir=$(rpm -E %_topdir)
195         for pkg in "$@"; do
196                 cd $rpmdir
197                 # strip branches
198                 pkg=${pkg%:*}
199                 # strip .spec extension
200                 pkg=${pkg%.spec}
201                 # checkout only if missing
202                 if [ ! -e $pkg/$pkg.spec ]; then
203                         $rpmdir/builder -g $pkg -ns -r HEAD 1>&2
204                 fi
205                 if [ ! -e $pkg/$pkg.spec ]; then
206                         # just print it out, to fallback to base pkg name
207                         echo "$pkg"
208                 else
209                         cd $pkg
210                         autotag $pkg.spec
211                 fi
212         done
213 }
214
215 relup() {
216         local script=$(dirname $(rpm -E %_topdir))/rpm-build-tools/relup.sh
217         $script -u -i "$@"
218 }
219
220 usage() {
221         cat <<EOF
222 Usage: make-request.sh [OPTION] ... [SPECFILE] ....
223
224 Mandatory arguments to long options are mandatory for short options too.
225
226       --config-file /path/to/config/file
227             Source additional config file (after $USER_CFG), useful when
228             when sending build requests to Ac/Th from the same account
229       -a
230             Try to use latest auto-tag for the spec when building
231             WARNING: This will checkout new files to your packages dir
232       -b 'BUILDER BUILDER ...',  --builder='BUILDER BUILDER ...'
233             Sends request to given builders (in 'version-arch' format)
234       --with VALUE, --without VALUE
235             Build package with(out) a given bcond
236       --kernel VALUE
237             set alt_kernel to VALUE
238       --target VALUE
239             set --target to VALUE
240       -D "NAME VALUE"|--define "NAME VALUE"
241             define macro named NAME with value VALUE
242       -s BUILD_ID, --skip BUILD_ID[,BUILD_ID][,BUILD_ID]
243             mark build ids on src builder to be skipped (instructs srcbuilder to create 'skipme' file)
244       --branch VALUE
245             specify default branch for specs in request
246       -t, --test-build
247             Performs a 'test-build'. Package will be uploaded to hidden .test-builds/
248             ftp tree and won't be upgraded on builders.
249       -r, --ready-build
250             Preforms a 'ready' build. Package will be built and uploaded to test/ ftp tree
251             (and later moved by release manager staff to ready/ and main ftp tree)
252       -u, --upgrade
253             Forces package upgrade (for use with -c or -q, not -t)
254       --relup
255             Bump package release, see also --relup
256       -m, --message
257             Set commit message for relup
258       -n, --no-upgrade
259             Disables package upgrade (for use with -r)
260       -ni, --no-install-br
261             Do not install missing BuildRequires (--nodeps)
262       -nd, --no-depend
263             Do not add dependency of build jobs, each job in batch runs itself
264       -j, --jobs
265             Number of parallel jobs for single build
266       -f, --flag
267       -d, --dist DISTRIBUTION_ID
268             Specify value for \$dist
269       -df,  --distfiles-fetch[-request] PACKAGE
270             Send distfiles request to fetch sources for PACKAGE
271       -cf, --command-flag
272             Not yet documented
273       -c, --command
274             Executes a given command on builders (prepended to build jobs if build jobs included)
275       -C, --post-command
276             Executes a given command on builders (appended to build jobs if build jobs included)
277       --test-remove-pkg
278             shortcut for --command poldek -evt ARGS
279       --remove-pkg
280             shortcut for --command poldek -ev --noask ARGS
281       --upgrade-pkg
282             shortcut for --command poldek --up -Uv ARGS
283       --pull
284             Updates builders infrastructure (outside chroot)
285       --update-macros
286             Updates rpm-build-macros on src builder
287       --update-scripts
288             Updates ~/rpm/rpm-build-tools on builder
289       --requester username
290             Override the requester
291       -q
292             shortcut for --command rpm -q ARGS
293       -g, --gpg-opts "opts"
294             Pass additional options to gpg binary
295       -p, --priority VALUE
296             sets request priority (default 2)
297       -w SECONDS
298             Wait SECONDS before sending actual request. Note: gpg passphrase is still asked immediately.
299             This may be useful if you just commited package and want to send it
300             for test build after distfiles has fetched the file.
301       -h, --help
302             Displays this help message
303       -v
304             Verbose. Print some more debug on screen
305 EOF
306         exit 0
307 }
308
309 # validate distro, set $dist
310 set_dist() {
311         case "$1" in
312         ti)
313                 ;;
314         ti-dev)
315                 ;;
316         *)
317                 die "dist \`$1' not known"
318                 ;;
319         esac
320
321         dist=$1
322 }
323
324 while [ $# -gt 0 ]; do
325         case "$1" in
326                 -d | --dist | --distro)
327                         set_dist $2
328                         shift
329                         ;;
330
331                 --config-file)
332                         [ -f "$2" ] && . $2 || die "Config file not found"
333                         shift
334                         ;;
335
336                 --builder | -b)
337                         for b in $2; do
338                                 builders="$builders ${b%:*}"
339                         done
340                         shift
341                         ;;
342
343                 -a)
344                         autotag=yes
345                         ;;
346
347                 -m)
348                         shift
349                         message=$1
350                         ;;
351
352                 --relup)
353                         relup=yes
354                         ;;
355
356                 --with)
357                         with="$with $(echo "$2" | tr ',' ' ')"
358                         shift
359                         ;;
360
361                 --without)
362                         without="$without $(echo "$2" | tr ',' ' ')"
363                         shift
364                         ;;
365
366                 --test-build | -t)
367                         build_mode=test
368                         f_upgrade=no
369                         ;;
370
371                 --kernel)
372                         kernel=$2
373                         shift
374                         ;;
375
376                 --target)
377                         target=$2
378                         shift
379                         ;;
380
381                 -D|--define)
382                         value=${2#* }
383                         name=${2%% *}
384                         define="$define$cr$name=$value"
385                         shift
386                         ;;
387
388                 -s|--skip)
389                         skip="$2"
390                         shift
391                         ;;
392
393                 --branch)
394                         branch=$2
395                         shift
396                         ;;
397
398                 --priority | -p)
399                         priority=$2
400                         shift
401                         ;;
402
403                 --ready-build | -r)
404                         build_mode=ready
405                         ;;
406
407                 --upgrade | -u)
408                         f_upgrade=yes
409                         ;;
410
411                 --no-upgrade | -n)
412                         f_upgrade=no
413                         ;;
414
415                 --no-depend | -nd)
416                         no_depend=yes
417                         ;;
418
419                 --no-install-br | -ni)
420                         flags="$flags no-install-br"
421                         ;;
422
423                 -j | --jobs)
424                         jobs="$2"
425                         shift
426                         ;;
427
428                 -j*)
429                         jobs="${1#-j}"
430                         ;;
431
432                 -w)
433                         wait="$2"
434                         shift
435                         ;;
436
437                 -v)
438                         verbose=yes
439                         ;;
440
441                 --flag | -f)
442                         flags="$flags $2"
443                         shift
444                         ;;
445
446                 --command-flags | -cf)
447                         command_flags="$2"
448                         shift
449                         ;;
450
451                 --command | -c)
452                         command="$2"
453                         if [ "$command" = - ]; then
454                                 echo >&2 "Reading command from STDIN"
455                                 echo >&2 "---"
456                                 command=$(cat)
457                                 echo >&2 "---"
458                         fi
459                         shift
460                         ;;
461                 --post-command | -C)
462                         post_command="$2"
463                         if [ "$post_command" = - ]; then
464                                 echo >&2 "Reading post_command from STDIN"
465                                 echo >&2 "---"
466                                 post_command=$(cat)
467                                 echo >&2 "---"
468                         fi
469                         shift
470                         ;;
471                 --test-remove-pkg)
472                         command="poldek -evt $2"
473                         f_upgrade=no
474                         shift
475                         ;;
476                 --remove-pkg)
477                         command="for a in $2; do poldek -ev --noask \$a; done"
478                         f_upgrade=no
479                         shift
480                         ;;
481                 --upgrade-pkg|-Uhv)
482                         command="poldek --up; poldek -uv $2"
483                         f_upgrade=no
484                         shift
485                         ;;
486                 -q)
487                         command="rpm -q $2"
488                         f_upgrade=no
489                         shift
490                         ;;
491
492                 --pull)
493                         command_flags="no-chroot"
494                         command="git pull"
495                         f_upgrade=no
496                         ;;
497
498                 --update-macros)
499                         upgrade_macros="yes"
500                         ;;
501
502                 --update-scripts)
503                         upgrade_scripts='yes'
504                         ;;
505
506                 -df | --distfiles-fetch | --distfiles-fetch-request)
507                         df_fetch=yes
508                         ;;
509
510                 --gpg-opts | -g)
511                         gpg_opts="$2"
512                         shift
513                         ;;
514
515                 --help | -h)
516                         usage
517                         ;;
518
519                 --requester)
520                         requester_override="$2"
521                         shift
522                         ;;
523
524                 -*)
525                         die "unknown knob: $1"
526                         ;;
527
528                 *:* | *)
529                         specs="$specs $1"
530                         ;;
531         esac
532         shift
533 done
534
535 case "$dist" in
536 ti)
537         builder_email="builderti@kraz.tld-linux.org"
538         default_builders="ti-*"
539         url="http://kraz.tld-linux.org:1231/"
540         control_url="http://kraz.tld-linux.org/~builderti"
541         ;;
542 ti-dev)
543         builder_email="buildertidev@kraz.tld-linux.org"
544         default_builders="ti-dev-*"
545         url="http://kraz.tld-linux.org:1232/"
546         control_url="http://kraz.tld-linux.org/~buildertidev"
547         ;;
548 *)
549         die "dist \`$dist' not known"
550         ;;
551 esac
552
553 # need to do this after dist selection
554 if [ "$skip" ]; then
555         skip=$(skip="$skip" control_url="$control_url" python -c '
556 import urllib2
557 import sys
558 import StringIO
559 import gzip
560 import re
561 import os
562 import string
563 from xml.dom import minidom
564
565 skip = os.environ.get("skip").split(",");
566 control_url = os.environ.get("control_url")
567
568 print >> sys.stderr, "* Check queue_id-s against %s" % control_url
569
570 try:
571         headers = { "Cache-Control": "no-cache", "Pragma": "no-cache" }
572         req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
573         f = urllib2.urlopen(req)
574 except Exception, e:
575         print >> sys.stderr, "Fetch error %s: %s" % (control_url + "/queue.gz", e)
576         sys.exit(1)
577
578 sio = StringIO.StringIO()
579 sio.write(f.read())
580 f.close()
581 sio.seek(0)
582 f = gzip.GzipFile(fileobj = sio)
583
584 xml = re.compile("(<queue>.*?</queue>)", re.DOTALL).match(f.read()).group(1)
585 d = minidom.parseString(xml)
586
587 q = []
588 for c in d.documentElement.childNodes:
589         if c.nodeName != "group":
590                 continue
591         q.append(c.attributes["id"].value)
592
593 err = 0
594 for s in skip:
595         if s not in q:
596                 print >> sys.stderr, "- Check %s: ERROR: Not valid queue-id" % s
597                 err = 1
598         else:
599                 print >> sys.stderr, "- Check %s: OK" % s
600 if err == 1:
601         sys.exit(1)
602 print string.join(skip, ",")
603 ') || exit $?
604         f_upgrade=no
605         build_mode=test
606         priority=-1
607         command="skip:$skip"
608         command_flags="no-chroot"
609         builders="$dist-src"
610 fi
611
612 branch=${branch:-$default_branch}
613
614 specs=`for s in $specs; do
615         case "$s" in
616         ^)
617                 # skip marker - pass it along
618                 echo $s
619                 ;;
620         *:*) # package name with branch
621                 _specfile=$(basename ${s%:*})
622                 echo ${_specfile%.spec}.spec:${s##*:}
623                 ;;
624         *) # just package name
625                 echo $(basename ${s%.spec}).spec:$branch
626                 ;;
627         esac
628 done`
629
630 if [ "$relup" = "yes" ]; then
631         msg "Auto relup enabled"
632         relup ${message:+-m "$message"} $specs
633 fi
634
635 if [ "$autotag" = "yes" ]; then
636         msg "Auto autotag build enabled"
637         specs=$(get_autotag $specs)
638 fi
639
640 if [ "$df_fetch" = "yes" ]; then
641         df_fetch $specs
642         exit 0
643 fi
644
645 if [ "$upgrade_macros" = "yes" ]; then
646         command="poldek --up; poldek -uv rpm-build-macros"
647         builders="$dist-src"
648         f_upgrade=no
649         build_mode=test
650 fi
651
652 if [ "$upgrade_scripts" = "yes" ]; then
653         command="cd ~/rpm/rpm-build-tools && git pull"
654         command_flags=as-builder
655         builders="$dist-src"
656         f_upgrade=no
657         build_mode=test
658 fi
659
660 if [[ "$requester" != *@* ]] ; then
661         requester="$requester@tld-linux.org"
662 fi
663
664 if [ -z "$builders" ] ; then
665         builders="$default_builders"
666 fi
667
668 if [ "$f_upgrade" = "yes" ] ; then
669         flags="$flags upgrade"
670 fi
671
672 if [ "$build_mode" = "test" ] ; then
673         if [ "$f_upgrade" = "yes" ] ; then
674                 die "--upgrade and --test-build are mutually exclusive"
675         fi
676         flags="$flags test-build"
677 fi
678
679 if [ -z "$build_mode" ] ; then
680         # missing build mode, builders go crazy when you proceed"
681         die "please specify build mode"
682 fi
683
684
685 ok=
686 for s in $specs; do
687         ok=1
688 done
689
690 if [ -z "$specs" -a -z "$command" ]; then
691         die "no packages to build or command to invoke specified"
692 fi
693
694 id=$(uuidgen)
695
696 gen_req() {
697         echo "<group id='$id' no='0' flags='$flags'>"
698         echo "  <time>$(date +%s)</time>"
699         msg "Using priority $priority"
700         echo "  <priority>$priority</priority>"
701         if [ -n "$jobs" ]; then
702                 msg "Using jobs $jobs"
703                 echo "  <maxjobs>$jobs</maxjobs>"
704         fi
705         if [ -z "$url" ]; then
706                 msg "Using email $builder_email"
707         else
708                 msg "Using URL $url"
709         fi
710
711         if [ "$build_mode" = "ready" ]; then
712                 msg "Build mode: $(tput setaf 2)$build_mode$c_norm"
713         else
714                 msg "Build mode: $(tput setaf 3)$build_mode$c_norm"
715         fi
716
717         if [ -n "$requester_override" ] ; then
718                 echo "  <requester>$requester_override</requester>"
719         fi
720
721         msg "Queue-ID: $id"
722         echo
723
724         # job to depend on
725         local depend=
726         local b i=1 val
727         local name branch builders_xml
728
729         for b in $builders; do
730                 msg "Builder: $(red $b)"
731                 builders_xml="$builders_xml <builder>$b</builder>"
732         done
733
734         if [ "$command" ]; then
735                 bid=$(uuidgen)
736                 echo -E >&2 "* Command: $command"
737                 echo "  <batch id='$bid' depends-on=''>"
738                 echo "           <command flags='$command_flags'>"
739                 hsc "$command"
740                 echo "</command>"
741                 echo "           <info></info>"
742                 echo "$builders_xml"
743                 echo "  </batch>"
744                 depend=$bid
745         fi
746
747         if [ "$f_upgrade" = "yes" ] ; then
748                 msg "Upgrade mode: $f_upgrade"
749         fi
750
751         for s in $specs; do
752                 # skip marker
753                 if [ "$s" = "^" ]; then
754                         depend=
755                         continue
756                 fi
757                 if [ "$no_depend" = yes ]; then
758                         depend=
759                 fi
760                 bid=$(uuidgen)
761                 echo "  <batch id='$bid' depends-on='$depend'>"
762
763                 name=$(echo "$s" | sed -e 's|:.*||')
764                 branch=$(echo "$s" | sed -e 's|.*:||')
765                 msg "Adding #$i $name:$branch${kernel:+ alt_kernel=$kernel}${target:+ target=$target}${depend:+ depends on $depend}"
766                 echo "           <spec>$name</spec>"
767                 echo "           <branch>$branch</branch>"
768                 echo "           ${kernel:+<kernel>$kernel</kernel>}"
769                 echo "           ${target:+<target>$target</target>}"
770
771                 oIFS=$IFS; IFS=$cr
772                 for b in $define; do
773                         [ -z "$b" ] && continue
774                         value=${b#*=}
775                         b=${b%%=*}
776                         echo "           <define name='$(hsc $b)'>$(hsc "$value")</define>"
777                         msg "- define $b=$value"
778                 done
779                 IFS=$oIFS
780
781                 echo "           <info></info>"
782                 echo
783                 for b in $with; do
784                         echo "           <with>$b</with>"
785                 done
786                 for b in $without; do
787                         echo "           <without>$b</without>"
788                 done
789                 echo
790                 echo "$builders_xml"
791                 echo "  </batch>"
792                 i=$((i+1))
793
794                 # let next job depend on previous
795                 depend=$bid
796         done
797
798         if [ "$post_command" ]; then
799                 bid=$(uuidgen)
800                 if [ "$no_depend" = yes ]; then
801                         depend=
802                 fi
803                 echo -E >&2 "* Post-Command: $post_command"
804                 echo "  <batch id='$bid' depends-on='$depend'>"
805                 echo "           <command flags='$command_flags'>"
806                 hsc "$post_command"
807                 echo "</command>"
808                 echo "           <info></info>"
809                 echo "$builders_xml"
810                 echo "  </batch>"
811                 depend=$bid
812         fi
813
814         echo "</group>"
815 }
816
817 gen_email () {
818         # make request first, so the STDERR/STDOUT streams won't be mixed
819         local tmp req
820         tmp=$(mktemp)
821         gen_req > $tmp
822
823         if [ "$verbose" = "yes" ]; then
824                 cat $tmp >&2
825         fi
826
827         cat <<-EOF
828         From: $requester
829         To: $builder_email
830         Subject: build request
831         Message-Id: <$id@$(hostname)>
832         X-New-TLD-Builder: request
833         X-Requester-Version: $VERSION
834
835         EOF
836
837         gpg --clearsign --default-key $default_key $gpg_opts --output=- $tmp
838         rm -f $tmp
839 }
840
841 gen_email | send_request