]> TLD Linux GIT Repositories - tld-builder.git/blob - client/make-request.sh
- hostname changed
[tld-builder.git] / client / make-request.sh
1 #!/bin/sh
2 # vim:noet:ts=4:sw=4
3 VERSION=1.89
4
5 # prevent "*" from being expanded in builders var
6 set -f
7
8 builders=
9 with=
10 without=
11 flags=
12 command=
13 command_flags=
14 gpg_opts=
15 default_branch='HEAD'
16 dist=
17 url=
18 no_depend=no
19 verbose=no
20 autotag=no
21 requester_override=
22 relup=no
23
24 if [ -x /usr/bin/python ]; then
25         send_mode="python"
26 else
27         echo "No python present, using mail mode"
28         send_mode="mail"
29 fi
30
31 if [ -n "$HOME_ETC" ]; then
32         USER_CFG=$HOME_ETC/.requestrc
33 else
34         USER_CFG=$HOME/.requestrc
35 fi
36
37 if [ ! -f "$USER_CFG" ]; then
38         echo "Creating config file $USER_CFG. You *must* edit it."
39         cat > $USER_CFG <<EOF
40 priority=2
41 requester=deviloper@tld-linux.org
42 default_key=deviloper@tld-linux.org
43 send_mode="$send_mode"
44 url="$url"
45 mailer="/usr/lib/sendmail -t"
46 gpg_opts=""
47 dist=ti
48 url="https://giant.tld-linux.org:1241/"
49
50 # defaults:
51 f_upgrade=yes
52 EOF
53 exit
54 fi
55
56 if [ -f "$USER_CFG" ]; then
57         . $USER_CFG
58         # legacy fallback
59         if [ "${distro:+set}" = "set" ]; then
60                 dist=$distro
61         fi
62 fi
63
64 # internal options, not to be overriden
65 specs=
66 df_fetch=no
67 upgrade_macros=no
68 upgrade_scripts=no
69 cr=$(printf "\r")
70
71 # Set colors
72 c_star=$(tput setaf 2)
73 c_red=$(tput setaf 1)
74 c_norm=$(tput op)
75 msg() {
76         echo >&2 "${c_star}*${c_norm} $*"
77 }
78 red() {
79         echo "${c_red}$*${c_norm}"
80 }
81
82 die() {
83         echo >&2 "$0: $*"
84         exit 1
85 }
86
87 send_request() {
88         # switch to mail mode, if no url set
89         [ -z "$url" ] && send_mode="mail"
90
91         if [ -n "$wait" ]; then
92                 msg "Waiting $wait seconds before sending request"
93                 sleep $wait
94                 msg "Wait has ended, proceeding!"
95         fi
96
97         case "$send_mode" in
98         "mail")
99                 msg "Sending using mail mode"
100                 cat - | $mailer
101                 ;;
102         *)
103                 msg "Sending using HTTP mode to $url"
104                 cat - | python -c '
105 import sys, socket, urllib2, ssl
106
107 try:
108     _create_unverified_https_context = ssl._create_unverified_context
109 except AttributeError:
110     pass
111 else:
112     ssl._create_default_https_context = _create_unverified_https_context
113
114 try:
115         data = sys.stdin.read()
116         url = sys.argv[1]
117         socket.setdefaulttimeout(30)
118         req = urllib2.Request(url, data)
119         f = urllib2.urlopen(req)
120         f.close()
121 except Exception, e:
122         print >> sys.stderr, "Problem while sending request via HTTP: %s: %s" % (url, e)
123         sys.exit(1)
124 print >> sys.stdout, "Request queued via HTTP."
125 ' "$url"
126                 ;;
127         esac
128 }
129
130 # htmlspecialchars: escape <, > and &
131 hsc() {
132         local input=$1
133         echo -E "$input" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
134 }
135
136 # simple df_fetcher, based on packages/fetchsrc_request
137 # TODO: tcp (smtp) mode
138 # TODO: adjust for ~/.requestrc config
139 df_fetch() {
140         local specs="$@"
141
142         # Sending by
143         local MAILER='/usr/lib/sendmail'
144         # MAILER='/usr/bin/msmtp'
145         # Sending via
146         local VIA="SENDMAIL"
147         #VIA="localhost"
148         local VIA_ARGS=""
149         #VIA_ARGS="some additional flags"
150         # e.g. for msmtp:
151         # VIA_ARGS='-a gmail'
152         #
153         # DISTFILES EMAIL
154         local DMAIL="df@tld-linux.org"
155
156         local HOST=$(hostname -f)
157         local LOGIN=${requester%@*}
158
159         local SPEC BRANCH
160         for spec in $specs; do
161                 SPEC=$(echo "$spec" | sed -e 's|:.*||')
162                 SPEC=${SPEC%.spec}
163                 BRANCH=$(echo "$spec" | sed -e 's|.*:||')
164                 echo >&2 "Distfiles Request: $SPEC:$BRANCH via $MAILER ${VIA_ARGS:+ ($VIA_ARGS)}"
165                 cat <<-EOF | "$MAILER" -t -i $VIA_ARGS
166                         To: $DMAIL
167                         From: $LOGIN <$LOGIN@$HOST>
168                         Subject: fetchsrc_request notify
169                         X-distfiles-request: yes
170                         X-Login: $LOGIN
171                         X-Package: $SPEC
172                         X-Branch: $BRANCH
173                         X-Flags: force-reply
174
175                         .
176                         EOF
177         done
178 }
179
180 # autotag from rpm-build-macros
181 # displays latest used tag for a specfile
182 autotag() {
183         local out s
184         for s in "$@"; do
185                 # strip branches
186                 s=${s%:*}
187                 # ensure package ends with .spec
188                 s=${s%.spec}.spec
189                 git fetch --tags
190                 out=$(git for-each-ref --count=1 --sort=-authordate refs/tags/auto/$dist \
191                         --format='%(refname:short)')
192                 echo "$s:$out"
193         done
194 }
195
196 # get autotag for specs
197 # WARNING: This may checkout some files from VCS
198 get_autotag() {
199         local atag pkg spec rpmdir
200
201         rpmdir=$(rpm -E %_topdir)
202         for pkg in "$@"; do
203                 cd $rpmdir
204                 # strip branches
205                 pkg=${pkg%:*}
206                 # strip .spec extension
207                 pkg=${pkg%.spec}
208                 wget http://pkgrevs.tld-linux.org/list/$pkg -O /tmp/$pkg.tags 1>/dev/null 2>&1
209                 atag=$(head -n 1 /tmp/$pkg.tags)
210                 rm -f /tmp/$pkg.tags
211                 if [ "x$atag" = "x" ]; then
212                         echo "$pkg.spec"
213                 else
214                         echo "$pkg.spec:$atag"
215                 fi
216         done
217 }
218
219 relup() {
220         local script=$(dirname $(rpm -E %_topdir))/rpm-build-tools/relup.sh
221         $script -u -i "$@"
222 }
223
224 usage() {
225         cat <<EOF
226 Usage: make-request.sh [OPTION] ... [SPECFILE] ....
227
228 Mandatory arguments to long options are mandatory for short options too.
229
230       --config-file /path/to/config/file
231             Source additional config file (after $USER_CFG), useful when
232             when sending build requests to Ac/Th from the same account
233       -a
234             Try to use latest auto-tag for the spec when building
235             WARNING: This will checkout new files to your packages dir
236       -b 'BUILDER BUILDER ...',  --builder='BUILDER BUILDER ...'
237             Sends request to given builders (in 'version-arch' format)
238       --with VALUE, --without VALUE
239             Build package with(out) a given bcond
240       --kernel VALUE
241             set alt_kernel to VALUE
242       --target VALUE
243             set --target to VALUE
244       -D "NAME VALUE"|--define "NAME VALUE"
245             define macro named NAME with value VALUE
246       -s BUILD_ID, --skip BUILD_ID[,BUILD_ID][,BUILD_ID]
247             mark build ids on src builder to be skipped (instructs srcbuilder to create 'skipme' file)
248       --branch VALUE
249             specify default branch for specs in request
250       -t, --test-build
251             Performs a 'test-build'. Package will be uploaded to hidden .test-builds/
252             ftp tree and won't be upgraded on builders.
253       -r, --ready-build
254             Preforms a 'ready' build. Package will be built and uploaded to test/ ftp tree
255             (and later moved by release manager staff to ready/ and main ftp tree)
256       -u, --upgrade
257             Forces package upgrade (for use with -c or -q, not -t)
258       --relup
259             Bump package release, see also --relup
260       -m, --message
261             Set commit message for relup
262       -n, --no-upgrade
263             Disables package upgrade (for use with -r)
264       -ni, --no-install-br
265             Do not install missing BuildRequires (--nodeps)
266       -nd, --no-depend
267             Do not add dependency of build jobs, each job in batch runs itself
268       -j, --jobs
269             Number of parallel jobs for single build
270       -f, --flag
271       -d, --dist DISTRIBUTION_ID
272             Specify value for \$dist
273       -df,  --distfiles-fetch[-request] PACKAGE
274             Send distfiles request to fetch sources for PACKAGE
275       -cf, --command-flag
276             Not yet documented
277       -c, --command
278             Executes a given command on builders (prepended to build jobs if build jobs included)
279       -C, --post-command
280             Executes a given command on builders (appended to build jobs if build jobs included)
281       --test-remove-pkg
282             shortcut for --command poldek -evt ARGS
283       --remove-pkg
284             shortcut for --command poldek -ev --noask ARGS
285       --upgrade-pkg
286             shortcut for --command poldek --up -Uv ARGS
287       --pull
288             Updates builders infrastructure (outside chroot)
289       --update-macros
290             Updates rpm-build-macros on src builder
291       --update-scripts
292             Updates ~/rpm/rpm-build-tools on builder
293       --requester username
294             Override the requester
295       -q
296             shortcut for --command rpm -q ARGS
297       -g, --gpg-opts "opts"
298             Pass additional options to gpg binary
299       -p, --priority VALUE
300             sets request priority (default 2)
301       -w SECONDS
302             Wait SECONDS before sending actual request. Note: gpg passphrase is still asked immediately.
303             This may be useful if you just commited package and want to send it
304             for test build after distfiles has fetched the file.
305       -h, --help
306             Displays this help message
307       -v
308             Verbose. Print some more debug on screen
309 EOF
310         exit 0
311 }
312
313 # validate distro, set $dist
314 set_dist() {
315         case "$1" in
316         ti)
317                 ;;
318         ti-dev)
319                 ;;
320         *)
321                 die "dist \`$1' not known"
322                 ;;
323         esac
324
325         dist=$1
326 }
327
328 while [ $# -gt 0 ]; do
329         case "$1" in
330                 -d | --dist | --distro)
331                         set_dist $2
332                         shift
333                         ;;
334
335                 --config-file)
336                         [ -f "$2" ] && . $2 || die "Config file not found"
337                         shift
338                         ;;
339
340                 --builder | -b)
341                         for b in $2; do
342                                 builders="$builders ${b%:*}"
343                         done
344                         shift
345                         ;;
346
347                 -a)
348                         autotag=yes
349                         ;;
350
351                 -m)
352                         shift
353                         message=$1
354                         ;;
355
356                 --relup)
357                         relup=yes
358                         ;;
359
360                 --with)
361                         with="$with $(echo "$2" | tr ',' ' ')"
362                         shift
363                         ;;
364
365                 --without)
366                         without="$without $(echo "$2" | tr ',' ' ')"
367                         shift
368                         ;;
369
370                 --test-build | -t)
371                         build_mode=test
372                         f_upgrade=no
373                         ;;
374
375                 --kernel)
376                         kernel=$2
377                         shift
378                         ;;
379
380                 --target)
381                         target=$2
382                         shift
383                         ;;
384
385                 -D|--define)
386                         value=${2#* }
387                         name=${2%% *}
388                         define="$define$cr$name=$value"
389                         shift
390                         ;;
391
392                 -s|--skip)
393                         skip="$2"
394                         shift
395                         ;;
396
397                 --branch)
398                         branch=$2
399                         shift
400                         ;;
401
402                 --priority | -p)
403                         priority=$2
404                         shift
405                         ;;
406
407                 --ready-build | -r)
408                         build_mode=ready
409                         ;;
410
411                 --upgrade | -u)
412                         f_upgrade=yes
413                         ;;
414
415                 --no-upgrade | -n)
416                         f_upgrade=no
417                         ;;
418
419                 --no-depend | -nd)
420                         no_depend=yes
421                         ;;
422
423                 --no-install-br | -ni)
424                         flags="$flags no-install-br"
425                         ;;
426
427                 -j | --jobs)
428                         jobs="$2"
429                         shift
430                         ;;
431
432                 -j*)
433                         jobs="${1#-j}"
434                         ;;
435
436                 -w)
437                         wait="$2"
438                         shift
439                         ;;
440
441                 -v)
442                         verbose=yes
443                         ;;
444
445                 --flag | -f)
446                         flags="$flags $2"
447                         shift
448                         ;;
449
450                 --command-flags | -cf)
451                         command_flags="$2"
452                         shift
453                         ;;
454
455                 --command | -c)
456                         command="$2"
457                         if [ "$command" = - ]; then
458                                 echo >&2 "Reading command from STDIN"
459                                 echo >&2 "---"
460                                 command=$(cat)
461                                 echo >&2 "---"
462                         fi
463                         shift
464                         ;;
465                 --post-command | -C)
466                         post_command="$2"
467                         if [ "$post_command" = - ]; then
468                                 echo >&2 "Reading post_command from STDIN"
469                                 echo >&2 "---"
470                                 post_command=$(cat)
471                                 echo >&2 "---"
472                         fi
473                         shift
474                         ;;
475                 --test-remove-pkg)
476                         command="poldek -evt $2"
477                         f_upgrade=no
478                         shift
479                         ;;
480                 --remove-pkg)
481                         command="for a in $2; do poldek -ev --noask \$a; done"
482                         f_upgrade=no
483                         shift
484                         ;;
485                 --upgrade-pkg|-Uhv)
486                         command="poldek --up; poldek -uv $2"
487                         f_upgrade=no
488                         shift
489                         ;;
490                 -q)
491                         command="rpm -q $2"
492                         f_upgrade=no
493                         shift
494                         ;;
495
496                 --pull)
497                         command_flags="no-chroot"
498                         command="git pull"
499                         f_upgrade=no
500                         ;;
501
502                 --update-macros)
503                         upgrade_macros="yes"
504                         ;;
505
506                 --update-scripts)
507                         upgrade_scripts='yes'
508                         ;;
509
510                 -df | --distfiles-fetch | --distfiles-fetch-request)
511                         df_fetch=yes
512                         ;;
513
514                 --gpg-opts | -g)
515                         gpg_opts="$2"
516                         shift
517                         ;;
518
519                 --help | -h)
520                         usage
521                         ;;
522
523                 --requester)
524                         requester_override="$2"
525                         shift
526                         ;;
527
528                 -*)
529                         die "unknown knob: $1"
530                         ;;
531
532                 *:* | *)
533                         specs="$specs $1"
534                         ;;
535         esac
536         shift
537 done
538
539 case "$dist" in
540 ti)
541         builder_email="builderti@giant.tld-linux.org"
542         default_builders="ti-*"
543         url="https://giant.tld-linux.org:1241/"
544         control_url="http://giant.tld-linux.org/~builderti"
545         ;;
546 ti-dev)
547         builder_email="buildertidev@giant.tld-linux.org"
548         default_builders="ti-dev-*"
549         url="https://giant.tld-linux.org:1242/"
550         control_url="http://giant.tld-linux.org/~buildertidev"
551         ;;
552 *)
553         die "dist \`$dist' not known"
554         ;;
555 esac
556
557 # need to do this after dist selection
558 if [ "$skip" ]; then
559         skip=$(skip="$skip" control_url="$control_url" python -c '
560 import urllib2
561 import sys
562 import StringIO
563 import gzip
564 import re
565 import os
566 import string
567 from xml.dom import minidom
568
569 skip = os.environ.get("skip").split(",");
570 control_url = os.environ.get("control_url")
571
572 print >> sys.stderr, "* Check queue_id-s against %s" % control_url
573
574 try:
575         headers = { "Cache-Control": "no-cache", "Pragma": "no-cache" }
576         req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
577         f = urllib2.urlopen(req)
578 except Exception, e:
579         print >> sys.stderr, "Fetch error %s: %s" % (control_url + "/queue.gz", e)
580         sys.exit(1)
581
582 sio = StringIO.StringIO()
583 sio.write(f.read())
584 f.close()
585 sio.seek(0)
586 f = gzip.GzipFile(fileobj = sio)
587
588 xml = re.compile("(<queue>.*?</queue>)", re.DOTALL).match(f.read()).group(1)
589 d = minidom.parseString(xml)
590
591 q = []
592 for c in d.documentElement.childNodes:
593         if c.nodeName != "group":
594                 continue
595         q.append(c.attributes["id"].value)
596
597 err = 0
598 for s in skip:
599         if s not in q:
600                 print >> sys.stderr, "- Check %s: ERROR: Not valid queue-id" % s
601                 err = 1
602         else:
603                 print >> sys.stderr, "- Check %s: OK" % s
604 if err == 1:
605         sys.exit(1)
606 print string.join(skip, ",")
607 ') || exit $?
608         f_upgrade=no
609         build_mode=test
610         priority=-1
611         command="skip:$skip"
612         command_flags="no-chroot"
613         builders="$dist-src"
614 fi
615
616 branch=${branch:-$default_branch}
617
618 specs=`for s in $specs; do
619         case "$s" in
620         ^)
621                 # skip marker - pass it along
622                 echo $s
623                 ;;
624         *:*) # package name with branch
625                 _specfile=$(basename ${s%:*})
626                 echo ${_specfile%.spec}.spec:${s##*:}
627                 ;;
628         *) # just package name
629                 echo $(basename ${s%.spec}).spec:$branch
630                 ;;
631         esac
632 done`
633
634 if [ "$relup" = "yes" ]; then
635         msg "Auto relup enabled"
636         relup ${message:+-m "$message"} $specs
637 fi
638
639 if [ "$autotag" = "yes" ]; then
640         msg "Auto autotag build enabled"
641         specs=$(get_autotag $specs)
642 fi
643
644 if [ "$df_fetch" = "yes" ]; then
645         df_fetch $specs
646         exit 0
647 fi
648
649 if [ "$upgrade_macros" = "yes" ]; then
650         command="poldek --up; poldek -uv rpm-build-macros"
651         builders="$dist-src"
652         f_upgrade=no
653         build_mode=test
654 fi
655
656 if [ "$upgrade_scripts" = "yes" ]; then
657         command="cd ~/rpm/rpm-build-tools && git pull"
658         command_flags=as-builder
659         builders="$dist-src"
660         f_upgrade=no
661         build_mode=test
662 fi
663
664 if [[ "$requester" != *@* ]] ; then
665         requester="$requester@tld-linux.org"
666 fi
667
668 if [ -z "$builders" ] ; then
669         builders="$default_builders"
670 fi
671
672 if [ "$f_upgrade" = "yes" ] ; then
673         flags="$flags upgrade"
674 fi
675
676 if [ "$build_mode" = "test" ] ; then
677         if [ "$f_upgrade" = "yes" ] ; then
678                 die "--upgrade and --test-build are mutually exclusive"
679         fi
680         flags="$flags test-build"
681 fi
682
683 if [ -z "$build_mode" ] ; then
684         # missing build mode, builders go crazy when you proceed"
685         die "please specify build mode"
686 fi
687
688
689 ok=
690 for s in $specs; do
691         ok=1
692 done
693
694 if [ -z "$specs" -a -z "$command" ]; then
695         die "no packages to build or command to invoke specified"
696 fi
697
698 id=$(uuidgen)
699
700 gen_req() {
701         echo "<group id='$id' no='0' flags='$flags'>"
702         echo "  <time>$(date +%s)</time>"
703         msg "Using priority $priority"
704         echo "  <priority>$priority</priority>"
705         if [ -n "$jobs" ]; then
706                 msg "Using jobs $jobs"
707                 echo "  <maxjobs>$jobs</maxjobs>"
708         fi
709         if [ -z "$url" ]; then
710                 msg "Using email $builder_email"
711         else
712                 msg "Using URL $url"
713         fi
714
715         if [ "$build_mode" = "ready" ]; then
716                 msg "Build mode: $(tput setaf 2)$build_mode$c_norm"
717         else
718                 msg "Build mode: $(tput setaf 3)$build_mode$c_norm"
719         fi
720
721         if [ -n "$requester_override" ] ; then
722                 echo "  <requester>$requester_override</requester>"
723         fi
724
725         msg "Queue-ID: $id"
726         echo
727
728         # job to depend on
729         local depend=
730         local b i=1 val
731         local name branch builders_xml
732
733         for b in $builders; do
734                 msg "Builder: $(red $b)"
735                 builders_xml="$builders_xml <builder>$b</builder>"
736         done
737
738         if [ "$command" ]; then
739                 bid=$(uuidgen)
740                 echo -E >&2 "* Command: $command"
741                 echo "  <batch id='$bid' depends-on=''>"
742                 echo "           <command flags='$command_flags'>"
743                 hsc "$command"
744                 echo "</command>"
745                 echo "           <info></info>"
746                 echo "$builders_xml"
747                 echo "  </batch>"
748                 depend=$bid
749         fi
750
751         if [ "$f_upgrade" = "yes" ] ; then
752                 msg "Upgrade mode: $f_upgrade"
753         fi
754
755         for s in $specs; do
756                 # skip marker
757                 if [ "$s" = "^" ]; then
758                         depend=
759                         continue
760                 fi
761                 if [ "$no_depend" = yes ]; then
762                         depend=
763                 fi
764                 bid=$(uuidgen)
765                 echo "  <batch id='$bid' depends-on='$depend'>"
766
767                 name=$(echo "$s" | sed -e 's|:.*||')
768                 branch=$(echo "$s" | sed -e 's|.*:||')
769                 msg "Adding #$i $name:$branch${kernel:+ alt_kernel=$kernel}${target:+ target=$target}${depend:+ depends on $depend}"
770                 echo "           <spec>$name</spec>"
771                 echo "           <branch>$branch</branch>"
772                 echo "           ${kernel:+<kernel>$kernel</kernel>}"
773                 echo "           ${target:+<target>$target</target>}"
774
775                 oIFS=$IFS; IFS=$cr
776                 for b in $define; do
777                         [ -z "$b" ] && continue
778                         value=${b#*=}
779                         b=${b%%=*}
780                         echo "           <define name='$(hsc $b)'>$(hsc "$value")</define>"
781                         msg "- define $b=$value"
782                 done
783                 IFS=$oIFS
784
785                 echo "           <info></info>"
786                 echo
787                 for b in $with; do
788                         echo "           <with>$b</with>"
789                 done
790                 for b in $without; do
791                         echo "           <without>$b</without>"
792                 done
793                 echo
794                 echo "$builders_xml"
795                 echo "  </batch>"
796                 i=$((i+1))
797
798                 # let next job depend on previous
799                 depend=$bid
800         done
801
802         if [ "$post_command" ]; then
803                 bid=$(uuidgen)
804                 if [ "$no_depend" = yes ]; then
805                         depend=
806                 fi
807                 echo -E >&2 "* Post-Command: $post_command"
808                 echo "  <batch id='$bid' depends-on='$depend'>"
809                 echo "           <command flags='$command_flags'>"
810                 hsc "$post_command"
811                 echo "</command>"
812                 echo "           <info></info>"
813                 echo "$builders_xml"
814                 echo "  </batch>"
815                 depend=$bid
816         fi
817
818         echo "</group>"
819 }
820
821 gen_email () {
822         # make request first, so the STDERR/STDOUT streams won't be mixed
823         local tmp req
824         tmp=$(mktemp)
825         gen_req > $tmp
826
827         if [ "$verbose" = "yes" ]; then
828                 cat $tmp >&2
829         fi
830
831         cat <<-EOF
832         From: $requester
833         To: $builder_email
834         Subject: build request
835         Message-Id: <$id@$(hostname)>
836         X-New-TLD-Builder: request
837         X-Requester-Version: $VERSION
838
839         EOF
840
841         gpg --clearsign --default-key $default_key $gpg_opts --output=- $tmp
842         rm -f $tmp
843 }
844
845 gen_email | send_request