]> TLD Linux GIT Repositories - tld-builder.git/blob - client/make-request.sh
- switch to HTTPS, disable SSL certificate validation
[tld-builder.git] / client / make-request.sh
1 #!/bin/sh
2 # vim:noet:ts=4:sw=4
3 VERSION=1.89
4
5 # prevent "*" from being expanded in builders var
6 set -f
7
8 builders=
9 with=
10 without=
11 flags=
12 command=
13 command_flags=
14 gpg_opts=
15 default_branch='HEAD'
16 dist=
17 url=
18 no_depend=no
19 verbose=no
20 autotag=no
21 requester_override=
22 relup=no
23
24 if [ -x /usr/bin/python ]; then
25         send_mode="python"
26 else
27         echo "No python present, using mail mode"
28         send_mode="mail"
29 fi
30
31 if [ -n "$HOME_ETC" ]; then
32         USER_CFG=$HOME_ETC/.requestrc
33 else
34         USER_CFG=$HOME/.requestrc
35 fi
36
37 if [ ! -f "$USER_CFG" ]; then
38         echo "Creating config file $USER_CFG. You *must* edit it."
39         cat > $USER_CFG <<EOF
40 priority=2
41 requester=deviloper@tld-linux.org
42 default_key=deviloper@tld-linux.org
43 send_mode="$send_mode"
44 url="$url"
45 mailer="/usr/lib/sendmail -t"
46 gpg_opts=""
47 dist=ti
48 url="https://kraz.tld-linux.org:1241/"
49
50 # defaults:
51 f_upgrade=yes
52 EOF
53 exit
54 fi
55
56 if [ -f "$USER_CFG" ]; then
57         . $USER_CFG
58         # legacy fallback
59         if [ "${distro:+set}" = "set" ]; then
60                 dist=$distro
61         fi
62 fi
63
64 # internal options, not to be overriden
65 specs=
66 df_fetch=no
67 upgrade_macros=no
68 upgrade_scripts=no
69 cr=$(printf "\r")
70
71 # Set colors
72 c_star=$(tput setaf 2)
73 c_red=$(tput setaf 1)
74 c_norm=$(tput op)
75 msg() {
76         echo >&2 "${c_star}*${c_norm} $*"
77 }
78 red() {
79         echo "${c_red}$*${c_norm}"
80 }
81
82 die() {
83         echo >&2 "$0: $*"
84         exit 1
85 }
86
87 send_request() {
88         # switch to mail mode, if no url set
89         [ -z "$url" ] && send_mode="mail"
90
91         if [ -n "$wait" ]; then
92                 msg "Waiting $wait seconds before sending request"
93                 sleep $wait
94                 msg "Wait has ended, proceeding!"
95         fi
96
97         case "$send_mode" in
98         "mail")
99                 msg "Sending using mail mode"
100                 cat - | $mailer
101                 ;;
102         *)
103                 msg "Sending using HTTP mode to $url"
104                 cat - | python -c '
105 import sys, socket, urllib2, ssl
106
107 try:
108     _create_unverified_https_context = ssl._create_unverified_context
109 except AttributeError:
110     pass
111 else:
112     ssl._create_default_https_context = _create_unverified_https_context
113
114 try:
115         data = sys.stdin.read()
116         url = sys.argv[1]
117         socket.setdefaulttimeout(30)
118         req = urllib2.Request(url, data)
119         f = urllib2.urlopen(req)
120         f.close()
121 except Exception, e:
122         print >> sys.stderr, "Problem while sending request via HTTP: %s: %s" % (url, e)
123         sys.exit(1)
124 print >> sys.stdout, "Request queued via HTTP."
125 ' "$url"
126                 ;;
127         esac
128 }
129
130 # htmlspecialchars: escape <, > and &
131 hsc() {
132         local input=$1
133         echo -E "$input" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
134 }
135
136 # simple df_fetcher, based on packages/fetchsrc_request
137 # TODO: tcp (smtp) mode
138 # TODO: adjust for ~/.requestrc config
139 df_fetch() {
140         local specs="$@"
141
142         # Sending by
143         local MAILER='/usr/lib/sendmail'
144         # MAILER='/usr/bin/msmtp'
145         # Sending via
146         local VIA="SENDMAIL"
147         #VIA="localhost"
148         local VIA_ARGS=""
149         #VIA_ARGS="some additional flags"
150         # e.g. for msmtp:
151         # VIA_ARGS='-a gmail'
152         #
153         # DISTFILES EMAIL
154         local DMAIL="df@tld-linux.org"
155
156         local HOST=$(hostname -f)
157         local LOGIN=${requester%@*}
158
159         local SPEC BRANCH
160         for spec in $specs; do
161                 SPEC=$(echo "$spec" | sed -e 's|:.*||')
162                 SPEC=${SPEC%.spec}
163                 BRANCH=$(echo "$spec" | sed -e 's|.*:||')
164                 echo >&2 "Distfiles Request: $SPEC:$BRANCH via $MAILER ${VIA_ARGS:+ ($VIA_ARGS)}"
165                 cat <<-EOF | "$MAILER" -t -i $VIA_ARGS
166                         To: $DMAIL
167                         From: $LOGIN <$LOGIN@$HOST>
168                         Subject: fetchsrc_request notify
169                         X-distfiles-request: yes
170                         X-Login: $LOGIN
171                         X-Package: $SPEC
172                         X-Branch: $BRANCH
173                         X-Flags: force-reply
174
175                         .
176                         EOF
177         done
178 }
179
180 # autotag from rpm-build-macros
181 # displays latest used tag for a specfile
182 autotag() {
183         local out s
184         for s in "$@"; do
185                 # strip branches
186                 s=${s%:*}
187                 # ensure package ends with .spec
188                 s=${s%.spec}.spec
189                 git fetch --tags
190                 out=$(git for-each-ref --count=1 --sort=-authordate refs/tags/auto/$dist \
191                         --format='%(refname:short)')
192                 echo "$s:$out"
193         done
194 }
195
196 # get autotag for specs
197 # WARNING: This may checkout some files from VCS
198 get_autotag() {
199         local pkg spec rpmdir
200
201         rpmdir=$(rpm -E %_topdir)
202         for pkg in "$@"; do
203                 cd $rpmdir
204                 # strip branches
205                 pkg=${pkg%:*}
206                 # strip .spec extension
207                 pkg=${pkg%.spec}
208                 # checkout only if missing
209                 if [ ! -e $pkg/$pkg.spec ]; then
210                         $rpmdir/builder -g $pkg -ns -r HEAD 1>&2
211                 fi
212                 if [ ! -e $pkg/$pkg.spec ]; then
213                         # just print it out, to fallback to base pkg name
214                         echo "$pkg"
215                 else
216                         cd $pkg
217                         autotag $pkg.spec
218                 fi
219         done
220 }
221
222 relup() {
223         local script=$(dirname $(rpm -E %_topdir))/rpm-build-tools/relup.sh
224         $script -u -i "$@"
225 }
226
227 usage() {
228         cat <<EOF
229 Usage: make-request.sh [OPTION] ... [SPECFILE] ....
230
231 Mandatory arguments to long options are mandatory for short options too.
232
233       --config-file /path/to/config/file
234             Source additional config file (after $USER_CFG), useful when
235             when sending build requests to Ac/Th from the same account
236       -a
237             Try to use latest auto-tag for the spec when building
238             WARNING: This will checkout new files to your packages dir
239       -b 'BUILDER BUILDER ...',  --builder='BUILDER BUILDER ...'
240             Sends request to given builders (in 'version-arch' format)
241       --with VALUE, --without VALUE
242             Build package with(out) a given bcond
243       --kernel VALUE
244             set alt_kernel to VALUE
245       --target VALUE
246             set --target to VALUE
247       -D "NAME VALUE"|--define "NAME VALUE"
248             define macro named NAME with value VALUE
249       -s BUILD_ID, --skip BUILD_ID[,BUILD_ID][,BUILD_ID]
250             mark build ids on src builder to be skipped (instructs srcbuilder to create 'skipme' file)
251       --branch VALUE
252             specify default branch for specs in request
253       -t, --test-build
254             Performs a 'test-build'. Package will be uploaded to hidden .test-builds/
255             ftp tree and won't be upgraded on builders.
256       -r, --ready-build
257             Preforms a 'ready' build. Package will be built and uploaded to test/ ftp tree
258             (and later moved by release manager staff to ready/ and main ftp tree)
259       -u, --upgrade
260             Forces package upgrade (for use with -c or -q, not -t)
261       --relup
262             Bump package release, see also --relup
263       -m, --message
264             Set commit message for relup
265       -n, --no-upgrade
266             Disables package upgrade (for use with -r)
267       -ni, --no-install-br
268             Do not install missing BuildRequires (--nodeps)
269       -nd, --no-depend
270             Do not add dependency of build jobs, each job in batch runs itself
271       -j, --jobs
272             Number of parallel jobs for single build
273       -f, --flag
274       -d, --dist DISTRIBUTION_ID
275             Specify value for \$dist
276       -df,  --distfiles-fetch[-request] PACKAGE
277             Send distfiles request to fetch sources for PACKAGE
278       -cf, --command-flag
279             Not yet documented
280       -c, --command
281             Executes a given command on builders (prepended to build jobs if build jobs included)
282       -C, --post-command
283             Executes a given command on builders (appended to build jobs if build jobs included)
284       --test-remove-pkg
285             shortcut for --command poldek -evt ARGS
286       --remove-pkg
287             shortcut for --command poldek -ev --noask ARGS
288       --upgrade-pkg
289             shortcut for --command poldek --up -Uv ARGS
290       --pull
291             Updates builders infrastructure (outside chroot)
292       --update-macros
293             Updates rpm-build-macros on src builder
294       --update-scripts
295             Updates ~/rpm/rpm-build-tools on builder
296       --requester username
297             Override the requester
298       -q
299             shortcut for --command rpm -q ARGS
300       -g, --gpg-opts "opts"
301             Pass additional options to gpg binary
302       -p, --priority VALUE
303             sets request priority (default 2)
304       -w SECONDS
305             Wait SECONDS before sending actual request. Note: gpg passphrase is still asked immediately.
306             This may be useful if you just commited package and want to send it
307             for test build after distfiles has fetched the file.
308       -h, --help
309             Displays this help message
310       -v
311             Verbose. Print some more debug on screen
312 EOF
313         exit 0
314 }
315
316 # validate distro, set $dist
317 set_dist() {
318         case "$1" in
319         ti)
320                 ;;
321         ti-dev)
322                 ;;
323         *)
324                 die "dist \`$1' not known"
325                 ;;
326         esac
327
328         dist=$1
329 }
330
331 while [ $# -gt 0 ]; do
332         case "$1" in
333                 -d | --dist | --distro)
334                         set_dist $2
335                         shift
336                         ;;
337
338                 --config-file)
339                         [ -f "$2" ] && . $2 || die "Config file not found"
340                         shift
341                         ;;
342
343                 --builder | -b)
344                         for b in $2; do
345                                 builders="$builders ${b%:*}"
346                         done
347                         shift
348                         ;;
349
350                 -a)
351                         autotag=yes
352                         ;;
353
354                 -m)
355                         shift
356                         message=$1
357                         ;;
358
359                 --relup)
360                         relup=yes
361                         ;;
362
363                 --with)
364                         with="$with $(echo "$2" | tr ',' ' ')"
365                         shift
366                         ;;
367
368                 --without)
369                         without="$without $(echo "$2" | tr ',' ' ')"
370                         shift
371                         ;;
372
373                 --test-build | -t)
374                         build_mode=test
375                         f_upgrade=no
376                         ;;
377
378                 --kernel)
379                         kernel=$2
380                         shift
381                         ;;
382
383                 --target)
384                         target=$2
385                         shift
386                         ;;
387
388                 -D|--define)
389                         value=${2#* }
390                         name=${2%% *}
391                         define="$define$cr$name=$value"
392                         shift
393                         ;;
394
395                 -s|--skip)
396                         skip="$2"
397                         shift
398                         ;;
399
400                 --branch)
401                         branch=$2
402                         shift
403                         ;;
404
405                 --priority | -p)
406                         priority=$2
407                         shift
408                         ;;
409
410                 --ready-build | -r)
411                         build_mode=ready
412                         ;;
413
414                 --upgrade | -u)
415                         f_upgrade=yes
416                         ;;
417
418                 --no-upgrade | -n)
419                         f_upgrade=no
420                         ;;
421
422                 --no-depend | -nd)
423                         no_depend=yes
424                         ;;
425
426                 --no-install-br | -ni)
427                         flags="$flags no-install-br"
428                         ;;
429
430                 -j | --jobs)
431                         jobs="$2"
432                         shift
433                         ;;
434
435                 -j*)
436                         jobs="${1#-j}"
437                         ;;
438
439                 -w)
440                         wait="$2"
441                         shift
442                         ;;
443
444                 -v)
445                         verbose=yes
446                         ;;
447
448                 --flag | -f)
449                         flags="$flags $2"
450                         shift
451                         ;;
452
453                 --command-flags | -cf)
454                         command_flags="$2"
455                         shift
456                         ;;
457
458                 --command | -c)
459                         command="$2"
460                         if [ "$command" = - ]; then
461                                 echo >&2 "Reading command from STDIN"
462                                 echo >&2 "---"
463                                 command=$(cat)
464                                 echo >&2 "---"
465                         fi
466                         shift
467                         ;;
468                 --post-command | -C)
469                         post_command="$2"
470                         if [ "$post_command" = - ]; then
471                                 echo >&2 "Reading post_command from STDIN"
472                                 echo >&2 "---"
473                                 post_command=$(cat)
474                                 echo >&2 "---"
475                         fi
476                         shift
477                         ;;
478                 --test-remove-pkg)
479                         command="poldek -evt $2"
480                         f_upgrade=no
481                         shift
482                         ;;
483                 --remove-pkg)
484                         command="for a in $2; do poldek -ev --noask \$a; done"
485                         f_upgrade=no
486                         shift
487                         ;;
488                 --upgrade-pkg|-Uhv)
489                         command="poldek --up; poldek -uv $2"
490                         f_upgrade=no
491                         shift
492                         ;;
493                 -q)
494                         command="rpm -q $2"
495                         f_upgrade=no
496                         shift
497                         ;;
498
499                 --pull)
500                         command_flags="no-chroot"
501                         command="git pull"
502                         f_upgrade=no
503                         ;;
504
505                 --update-macros)
506                         upgrade_macros="yes"
507                         ;;
508
509                 --update-scripts)
510                         upgrade_scripts='yes'
511                         ;;
512
513                 -df | --distfiles-fetch | --distfiles-fetch-request)
514                         df_fetch=yes
515                         ;;
516
517                 --gpg-opts | -g)
518                         gpg_opts="$2"
519                         shift
520                         ;;
521
522                 --help | -h)
523                         usage
524                         ;;
525
526                 --requester)
527                         requester_override="$2"
528                         shift
529                         ;;
530
531                 -*)
532                         die "unknown knob: $1"
533                         ;;
534
535                 *:* | *)
536                         specs="$specs $1"
537                         ;;
538         esac
539         shift
540 done
541
542 case "$dist" in
543 ti)
544         builder_email="builderti@kraz.tld-linux.org"
545         default_builders="ti-*"
546         url="https://kraz.tld-linux.org:1241/"
547         control_url="http://kraz.tld-linux.org/~builderti"
548         ;;
549 ti-dev)
550         builder_email="buildertidev@kraz.tld-linux.org"
551         default_builders="ti-dev-*"
552         url="https://kraz.tld-linux.org:1242/"
553         control_url="http://kraz.tld-linux.org/~buildertidev"
554         ;;
555 *)
556         die "dist \`$dist' not known"
557         ;;
558 esac
559
560 # need to do this after dist selection
561 if [ "$skip" ]; then
562         skip=$(skip="$skip" control_url="$control_url" python -c '
563 import urllib2
564 import sys
565 import StringIO
566 import gzip
567 import re
568 import os
569 import string
570 from xml.dom import minidom
571
572 skip = os.environ.get("skip").split(",");
573 control_url = os.environ.get("control_url")
574
575 print >> sys.stderr, "* Check queue_id-s against %s" % control_url
576
577 try:
578         headers = { "Cache-Control": "no-cache", "Pragma": "no-cache" }
579         req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
580         f = urllib2.urlopen(req)
581 except Exception, e:
582         print >> sys.stderr, "Fetch error %s: %s" % (control_url + "/queue.gz", e)
583         sys.exit(1)
584
585 sio = StringIO.StringIO()
586 sio.write(f.read())
587 f.close()
588 sio.seek(0)
589 f = gzip.GzipFile(fileobj = sio)
590
591 xml = re.compile("(<queue>.*?</queue>)", re.DOTALL).match(f.read()).group(1)
592 d = minidom.parseString(xml)
593
594 q = []
595 for c in d.documentElement.childNodes:
596         if c.nodeName != "group":
597                 continue
598         q.append(c.attributes["id"].value)
599
600 err = 0
601 for s in skip:
602         if s not in q:
603                 print >> sys.stderr, "- Check %s: ERROR: Not valid queue-id" % s
604                 err = 1
605         else:
606                 print >> sys.stderr, "- Check %s: OK" % s
607 if err == 1:
608         sys.exit(1)
609 print string.join(skip, ",")
610 ') || exit $?
611         f_upgrade=no
612         build_mode=test
613         priority=-1
614         command="skip:$skip"
615         command_flags="no-chroot"
616         builders="$dist-src"
617 fi
618
619 branch=${branch:-$default_branch}
620
621 specs=`for s in $specs; do
622         case "$s" in
623         ^)
624                 # skip marker - pass it along
625                 echo $s
626                 ;;
627         *:*) # package name with branch
628                 _specfile=$(basename ${s%:*})
629                 echo ${_specfile%.spec}.spec:${s##*:}
630                 ;;
631         *) # just package name
632                 echo $(basename ${s%.spec}).spec:$branch
633                 ;;
634         esac
635 done`
636
637 if [ "$relup" = "yes" ]; then
638         msg "Auto relup enabled"
639         relup ${message:+-m "$message"} $specs
640 fi
641
642 if [ "$autotag" = "yes" ]; then
643         msg "Auto autotag build enabled"
644         specs=$(get_autotag $specs)
645 fi
646
647 if [ "$df_fetch" = "yes" ]; then
648         df_fetch $specs
649         exit 0
650 fi
651
652 if [ "$upgrade_macros" = "yes" ]; then
653         command="poldek --up; poldek -uv rpm-build-macros"
654         builders="$dist-src"
655         f_upgrade=no
656         build_mode=test
657 fi
658
659 if [ "$upgrade_scripts" = "yes" ]; then
660         command="cd ~/rpm/rpm-build-tools && git pull"
661         command_flags=as-builder
662         builders="$dist-src"
663         f_upgrade=no
664         build_mode=test
665 fi
666
667 if [[ "$requester" != *@* ]] ; then
668         requester="$requester@tld-linux.org"
669 fi
670
671 if [ -z "$builders" ] ; then
672         builders="$default_builders"
673 fi
674
675 if [ "$f_upgrade" = "yes" ] ; then
676         flags="$flags upgrade"
677 fi
678
679 if [ "$build_mode" = "test" ] ; then
680         if [ "$f_upgrade" = "yes" ] ; then
681                 die "--upgrade and --test-build are mutually exclusive"
682         fi
683         flags="$flags test-build"
684 fi
685
686 if [ -z "$build_mode" ] ; then
687         # missing build mode, builders go crazy when you proceed"
688         die "please specify build mode"
689 fi
690
691
692 ok=
693 for s in $specs; do
694         ok=1
695 done
696
697 if [ -z "$specs" -a -z "$command" ]; then
698         die "no packages to build or command to invoke specified"
699 fi
700
701 id=$(uuidgen)
702
703 gen_req() {
704         echo "<group id='$id' no='0' flags='$flags'>"
705         echo "  <time>$(date +%s)</time>"
706         msg "Using priority $priority"
707         echo "  <priority>$priority</priority>"
708         if [ -n "$jobs" ]; then
709                 msg "Using jobs $jobs"
710                 echo "  <maxjobs>$jobs</maxjobs>"
711         fi
712         if [ -z "$url" ]; then
713                 msg "Using email $builder_email"
714         else
715                 msg "Using URL $url"
716         fi
717
718         if [ "$build_mode" = "ready" ]; then
719                 msg "Build mode: $(tput setaf 2)$build_mode$c_norm"
720         else
721                 msg "Build mode: $(tput setaf 3)$build_mode$c_norm"
722         fi
723
724         if [ -n "$requester_override" ] ; then
725                 echo "  <requester>$requester_override</requester>"
726         fi
727
728         msg "Queue-ID: $id"
729         echo
730
731         # job to depend on
732         local depend=
733         local b i=1 val
734         local name branch builders_xml
735
736         for b in $builders; do
737                 msg "Builder: $(red $b)"
738                 builders_xml="$builders_xml <builder>$b</builder>"
739         done
740
741         if [ "$command" ]; then
742                 bid=$(uuidgen)
743                 echo -E >&2 "* Command: $command"
744                 echo "  <batch id='$bid' depends-on=''>"
745                 echo "           <command flags='$command_flags'>"
746                 hsc "$command"
747                 echo "</command>"
748                 echo "           <info></info>"
749                 echo "$builders_xml"
750                 echo "  </batch>"
751                 depend=$bid
752         fi
753
754         if [ "$f_upgrade" = "yes" ] ; then
755                 msg "Upgrade mode: $f_upgrade"
756         fi
757
758         for s in $specs; do
759                 # skip marker
760                 if [ "$s" = "^" ]; then
761                         depend=
762                         continue
763                 fi
764                 if [ "$no_depend" = yes ]; then
765                         depend=
766                 fi
767                 bid=$(uuidgen)
768                 echo "  <batch id='$bid' depends-on='$depend'>"
769
770                 name=$(echo "$s" | sed -e 's|:.*||')
771                 branch=$(echo "$s" | sed -e 's|.*:||')
772                 msg "Adding #$i $name:$branch${kernel:+ alt_kernel=$kernel}${target:+ target=$target}${depend:+ depends on $depend}"
773                 echo "           <spec>$name</spec>"
774                 echo "           <branch>$branch</branch>"
775                 echo "           ${kernel:+<kernel>$kernel</kernel>}"
776                 echo "           ${target:+<target>$target</target>}"
777
778                 oIFS=$IFS; IFS=$cr
779                 for b in $define; do
780                         [ -z "$b" ] && continue
781                         value=${b#*=}
782                         b=${b%%=*}
783                         echo "           <define name='$(hsc $b)'>$(hsc "$value")</define>"
784                         msg "- define $b=$value"
785                 done
786                 IFS=$oIFS
787
788                 echo "           <info></info>"
789                 echo
790                 for b in $with; do
791                         echo "           <with>$b</with>"
792                 done
793                 for b in $without; do
794                         echo "           <without>$b</without>"
795                 done
796                 echo
797                 echo "$builders_xml"
798                 echo "  </batch>"
799                 i=$((i+1))
800
801                 # let next job depend on previous
802                 depend=$bid
803         done
804
805         if [ "$post_command" ]; then
806                 bid=$(uuidgen)
807                 if [ "$no_depend" = yes ]; then
808                         depend=
809                 fi
810                 echo -E >&2 "* Post-Command: $post_command"
811                 echo "  <batch id='$bid' depends-on='$depend'>"
812                 echo "           <command flags='$command_flags'>"
813                 hsc "$post_command"
814                 echo "</command>"
815                 echo "           <info></info>"
816                 echo "$builders_xml"
817                 echo "  </batch>"
818                 depend=$bid
819         fi
820
821         echo "</group>"
822 }
823
824 gen_email () {
825         # make request first, so the STDERR/STDOUT streams won't be mixed
826         local tmp req
827         tmp=$(mktemp)
828         gen_req > $tmp
829
830         if [ "$verbose" = "yes" ]; then
831                 cat $tmp >&2
832         fi
833
834         cat <<-EOF
835         From: $requester
836         To: $builder_email
837         Subject: build request
838         Message-Id: <$id@$(hostname)>
839         X-New-TLD-Builder: request
840         X-Requester-Version: $VERSION
841
842         EOF
843
844         gpg --clearsign --default-key $default_key $gpg_opts --output=- $tmp
845         rm -f $tmp
846 }
847
848 gen_email | send_request