]> TLD Linux GIT Repositories - tld-builder.git/blob - client/make-request.sh
- proper TLD settings
[tld-builder.git] / client / make-request.sh
1 #!/bin/sh
2
3 # prevent "*" from being expanded in builders var
4 set -f
5
6 builders=
7 with=
8 without=
9 flags=
10 command=
11 command_flags=
12 gpg_opts=
13 default_branch='HEAD'
14 dist=
15 url=
16 no_depend=no
17 verbose=no
18 autotag=no
19
20 if [ -x /usr/bin/python ]; then
21         send_mode="python"
22 else
23         echo "No python present, using mail mode"
24         send_mode="mail"
25 fi
26
27 if [ -n "$HOME_ETC" ]; then
28         USER_CFG=$HOME_ETC/.requestrc
29 else
30         USER_CFG=$HOME/.requestrc
31 fi
32
33 if [ ! -f "$USER_CFG" ]; then
34         echo "Creating config file $USER_CFG. You *must* edit it."
35         cat > $USER_CFG <<EOF
36 priority=2
37 requester=deviloper@pld-linux.org
38 default_key=deviloper@pld-linux.org
39 send_mode="$send_mode"
40 url="$url"
41 mailer="/usr/sbin/sendmail -t"
42 gpg_opts=""
43 dist=th
44 url="http://src.th.pld-linux.org:1234/"
45
46 # defaults:
47 f_upgrade=yes
48 EOF
49 exit
50 fi
51
52 if [ -f "$USER_CFG" ]; then
53         . $USER_CFG
54         # legacy fallback
55         if [ "${distro:+set}" = "set" ]; then
56                 dist=$distro
57         fi
58 fi
59
60 # internal options, not to be overriden
61 specs=
62 df_fetch=no
63 upgrade_macros=no
64
65 # Set colors
66 c_star=$(tput setaf 2)
67 c_red=$(tput setaf 1)
68 c_norm=$(tput op)
69 msg() {
70         echo >&2 "${c_star}*${c_norm} $*"
71 }
72 red() {
73         echo "${c_red}$*${c_norm}"
74 }
75
76 die() {
77         echo >&2 "$0: $*"
78         exit 1
79 }
80
81 send_request() {
82         # switch to mail mode, if no url set
83         [ -z "$url" ] && send_mode="mail"
84
85         case "$send_mode" in
86         "mail")
87                 msg "Sending using mail mode"
88                 cat - | $mailer
89                 ;;
90         *)
91                 msg "Sending using http mode to $url"
92                 cat - | python -c '
93 import sys, socket, urllib2
94
95 try:
96         data = sys.stdin.read()
97         url = sys.argv[1]
98         socket.setdefaulttimeout(10)
99         req = urllib2.Request(url, data)
100         f = urllib2.urlopen(req)
101         f.close()
102 except Exception, e:
103         print >> sys.stderr, "Problem while sending request via HTTP: %s: %s" % (url, e)
104         sys.exit(1)
105 print >> sys.stdout, "Request queued via HTTP."
106 ' "$url"
107                 ;;
108         esac
109 }
110
111 # simple df_fetcher, based on packages/fetchsrc_request
112 # TODO: tcp (smtp) mode
113 # TODO: adjust for ~/.requestrc config
114 df_fetch() {
115         local specs="$@"
116
117         # Sending by
118         local MAILER='/usr/sbin/sendmail'
119         # MAILER='/usr/bin/msmtp'
120         # Sending via
121         local VIA="SENDMAIL"
122         #VIA="localhost"
123         local VIA_ARGS=""
124         #VIA_ARGS="some additional flags"
125         # e.g. for msmtp:
126         # VIA_ARGS='-a gmail'
127         #
128         # DISTFILES EMAIL
129         local DMAIL="distfiles@pld-linux.org"
130
131         local HOST=$(hostname -f)
132         local LOGIN=${requester%@*}
133
134         for spec in $specs; do
135                 local SPEC=$(echo "$spec" | sed -e 's|:.*||')
136                 local BRANCH=$(echo "$spec" | sed -e 's|.*:||')
137                 echo >&2 "Distfiles Request: $SPEC:$BRANCH via $MAILER ${VIA_ARGS:+ ($VIA_ARGS)}"
138                 cat <<-EOF | "$MAILER" -t -i $VIA_ARGS
139                         To: $DMAIL
140                         From: $LOGIN <$LOGIN@$HOST>
141                         Subject: fetchsrc_request notify
142                         X-CVS-Module: SPECS
143                         X-distfiles-request: yes
144                         X-Login: $LOGIN
145                         X-Spec: $SPEC
146                         X-Branch: $BRANCH
147                         X-Flags: force-reply
148
149                         .
150                         EOF
151         done
152 }
153
154 # autotag from rpm-build-macros
155 # displays latest used tag for a specfile
156 autotag() {
157         local out s
158         for s in "$@"; do
159                 # strip branches
160                 s=${s%:*}
161                 # ensure package ends with .spec
162                 s=${s%.spec}.spec
163                 out=$(cvs status -v $s | awk "!/Sticky/&&/auto-$dist-/{if (!a++) print \$1}")
164                 echo "$s:$out"
165         done
166 }
167
168 # get autotag for specs
169 # WARNING: This may checkout some files from CVS
170 get_autotag() {
171         local pkg spec rpmdir
172
173         rpmdir=$(rpm -E %_topdir)
174         cd $rpmdir
175         for pkg in "$@"; do
176                 # strip branches
177                 pkg=${pkg%:*}
178                 # strip .spec extension
179                 pkg=${pkg%.spec}
180                 # checkout only if missing
181                 if [ ! -e $pkg/$pkg.spec ]; then
182                         $rpmdir/builder -g $pkg -ns -r HEAD 1>&2
183                 fi
184                 if [ ! -e $pkg/$pkg.spec ]; then
185                         # just print it out, to fallback to base pkg name
186                         echo "$pkg"
187                 else
188                         autotag $pkg/$pkg.spec
189                 fi
190         done
191 }
192
193 usage() {
194         cat <<EOF
195 Usage: make-request.sh [OPTION] ... [SPECFILE] ....
196
197 Mandatory arguments to long options are mandatory for short options too.
198
199       --config-file /path/to/config/file
200             Source additional config file (after $USER_CFG), useful when
201             when sending build requests to Ac/Th from the same account
202       -a
203             Try to use latest auto-tag for the spec when building
204             WARNING: This will checkout new files to your packages dir
205       -b 'BUILDER BUILDER ...',  --builder='BUILDER BUILDER ...'
206             Sends request to given builders (in 'version-arch' format)
207       --with VALUE, --without VALUE
208             Build package with(out) a given bcond
209       --kernel VALUE
210             set alt_kernel to VALUE
211       --target VALUE
212             set --target to VALUE
213       -s BUILD_ID, --skip BUILD_ID[,BUILD_ID][,BUILD_ID]
214             mark build ids on src builder to be skipped
215       --branch VALUE
216             specify default branch for specs in request
217       -t, --test-build
218             Performs a 'test-build'. Package will be uploaded to hidden .test-builds/
219             ftp tree and won't be upgraded on builders.
220       -r, --ready-build
221             Preforms a 'ready' build. Package will be built and uploaded to test/ ftp tree
222             (and later moved by release manager staff to ready/ and main ftp tree)
223       -u, --upgrade
224             Forces package upgrade (for use with -c or -q, not -t)
225       -n, --no-upgrade
226             Disables package upgrade (for use with -r)
227       -ni, --no-install-br
228             Do not install missing BuildRequires (--nodeps)
229       -nd, --no-depend
230             Do not add dependency of build jobs, each job in batch runs itself
231       -j, --jobs
232             Number of parallel jobs for single build
233       -f, --flag
234       -d, --dist DISTRIBUTION_ID
235             Specify value for \$dist
236       -df,  --distfiles-fetch[-request] PACKAGE
237             Send distfiles request to fetch sources for PACKAGE
238       -cf, --command-flag
239             Not yet documented
240       -c, --command
241             Executes a given command on builders (prepended to build jobs if build jobs included)
242       -C, --post-command
243             Executes a given command on builders (appended to build jobs if build jobs included)
244       --test-remove-pkg
245             shortcut for --command poldek -evt ARGS
246       --remove-pkg
247             shortcut for --command poldek -ev --noask ARGS
248       --upgrade-pkg
249             shortcut for --command poldek --up -Uv ARGS
250       --pull
251             Updates builders infrastructure (outside chroot)
252       --update-macros
253             Updates rpm-build-macros on src builder
254       -q
255             shortcut for --command rpm -q ARGS
256       -g, --gpg-opts "opts"
257             Pass additional options to gpg binary
258       -p, --priority VALUE
259             sets request priority (default 2)
260       -h, --help
261             Displays this help message
262 EOF
263         exit 0
264 }
265
266 # validate distro, set $dist
267 set_dist() {
268         case "$1" in
269         ac)
270                 ;;
271         ac-java|ac-xen)
272                 ;;
273         ti)
274                 ;;
275         ti-dev)
276                 ;;
277         th)
278                 ;;
279         th-java)
280                 ;;
281         aidath)
282                 ;;
283         *)
284                 die "dist \`$1' not known"
285                 ;;
286         esac
287
288         dist=$1
289 }
290
291 while [ $# -gt 0 ] ; do
292         case "$1" in
293                 -d | --dist | --distro)
294                         set_dist $2
295                         shift
296                         ;;
297
298                 --config-file)
299                         [ -f "$2" ] && . $2 || die "Config file not found"
300                         shift
301                         ;;
302
303                 --builder | -b)
304                         for b in $2; do
305                                 builders="$builders ${b%:*}"
306                         done
307                         shift
308                         ;;
309
310                 -a)
311                         autotag=yes
312                         ;;
313
314                 --with)
315                         with="$with $(echo "$2" | tr ',' ' ')"
316                         shift
317                         ;;
318
319                 --without)
320                         without="$without $(echo "$2" | tr ',' ' ')"
321                         shift
322                         ;;
323
324                 --test-build | -t)
325                         build_mode=test
326                         f_upgrade=no
327                         ;;
328
329                 --kernel)
330                         kernel=$2
331                         shift
332                         ;;
333
334                 --target)
335                         target=$2
336                         shift
337                         ;;
338
339                 -s|--skip)
340                         skip="$2"
341                         shift
342                         ;;
343
344                 --branch)
345                         branch=$2
346                         shift
347                         ;;
348
349                 --priority | -p)
350                         priority=$2
351                         shift
352                         ;;
353
354                 --ready-build | -r)
355                         build_mode=ready
356                         ;;
357
358                 --upgrade | -u)
359                         f_upgrade=yes
360                         ;;
361
362                 --no-upgrade | -n)
363                         f_upgrade=no
364                         ;;
365
366                 --no-depend | -nd)
367                         no_depend=yes
368                         ;;
369
370                 --no-install-br | -ni)
371                         flags="$flags no-install-br"
372                         ;;
373
374                 -j | --jobs)
375                         jobs="$2"
376                         shift
377                         ;;
378
379                 -j*)
380                         jobs="${1#-j}"
381                         ;;
382
383                 -v)
384                         verbose=yes
385                         ;;
386
387                 --flag | -f)
388                         flags="$flags $2"
389                         shift
390                         ;;
391
392                 --command-flags | -cf)
393                         command_flags="$2"
394                         shift
395                         ;;
396
397                 --command | -c)
398                         command="$2"
399                         if [ "$command" = - ]; then
400                                 echo >&2 "Reading command from STDIN"
401                                 echo >&2 "---"
402                                 command=$(cat)
403                                 echo >&2 "---"
404                         fi
405                         shift
406                         ;;
407                 --post-command | -C)
408                         post_command="$2"
409                         if [ "$post_command" = - ]; then
410                                 echo >&2 "Reading post_command from STDIN"
411                                 echo >&2 "---"
412                                 post_command=$(cat)
413                                 echo >&2 "---"
414                         fi
415                         shift
416                         ;;
417                 --test-remove-pkg)
418                         command="poldek -evt $2"
419                         f_upgrade=no
420                         shift
421                         ;;
422                 --remove-pkg)
423                         command="for a in $2; do poldek -ev --noask \$a; done"
424                         f_upgrade=no
425                         shift
426                         ;;
427                 --upgrade-pkg|-Uhv)
428                         command="poldek --up; poldek -uv $2"
429                         f_upgrade=no
430                         shift
431                         ;;
432                 -q)
433                         command="rpm -q $2"
434                         f_upgrade=no
435                         shift
436                         ;;
437
438                 --pull)
439                         command_flags="no-chroot"
440                         command="git pull"
441                         f_upgrade=no
442                         ;;
443
444                 --update-macros)
445                         upgrade_macros="yes"
446                         ;;
447
448                 -df | --distfiles-fetch | --distfiles-fetch-request)
449                         df_fetch=yes
450                         ;;
451
452                 --gpg-opts | -g)
453                         gpg_opts="$2"
454                         shift
455                         ;;
456
457                 --help | -h)
458                         usage
459                         ;;
460
461                 -*)
462                         die "unknown knob: $1"
463                         ;;
464
465                 *:* | *)
466                         specs="$specs $1"
467                         ;;
468         esac
469         shift
470 done
471
472 case "$dist" in
473 ac)
474         builder_email="builder-ac@pld-linux.org"
475         default_builders="ac-*"
476         default_branch="AC-branch"
477         url="http://ep09.pld-linux.org:1289/"
478         control_url="http://ep09.pld-linux.org/~buildsrc"
479         ;;
480 ac-java) # fake "dist" for java available ac architectures
481         builder_email="builder-ac@pld-linux.org"
482         default_builders="ac-i586 ac-i686 ac-athlon ac-amd64"
483         default_branch="AC-branch"
484         url="http://ep09.pld-linux.org:1289/"
485         ;;
486 ac-xen) # fake "dist" for xen-enabled architectures
487         builder_email="builder-ac@pld-linux.org"
488         default_builders="ac-i686 ac-athlon ac-amd64"
489         default_branch="AC-branch"
490         ;;
491 ti)
492         builder_email="builderti@kraz.tld-linux.org"
493         default_builders="ti-*"
494         url="http://kraz.tld-linux.org:1231/"
495         control_url="http://kraz.tld-linux.org/~builderti"
496         ;;
497 ti-dev)
498         builder_email="buildertidev@kraz.tld-linux.org"
499         default_builders="ti-dev-*"
500         url="http://kraz.tld-linux.org:1232/"
501         control_url="http://kraz.tld-linux.org/~buildertidev"
502         ;;
503 th)
504         builder_email="builderth@pld-linux.org"
505         default_builders="th-*"
506         url="http://src.th.pld-linux.org:1234/"
507         control_url="http://src.th.pld-linux.org"
508         ;;
509 th-java) # fake "dist" for java available th architectures
510         builder_email="builderth@pld-linux.org"
511         default_builders="th-x86_64 th-athlon th-i686"
512         url="http://src.th.pld-linux.org:1234/"
513         ;;
514 aidath)
515         builder_email="builderaidath@ep09.pld-linux.org"
516         default_builders="aidath-*"
517         ;;
518 *)
519         die "dist \`$dist' not known"
520         ;;
521 esac
522
523 # need to do this after dist selection
524 if [ "$skip" ]; then
525         skip=$(skip="$skip" control_url="$control_url" python -c '
526 import urllib2
527 import sys
528 import StringIO
529 import gzip
530 import re
531 import os
532 import string
533 from xml.dom import minidom
534
535 skip = os.environ.get("skip").split(",");
536 control_url = os.environ.get("control_url")
537
538 print >> sys.stderr, "* Check queue_id-s against %s" % control_url
539
540 try:
541         headers = { "Cache-Control": "no-cache", "Pragma": "no-cache" }
542         req = urllib2.Request(url=control_url + "/queue.gz", headers=headers)
543         f = urllib2.urlopen(req)
544 except Exception, e:
545         print >> sys.stderr, "Fetch error %s: %s" % (control_url + "/queue.gz", e)
546         sys.exit(1)
547
548 sio = StringIO.StringIO()
549 sio.write(f.read())
550 f.close()
551 sio.seek(0)
552 f = gzip.GzipFile(fileobj = sio)
553
554 xml = re.compile("(<queue>.*?</queue>)", re.DOTALL).match(f.read()).group(1)
555 d = minidom.parseString(xml)
556
557 q = []
558 for c in d.documentElement.childNodes:
559         if c.nodeName != "group":
560                 continue
561         q.append(c.attributes["id"].value)
562
563 err = 0
564 for s in skip:
565         if s not in q:
566                 print >> sys.stderr, "- Check %s: ERROR: Not valid queue-id" % s
567                 err = 1
568         else:
569                 print >> sys.stderr, "- Check %s: OK" % s
570 if err == 1:
571         sys.exit(1)
572 print string.join(skip, ",")
573 ') || exit $?
574         f_upgrade=no
575         build_mode=test
576         priority=-1
577         command="skip:$skip"
578         command_flags="no-chroot"
579         builders="$dist-src"
580 fi
581
582 branch=${branch:-$default_branch}
583
584 specs=`for s in $specs; do
585         case "$s" in
586         ^)
587                 # skip marker - pass it along
588                 echo $s
589                 ;;
590         *.spec:*) # spec with branch
591                 basename $s
592                 ;;
593         *.spec) # spec without branch
594                 echo $(basename $s):$branch
595                 ;;
596         *:*) # package name with branch
597                 basename $s | sed -e 's/:/.spec:/'
598                 ;;
599         *) # just package name
600                 echo $(basename $s).spec:$branch
601                 ;;
602         esac
603 done`
604
605 if [ "$autotag" = "yes" ]; then
606         msg "Auto autotag build enabled"
607         specs=$(get_autotag $specs)
608 fi
609
610 if [ "$df_fetch" = "yes" ]; then
611         df_fetch $specs
612         exit 0
613 fi
614
615 if [ "$upgrade_macros" = "yes" ]; then
616         command="poldek --up; poldek -uv rpm-build-macros"
617         builders="$dist-src"
618         f_upgrade=no
619         build_mode=test
620 fi
621
622 if [[ "$requester" != *@* ]] ; then
623         requester="$requester@pld-linux.org"
624 fi
625
626 if [ -z "$builders" ] ; then
627         builders="$default_builders"
628 fi
629
630 if [ "$f_upgrade" = "yes" ] ; then
631         flags="$flags upgrade"
632 fi
633
634 if [ "$build_mode" = "test" ] ; then
635         if [ "$f_upgrade" = "yes" ] ; then
636                 die "--upgrade and --test-build are mutually exclusive"
637         fi
638         flags="$flags test-build"
639 fi
640
641 if [ -z "$build_mode" ] ; then
642         # missing build mode, builders go crazy when you proceed"
643         die "please specify build mode"
644 fi
645
646
647 ok=
648 for s in $specs; do
649         ok=1
650 done
651
652 if [ -z "$specs" -a -z "$command" ]; then
653         die "no packages to build or command to invoke specified"
654 fi
655
656 id=$(uuidgen)
657
658 gen_req() {
659         echo "<group id='$id' no='0' flags='$flags'>"
660         echo "  <time>$(date +%s)</time>"
661         msg "Using priority $priority"
662         echo "  <priority>$priority</priority>"
663         if [ -n "$jobs" ]; then
664                 msg "Using jobs $jobs"
665                 echo "  <maxjobs>$jobs</maxjobs>"
666         fi
667         if [ -z "$url" ]; then
668                 msg "Using email $builder_email"
669         else
670                 msg "Using URL $url"
671         fi
672
673         if [ "$build_mode" = "ready" ]; then
674                 msg "Build mode: $(tput setaf 2)$build_mode$c_norm"
675         else
676                 msg "Build mode: $(tput setaf 3)$build_mode$c_norm"
677         fi
678
679         msg "Queue-ID: $id"
680         echo
681
682         # job to depend on
683         local depend=
684         local b i=1
685         local name branch builders_xml
686
687         for b in $builders; do
688                 msg "Builder: $(red $b)"
689                 builders_xml="$builders_xml <builder>$b</builder>"
690         done
691
692         if [ "$command" ]; then
693                 bid=$(uuidgen)
694                 echo -E >&2 "* Command: $command"
695                 echo "  <batch id='$bid' depends-on=''>"
696                 echo "           <command flags='$command_flags'>"
697                 echo -E "$command" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
698                 echo "</command>"
699                 echo "           <info></info>"
700                 echo "$builders_xml"
701                 echo "  </batch>"
702                 depend=$bid
703         fi
704
705         if [ "$f_upgrade" = "yes" ] ; then
706                 msg "Upgrade mode: $f_upgrade"
707         fi
708
709         for s in $specs; do
710                 # skip marker
711                 if [ "$s" = "^" ]; then
712                         depend=
713                         continue
714                 fi
715                 if [ "$no_depend" = yes ]; then
716                         depend=
717                 fi
718                 bid=$(uuidgen)
719                 echo "  <batch id='$bid' depends-on='$depend'>"
720
721                 name=$(echo "$s" | sed -e 's|:.*||')
722                 branch=$(echo "$s" | sed -e 's|.*:||')
723                 msg "Adding #$i $name:$branch${kernel:+ alt_kernel=$kernel}${target:+ target=$target}${depend:+ depends on $depend}"
724                 echo "           <spec>$name</spec>"
725                 echo "           <branch>$branch</branch>"
726                 echo "           ${kernel:+<kernel>$kernel</kernel>}"
727                 echo "           ${target:+<target>$target</target>}"
728                 echo "           <info></info>"
729                 echo
730                 for b in $with; do
731                         echo "           <with>$b</with>"
732                 done
733                 for b in $without; do
734                         echo "           <without>$b</without>"
735                 done
736                 echo
737                 echo "$builders_xml"
738                 echo "  </batch>"
739                 i=$((i+1))
740
741                 # let next job depend on previous
742                 depend=$bid
743         done
744
745         if [ "$post_command" ]; then
746                 bid=$(uuidgen)
747                 if [ "$no_depend" = yes ]; then
748                         depend=
749                 fi
750                 echo -E >&2 "* Post-Command: $post_command"
751                 echo "  <batch id='$bid' depends-on='$depend'>"
752                 echo "           <command flags='$command_flags'>"
753                 echo -E "$post_command" | sed -e 's,&,\&amp;,g;s,<,\&lt;,g;s,>,\&gt;,g'
754                 echo "</command>"
755                 echo "           <info></info>"
756                 echo "$builders_xml"
757                 echo "  </batch>"
758                 depend=$bid
759         fi
760
761         echo "</group>"
762 }
763
764 gen_email () {
765         # make request first, so the STDERR/STDOUT streams won't be mixed
766         local tmp req
767         tmp=$(mktemp)
768         gen_req > $tmp
769
770         if [ "$verbose" = "yes" ]; then
771                 cat $tmp >&2
772         fi
773
774         cat <<-EOF
775         From: $requester
776         To: $builder_email
777         Subject: build request
778         Message-Id: <$id@$(hostname)>
779         X-New-PLD-Builder: request
780         X-Requester-Version: \$Id$
781
782         EOF
783
784         gpg --clearsign --default-key $default_key $gpg_opts --output=- $tmp
785         rm -f $tmp
786 }
787
788 gen_email | send_request