From c486eff3f809b5ce544d5a032198e7680f2b7f2b Mon Sep 17 00:00:00 2001 From: Stan Cox Date: Tue, 9 Sep 2014 15:07:44 -0400 Subject: [PATCH 1/3] Add -fpic -fPIC to the list of accepted but ignored dtrace options. * dtrace.in (main): Add ignore_options. --- dtrace.in | 4 +++- testsuite/systemtap.base/dtrace.exp | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dtrace.in b/dtrace.in index d5f189d4fc9e..2f9fb6307e28 100644 --- a/dtrace.in +++ b/dtrace.in @@ -305,6 +305,8 @@ def main(): s_filename = "" includes = [] defines = [] + ignore_options = ["-64", "-32", "-fpic", "-fPIC"] + while i < len(sys.argv): if sys.argv[i] == "-o": i += 1 @@ -330,7 +332,7 @@ def main(): HAVE_PYP = False elif sys.argv[i] == "--types": print sys.argv[0] + ": note: obsolete option --types used" - elif sys.argv[i] == "-64" or sys.argv[i] == "-32": + elif sys.argv[i] in ignore_options: pass # dtrace users sometimes pass these flags elif sys.argv[i] == "--help": dtrace_help() diff --git a/testsuite/systemtap.base/dtrace.exp b/testsuite/systemtap.base/dtrace.exp index 252dad90ede5..e029748100d6 100644 --- a/testsuite/systemtap.base/dtrace.exp +++ b/testsuite/systemtap.base/dtrace.exp @@ -53,12 +53,12 @@ set incpath "/tmp/dtrace_inc" # ----------------------------------------------------------------- # test command line option and file handling -verbose -log "$dtrace -G -s $dpath -o XXX.o" -catch {exec $dtrace -G -s $dpath -o XXX.o} +verbose -log "$dtrace -G -64 -fPIC -s $dpath -o XXX.o" +catch {exec $dtrace -G -64 -fPIC -s $dpath -o XXX.o} if {[file exists XXX.o]} then { - pass "dtrace -G -o XXX.o" + pass "dtrace -G -64 -fPIC -o XXX.o" } else { - fail "dtrace -G -o XXX.o" + fail "dtrace -G -64 -fPIC -o XXX.o" } exec rm -f XXX.o -- 1.9.3 From 52cac9d8159a399b824201f4d2c98abe89a01767 Mon Sep 17 00:00:00 2001 From: Stan Cox Date: Tue, 23 Sep 2014 13:42:54 -0400 Subject: [PATCH 2/3] Ignore C declarations in .d file and use string pattern matching as a fallback mechanism. * dtrace.in (_PypProvider): SkipTo the provider{...} (main): If pyparsing fails, then fallback to pattern matching. * dtrace.exp: Add a fallback test. --- dtrace.in | 39 +++++++++++++++++++++++++++--------- testsuite/systemtap.base/dtrace.exp | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/dtrace.in b/dtrace.in index 2f9fb6307e28..04ace92181d9 100644 --- a/dtrace.in +++ b/dtrace.in @@ -29,13 +29,13 @@ try: from pyparsing import alphas, cStyleComment, delimitedList, Group, \ Keyword, lineno, Literal, nestedExpr, nums, oneOf, OneOrMore, \ Optional, ParseException, ParserElement, restOfLine, restOfLine, \ - Suppress, Word, ZeroOrMore + Suppress, SkipTo, Word, ZeroOrMore HAVE_PYP = True except ImportError: HAVE_PYP = False -# Common file creation methods for pyparsing and regexparsing +# Common file creation methods for pyparsing and string pattern matching class _HeaderCreator(object): def init_semaphores(self, fdesc): @@ -149,7 +149,7 @@ class _PypProvider(_HeaderCreator): provider_decl = (PROVIDER + Optional(ident) + lbrace + Group(probe_decls) + rbrace + Optional(semi)) - dtrace_statement = Group(decls | provider_decl) + dtrace_statement = Group (SkipTo("provider", include=False) + provider_decl) self.dtrace_statements = ZeroOrMore(dtrace_statement) cplusplus_linecomment = Literal("//") + restOfLine @@ -167,7 +167,10 @@ class _PypProvider(_HeaderCreator): for asti in self.ast: if len(asti) == 0: continue - elif asti[0] == "provider": + # ignore SkipTo token + if asti[0] != "provider": + del asti[0] + if asti[0] == "provider": # list of probes for prb in asti[2]: semaphores_def += self.add_semaphore(asti[1], prb[1]) @@ -186,15 +189,18 @@ class _PypProvider(_HeaderCreator): self.ast = self.bnf.parseFile(provider).asList() except ParseException, err: if len(self.current_probe): - print "%s:%s:%d: syntax error near:\nprobe %s\n" % (sys.argv[0],provider, self.current_lineno, self.current_probe) + print "Warning: %s:%s:%d: syntax error near:\nprobe %s\n" % (sys.argv[0],provider, self.current_lineno, self.current_probe) else: - print "%s:%s:%d syntax error near:\n%s\n" % (sys.argv[0],provider,err.lineno, err.line) - sys.exit(1) + print "Warning: %s:%s:%d syntax error near:\n%s\n" % (sys.argv[0],provider,err.lineno, err.line) + raise ParseException, err probes_def = "" for asti in self.ast: if len(asti) == 0: continue + # ignore SkipTo token + if asti[0] != "provider": + del asti[0] if asti[0] == "provider": # list of probes for prb in asti[2]: @@ -369,14 +375,29 @@ def main(): providers = _PypProvider() else: providers = _ReProvider() - providers.probe_write(s_filename, filename + suffix) + while True: + try: + providers.probe_write(s_filename, filename + suffix) + break; + # complex C declarations can fool the pyparsing grammar. + # we could increase the complexity of the grammar + # instead we fall back to string pattern matching + except ParseException, err: + print "Warning: Proceeding as if --no-pyparsing was given.\n" + providers = _ReProvider() elif build_source: if HAVE_PYP: providers = _PypProvider() else: providers = _ReProvider() (ignore, fname) = mkstemp(suffix=".h") - providers.probe_write(s_filename, fname) + while True: + try: + providers.probe_write(s_filename, fname) + break; + except ParseException, err: + print "Warning: Proceeding as if --no-pyparsing was given.\n" + providers = _ReProvider() if not keep_temps: os.remove(fname) else: diff --git a/testsuite/systemtap.base/dtrace.exp b/testsuite/systemtap.base/dtrace.exp index e029748100d6..60cab3f5abf2 100644 --- a/testsuite/systemtap.base/dtrace.exp +++ b/testsuite/systemtap.base/dtrace.exp @@ -207,6 +207,46 @@ if { $ok == 4} { } exec rm -f XXX.h +set ok 0 +set pypath "/tmp/pypath.d" +set $fp [open $pypath "w"] +puts $fp " +#include + +provider alpha { + probe request__start(string, uint8_t, uint16_t, int, void *); + probe request__one(string, uint8_t, uint32_t, int, int); + probe client__two(int, int); + probe client__three(int, string, pid_t, zoneid_t); + probe input__stop(int, int, uint32_t, uint32_t, int8_t, uint8_t*, double*); +}; + +#ifdef DCL_AFTER_PROVIDER +typedef unsigned short int __u_short; +typedef const static unsigned short __u_c_short; +#endif + +#pragma D attributes Unknown provider alpha provider +" +close $fp +verbose -log "$dtrace -C -h -s $pypath -o XXX.h" +spawn $dtrace -C -DDCL_AFTER_PROVIDER -h -s $pypath -o XXX.h +expect { + -re {Warning.*syntax error} {incr ok; exp_continue} + -re {Warning.*--no-pyparsing} {incr ok; exp_continue} + eof { } +} +catch {close}; catch {wait} +if {[file exists XXX.h]} then { + incr ok; +} +if { $ok == 3} { + pass "dtrace parser check" +} else { + fail "dtrace parser check $ok" +} +exec rm -f XXX.h + verbose -log "$dtrace -I$incpath -G -s $idpath" catch {exec $dtrace -G -s $dpath} if {[file exists test.o]} then { -- 1.9.3 From 3525152408f15e23dcffe2371bbd575f1646d691 Mon Sep 17 00:00:00 2001 From: Stan Cox Date: Thu, 25 Sep 2014 13:47:04 -0400 Subject: [PATCH 3/3] Add pyparsing / no-parsing compatibility test. * dtrace.exp: Add pyparsing compatibility test. --- testsuite/systemtap.base/dtrace.exp | 93 ++++++++++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 21 deletions(-) diff --git a/testsuite/systemtap.base/dtrace.exp b/testsuite/systemtap.base/dtrace.exp index 60cab3f5abf2..e455c298737a 100644 --- a/testsuite/systemtap.base/dtrace.exp +++ b/testsuite/systemtap.base/dtrace.exp @@ -8,6 +8,8 @@ if {[installtest_p]} { set dtrace ../dtrace } +# Create the test .d files + exec mkdir -p /tmp/dtrace set dpath "/tmp/dtrace/test.d" @@ -48,6 +50,29 @@ provider tstsyscall " close $fp +set pypath "/tmp/pypath.d" +set $fp [open $pypath "w"] +puts $fp " +#include + +provider alpha { + probe request__start(string, uint8_t, uint16_t, int, void *); + probe request__one(string, uint8_t, uint32_t, int, int); + probe client__two(int, int); + probe client__three(int, string, pid_t, zoneid_t); + probe input__stop(int, int, uint32_t, uint32_t, int8_t, uint8_t*, double*); +}; + +#ifdef DCL_AFTER_PROVIDER +typedef unsigned short int __u_short; +typedef const static unsigned short __u_c_short; +#endif + +#pragma D attributes Unknown provider alpha provider +" +close $fp + + set incpath "/tmp/dtrace_inc" # ----------------------------------------------------------------- @@ -156,6 +181,9 @@ if { $ok != 0} { fail "dtrace CFLAGS= CC=" } +# ----------------------------------------------------------------- +# test -h header file creation + set ok 0 verbose -log "$dtrace -C -h -s $dpath -o XXX.h" catch {exec $dtrace -C -h -s $dpath -o XXX.h} @@ -189,6 +217,9 @@ if { $ok == 4} { } exec rm -f XXX.h +# ----------------------------------------------------------------- +# test --no-pyparsing + set ok 0 verbose -log "$dtrace -C --no-pyparsing -I$incpath -h -s $idpath -o XXX.h" catch {exec $dtrace -C --no-pyparsing -I$incpath -h -s $idpath -o XXX.h} @@ -207,28 +238,10 @@ if { $ok == 4} { } exec rm -f XXX.h -set ok 0 -set pypath "/tmp/pypath.d" -set $fp [open $pypath "w"] -puts $fp " -#include - -provider alpha { - probe request__start(string, uint8_t, uint16_t, int, void *); - probe request__one(string, uint8_t, uint32_t, int, int); - probe client__two(int, int); - probe client__three(int, string, pid_t, zoneid_t); - probe input__stop(int, int, uint32_t, uint32_t, int8_t, uint8_t*, double*); -}; - -#ifdef DCL_AFTER_PROVIDER -typedef unsigned short int __u_short; -typedef const static unsigned short __u_c_short; -#endif +# ----------------------------------------------------------------- +# test fallback to --no-pyparsing -#pragma D attributes Unknown provider alpha provider -" -close $fp +set ok 0 verbose -log "$dtrace -C -h -s $pypath -o XXX.h" spawn $dtrace -C -DDCL_AFTER_PROVIDER -h -s $pypath -o XXX.h expect { @@ -247,6 +260,9 @@ if { $ok == 3} { } exec rm -f XXX.h +# ----------------------------------------------------------------- +# test -G object file creation + verbose -log "$dtrace -I$incpath -G -s $idpath" catch {exec $dtrace -G -s $dpath} if {[file exists test.o]} then { @@ -256,5 +272,40 @@ if {[file exists test.o]} then { } exec rm -f test.o +# ----------------------------------------------------------------- +# test dtrace for pyparsing / --no-pyparsing compatibility + +set ok 0 +set dfiles {dtrace} +foreach i $dfiles { + verbose -log "$dtrace $srcdir/$subdir/$i.d" + catch {exec $dtrace -C -h -s $srcdir/$subdir/$i.d -o $i-1.h} + catch {exec $dtrace -C -h --no-pyparsing -s $srcdir/$subdir/$i.d -o $i-2.h} + spawn diff -wqs $i-1.h $i-2.h + expect { + -re {Files.*identical} {incr ok; exp_continue} + eof { } + } + catch {exec $dtrace -C -G -s $srcdir/$subdir/$i.d -o $i-1.o} + catch {exec $dtrace -C -G --no-pyparsing -s $srcdir/$subdir/$i.d -o $i-2.o} + verbose -log "exec nm $i-1.o > $i-1.od" + catch {exec nm $i-1.o > $i-1.od} + catch {exec nm $i-2.o > $i-2.od} + spawn diff -qs $i-1.od $i-2.od + expect { + -re {Files.*identical} {incr ok; exp_continue} + eof { } + } + catch {exec /bin/rm $i-1.h $i-2.h $i-1.o $i-2.o} +} +if { $ok == 2} { + pass "dtrace known uses" +} else { + fail "dtrace known uses ${ok}" +} + +# ----------------------------------------------------------------- +# cleanup + exec /bin/rm -r /tmp/dtrace /tmp/dtrace_inc # ----------------------------------------------------------------- -- 1.9.3