diff options
Diffstat (limited to 'test')
-rw-r--r-- | test/CMakeLists.txt | 2 | ||||
-rw-r--r-- | test/ctest/CTestCustom.ctest.in | 110 | ||||
-rw-r--r-- | test/ctest/scripts/run_promela_test.cmake | 4 | ||||
-rwxr-xr-x | test/w3c/analyze_tests.pl | 101 |
4 files changed, 159 insertions, 58 deletions
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index da70494..bc61a2a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,4 +1,4 @@ -set(TEST_TIMEOUT 15) +set(TEST_TIMEOUT 30) set(TEST_BENCHMARK_ITERATIONS 1000) find_program(SPIN spin) diff --git a/test/ctest/CTestCustom.ctest.in b/test/ctest/CTestCustom.ctest.in index c5c2bd6..f63b925 100644 --- a/test/ctest/CTestCustom.ctest.in +++ b/test/ctest/CTestCustom.ctest.in @@ -134,67 +134,67 @@ set(CTEST_CUSTOM_TESTS_IGNORE ### Ignore for SPIN model checking # manual tests - # "w3c/spin/promela/test178.scxml" # two identical params in _event.raw - FAILED - # "w3c/spin/promela/test230.scxml" # autoforwarded events are identical - PASSED - # "w3c/spin/promela/test250.scxml" # no onexit in cancelled invoker - PASSED - # "w3c/spin/promela/test307.scxml" # declare variable via script - FAILED - # "w3c/spin/promela/test313.scxml" # assignment of 'return' - # "w3c/spin/promela/test314.scxml" # assignment of 'return' - # "w3c/spin/promela/test415.scxml" # terminate on toplevel final - PASSED + "w3c/spin/promela/test178.scxml" # two identical params in _event.raw - FAILED + "w3c/spin/promela/test230.scxml" # autoforwarded events are identical - PASSED + "w3c/spin/promela/test250.scxml" # no onexit in cancelled invoker - PASSED + "w3c/spin/promela/test307.scxml" # declare variable via script - FAILED + "w3c/spin/promela/test313.scxml" # assignment of 'return' + "w3c/spin/promela/test314.scxml" # assignment of 'return' + "w3c/spin/promela/test415.scxml" # terminate on toplevel final - PASSED # "w3c/spin/promela/test513.txt" # manual test - FAILED - # "w3c/spin/promela/test301.scxml" # reject invalid script - PASSED - # "w3c/spin/promela/test436.scxml" # In(s) -> _x.states[s] prevents completion as NULL dm is hardcoded + "w3c/spin/promela/test301.scxml" # reject invalid script - PASSED + "w3c/spin/promela/test436.scxml" # In(s) -> _x.states[s] prevents completion as NULL dm is hardcoded # fail for syntax - # "w3c/spin/promela/test152.scxml" # test that an illegal array or item value causes error.execution - # "w3c/spin/promela/test156.scxml" # test that an error causes the foreach to stop execution - # "w3c/spin/promela/test224.scxml" # string operation startWith - # "w3c/spin/promela/test277.scxml" # platform creates unbound variable if we assign an illegal value to it - # "w3c/spin/promela/test280.scxml" # late data binding / undeclared variable - # "w3c/spin/promela/test286.scxml" # assignment to a non-declared var causes an error - # "w3c/spin/promela/test294.scxml" # mixed types for event.data via donedata - # "w3c/spin/promela/test309.scxml" # 'return' as an invalid boolean expression ought to eval to false - # "w3c/spin/promela/test311.scxml" # assignment to a non-declared var - # "w3c/spin/promela/test312.scxml" # assignment of 'return' - # "w3c/spin/promela/test322.scxml" # assignment to _sessionid - # "w3c/spin/promela/test324.scxml" # assignment to _name - # "w3c/spin/promela/test325.scxml" # assignment from _ioprocessor - # "w3c/spin/promela/test326.scxml" # assignment from _ioprocessor - # "w3c/spin/promela/test329.scxml" # test that none of the system variables can be modified - # "w3c/spin/promela/test344.scxml" # 'return' as a cond - # "w3c/spin/promela/test346.scxml" # assignment to system variables - # "w3c/spin/promela/test350.scxml" # string concatenation - # "w3c/spin/promela/test354.scxml" # mixed types for event.data - # "w3c/spin/promela/test401.scxml" # variable not declared - # "w3c/spin/promela/test402.scxml" # variable not declared - # "w3c/spin/promela/test487.scxml" # assignment of 'return' - # "w3c/spin/promela/test509.scxml" # string operation contains - # "w3c/spin/promela/test518.scxml" # string operation contains - # "w3c/spin/promela/test519.scxml" # string operation contains - # "w3c/spin/promela/test520.scxml" # string operation contains - # "w3c/spin/promela/test525.scxml" # assumes unbound arrays - # "w3c/spin/promela/test530.scxml" # assigns DOM node to variable - # "w3c/spin/promela/test534.scxml" # string operation contains + "w3c/spin/promela/test152.scxml" # test that an illegal array or item value causes error.execution + "w3c/spin/promela/test156.scxml" # test that an error causes the foreach to stop execution + "w3c/spin/promela/test224.scxml" # string operation startWith + "w3c/spin/promela/test277.scxml" # platform creates unbound variable if we assign an illegal value to it + "w3c/spin/promela/test280.scxml" # late data binding / undeclared variable + "w3c/spin/promela/test286.scxml" # assignment to a non-declared var causes an error + "w3c/spin/promela/test294.scxml" # mixed types for event.data via donedata + "w3c/spin/promela/test309.scxml" # 'return' as an invalid boolean expression ought to eval to false + "w3c/spin/promela/test311.scxml" # assignment to a non-declared var + "w3c/spin/promela/test312.scxml" # assignment of 'return' + "w3c/spin/promela/test322.scxml" # assignment to _sessionid + "w3c/spin/promela/test324.scxml" # assignment to _name + "w3c/spin/promela/test325.scxml" # assignment from _ioprocessor + "w3c/spin/promela/test326.scxml" # assignment from _ioprocessor + "w3c/spin/promela/test329.scxml" # test that none of the system variables can be modified + "w3c/spin/promela/test344.scxml" # 'return' as a cond + "w3c/spin/promela/test346.scxml" # assignment to system variables + "w3c/spin/promela/test350.scxml" # string concatenation + "w3c/spin/promela/test354.scxml" # mixed types for event.data + "w3c/spin/promela/test401.scxml" # variable not declared + "w3c/spin/promela/test402.scxml" # variable not declared + "w3c/spin/promela/test487.scxml" # assignment of 'return' + "w3c/spin/promela/test509.scxml" # string operation contains + "w3c/spin/promela/test518.scxml" # string operation contains + "w3c/spin/promela/test519.scxml" # string operation contains + "w3c/spin/promela/test520.scxml" # string operation contains + "w3c/spin/promela/test525.scxml" # assumes unbound arrays + "w3c/spin/promela/test530.scxml" # assigns DOM node to variable + "w3c/spin/promela/test534.scxml" # string operation contains # fail for semantics - # "w3c/spin/promela/test159.scxml" # error raised causes all subsequent elements to be skipped - # "w3c/spin/promela/test194.scxml" # illegal target for send - # "w3c/spin/promela/test199.scxml" # invalid send type - # "w3c/spin/promela/test216.scxml" # nested SCXML document with srcexpr at invoke - # "w3c/spin/promela/test298.scxml" # non-existent data model location - # "w3c/spin/promela/test331.scxml" # tests _error.type via 'error.execution' - # "w3c/spin/promela/test332.scxml" # tests _error.sendid via 'error.execution' - # "w3c/spin/promela/test343.scxml" # test that illegal <param> produces error.execution - # "w3c/spin/promela/test488.scxml" # illegal expr in <param> produces error.execution - # "w3c/spin/promela/test496.scxml" # tests error.communication with illegal target - # "w3c/spin/promela/test521.scxml" # tests error.communication with illegal target - # "w3c/spin/promela/test528.scxml" # illegal 'expr' produces error.execution - # "w3c/spin/promela/test531.scxml" # uses _ioprocessors.basichttp.location - # "w3c/spin/promela/test532.scxml" # uses _ioprocessors.basichttp.location - # "w3c/spin/promela/test553.scxml" # error in namelist - # "w3c/spin/promela/test554.scxml" # evaluation of <invoke>'s args causes an error - # "w3c/spin/promela/test577.scxml" # send without target for basichttp + "w3c/spin/promela/test159.scxml" # error raised causes all subsequent elements to be skipped + "w3c/spin/promela/test194.scxml" # illegal target for send + "w3c/spin/promela/test199.scxml" # invalid send type + "w3c/spin/promela/test216.scxml" # nested SCXML document with srcexpr at invoke + "w3c/spin/promela/test298.scxml" # non-existent data model location + "w3c/spin/promela/test331.scxml" # tests _error.type via 'error.execution' + "w3c/spin/promela/test332.scxml" # tests _error.sendid via 'error.execution' + "w3c/spin/promela/test343.scxml" # test that illegal <param> produces error.execution + "w3c/spin/promela/test488.scxml" # illegal expr in <param> produces error.execution + "w3c/spin/promela/test496.scxml" # tests error.communication with illegal target + "w3c/spin/promela/test521.scxml" # tests error.communication with illegal target + "w3c/spin/promela/test528.scxml" # illegal 'expr' produces error.execution + "w3c/spin/promela/test531.scxml" # uses _ioprocessors.basichttp.location + "w3c/spin/promela/test532.scxml" # uses _ioprocessors.basichttp.location + "w3c/spin/promela/test553.scxml" # error in namelist + "w3c/spin/promela/test554.scxml" # evaluation of <invoke>'s args causes an error + "w3c/spin/promela/test577.scxml" # send without target for basichttp ### Ignore for generated C sources diff --git a/test/ctest/scripts/run_promela_test.cmake b/test/ctest/scripts/run_promela_test.cmake index 271434f..4acf628 100644 --- a/test/ctest/scripts/run_promela_test.cmake +++ b/test/ctest/scripts/run_promela_test.cmake @@ -20,8 +20,8 @@ if(CMD_RESULT) endif() message(STATUS "time for transforming to c") -message(STATUS "${CC_BIN} -DMEMLIM=1024 -DVECTORSZ=8192 -O2 -DXUSAFE -w -o ${OUTDIR}/pan ${OUTDIR}/pan.c") -execute_process(COMMAND time -p ${CC_BIN} -DMEMLIM=1024 -DVECTORSZ=8192 -O2 -DXUSAFE -w -o ${OUTDIR}/pan ${OUTDIR}/pan.c WORKING_DIRECTORY ${OUTDIR} RESULT_VARIABLE CMD_RESULT) +message(STATUS "${CC_BIN} -DMEMLIM=1024 -DVECTORSZ=2048 -O2 -DXUSAFE -w -o ${OUTDIR}/pan ${OUTDIR}/pan.c") +execute_process(COMMAND time -p ${CC_BIN} -DMEMLIM=1024 -DVECTORSZ=2048 -O2 -DXUSAFE -w -o ${OUTDIR}/pan ${OUTDIR}/pan.c WORKING_DIRECTORY ${OUTDIR} RESULT_VARIABLE CMD_RESULT) if(CMD_RESULT) message(FATAL_ERROR "Error running gcc ${CC_BIN}: ${CMD_RESULT}") endif() diff --git a/test/w3c/analyze_tests.pl b/test/w3c/analyze_tests.pl index 7a6ed08..bdb9dd8 100755 --- a/test/w3c/analyze_tests.pl +++ b/test/w3c/analyze_tests.pl @@ -1,5 +1,11 @@ #!/usr/bin/perl -w +#./analyze_tests.pl \ +# w3c pml.topml pml.toc pml.tobin pml.verify \ +#|awk '{printf("%f\n", ($2 + $3 + $4)); }' \ +#|sort -rn \ +#|gnuplot -e "set term png;p '-' u 0:1" > test.png + use strict; use File::Spec; use File::Basename; @@ -148,6 +154,101 @@ while ($block = <FILE>) { } } + # New Promela Tests ======== + if ($block =~ + / + uscxml-transform[^\n]+tpml + /x ) { + + if ($block =~ + / + \nreal\s+(\d+.?\d+) + \nuser\s+(\d+.?\d+) + \nsys\s+(\d+.?\d+) + \n--\stime\sfor\stransforming\sto\spromela + /x ) { + $test->{$currTest}->{'pml'}->{'toPML'} = $1; + } + + if ($block =~ + / + \nreal\s+(\d+.?\d+) + \nuser\s+(\d+.?\d+) + \nsys\s+(\d+.?\d+) + \n--\stime\sfor\stransforming\sto\sc + /x ) { + $test->{$currTest}->{'pml'}->{'toC'} = $1; + } + + if ($block =~ + / + \nreal\s+(\d+.?\d+) + \nuser\s+(\d+.?\d+) + \nsys\s+(\d+.?\d+) + \n--\stime\sfor\stransforming\sto\sbin + /x ) { + $test->{$currTest}->{'pml'}->{'toBIN'} = $1; + } + + if ($block =~ + / + \nreal\s+(\d+.?\d+) + \nuser\s+(\d+.?\d+) + \nsys\s+(\d+.?\d+) + \n--\stime\sfor\sverification + /x ) { + $test->{$currTest}->{'pml'}->{'verify'} = $1; + } + + if ($block =~ /State-vector (\d+) byte, depth reached (\d+), errors: (\d+)/) { + $test->{$currTest}->{'pml'}->{'states'}->{'stateSize'} = $1; + $test->{$currTest}->{'pml'}->{'states'}->{'depth'} = $2; + $test->{$currTest}->{'pml'}->{'states'}->{'errors'} = $3; + } + if ($block =~ + / + \s+(\d+)\sstates,\sstored\s\((\d+)\svisited\)\n + \s+(\d+)\sstates,\smatched\n + \s+(\d+)\stransitions\s\(=\svisited\+matched\)\n + \s+(\d+)\satomic\ssteps\n + hash\sconflicts:\s+(\d+)\s\(resolved\) + /x ) { + $test->{$currTest}->{'pml'}->{'states'}->{'stateStored'} = $1; + $test->{$currTest}->{'pml'}->{'states'}->{'stateVisited'} = $2; + $test->{$currTest}->{'pml'}->{'states'}->{'stateMatched'} = $3; + $test->{$currTest}->{'pml'}->{'states'}->{'transitions'} = $4; + $test->{$currTest}->{'pml'}->{'states'}->{'atomicSteps'} = $5; + $test->{$currTest}->{'pml'}->{'hashConflicts'} = $6; + } + + if ($block =~ + / + \s+([\d\.]+)\sequivalent\smemory\susage\sfor\sstates.* + \s+([\d\.]+)\sactual\smemory\susage\sfor\sstates\n + \s+([\d\.]+)\smemory\sused\sfor\shash\stable\s\(-w(\d+)\)\n + \s+([\d\.]+)\smemory\sused\sfor\sDFS\sstack\s\(-m(\d+)\) + \s+([\d\.]+)\stotal\sactual\smemory\susage + /x ) { + $test->{$currTest}->{'pml'}->{'memory'}->{'states'} = $1; + $test->{$currTest}->{'pml'}->{'memory'}->{'actual'} = $2; + $test->{$currTest}->{'pml'}->{'memory'}->{'hashTable'} = $3; + $test->{$currTest}->{'pml'}->{'memory'}->{'hashLimit'} = $4; + $test->{$currTest}->{'pml'}->{'memory'}->{'dfsStack'} = $5; + $test->{$currTest}->{'pml'}->{'memory'}->{'dfsLimit'} = $6; + $test->{$currTest}->{'pml'}->{'memory'}->{'total'} = $7; + } + + if ($block =~ + / + pan:\selapsed\stime\s(.*)\sseconds\n + /x ) { + $test->{$currTest}->{'pml'}->{'duration'} = $1; + } + + } + + + # Promela Test ======== if ($block =~ / |