diff options
Diffstat (limited to 'test/runtest')
| -rw-r--r-- | test/runtest/aegis/batch-output.py | 60 | ||||
| -rw-r--r-- | test/runtest/baseline/combined.py | 28 | ||||
| -rw-r--r-- | test/runtest/baseline/fail.py | 4 | ||||
| -rw-r--r-- | test/runtest/baseline/no_result.py | 4 | ||||
| -rw-r--r-- | test/runtest/baseline/pass.py | 4 | ||||
| -rw-r--r-- | test/runtest/fallback.py | 104 | ||||
| -rw-r--r-- | test/runtest/noqmtest.py | 94 | ||||
| -rw-r--r-- | test/runtest/print_time.py | 29 | ||||
| -rw-r--r-- | test/runtest/python.py | 42 | ||||
| -rw-r--r-- | test/runtest/simple/combined.py | 28 | ||||
| -rw-r--r-- | test/runtest/simple/fail.py | 4 | ||||
| -rw-r--r-- | test/runtest/simple/no_result.py | 4 | ||||
| -rw-r--r-- | test/runtest/simple/pass.py | 4 | ||||
| -rw-r--r-- | test/runtest/src.py | 21 | ||||
| -rw-r--r-- | test/runtest/testlistfile.py | 77 | ||||
| -rw-r--r-- | test/runtest/xml/output.py | 191 |
16 files changed, 624 insertions, 74 deletions
diff --git a/test/runtest/aegis/batch-output.py b/test/runtest/aegis/batch-output.py new file mode 100644 index 0000000..4020239 --- /dev/null +++ b/test/runtest/aegis/batch-output.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test writing Aegis batch output to a file. +""" + +import TestRuntest + +test = TestRuntest.TestRuntest() + +test.subdir('test') + +test.write_failing_test(['test', 'fail.py']) + +test.write_no_result_test(['test', 'no_result.py']) + +test.write_passing_test(['test', 'pass.py']) + +test.run(arguments = '-o aegis.out --aegis test', status=1) + +expect = """\ +test_result = [ + { file_name = "test/fail.py"; + exit_status = 1; }, + { file_name = "test/no_result.py"; + exit_status = 2; }, + { file_name = "test/pass.py"; + exit_status = 0; }, +]; +""" + +# The mode is 'r' (not default 'rb') because QMTest opens the file +# description on which we write as non-binary. +test.must_match('aegis.out', expect, mode='r') + +test.pass_test() diff --git a/test/runtest/baseline/combined.py b/test/runtest/baseline/combined.py index 6382cb1..ab91e87 100644 --- a/test/runtest/baseline/combined.py +++ b/test/runtest/baseline/combined.py @@ -30,8 +30,14 @@ Test a combination of a passing test, failing test, and no-result test with no argument on the command line. """ +import os.path + import TestRuntest +test_fail_py = os.path.join('test', 'fail.py') +test_no_result_py = os.path.join('test', 'no_result.py') +test_pass_py = os.path.join('test', 'pass.py') + test = TestRuntest.TestRuntest() test.subdir('test') @@ -45,37 +51,37 @@ test.write_passing_test(['test', 'pass.py']) # NOTE: The "test/fail.py : FAIL" and "test/pass.py : PASS" lines both # have spaces at the end. -expect = r"""qmtest.py run --output baseline.qmr --format none --result-stream=scons_tdb.AegisBaselineStream test +expect = r"""qmtest.py run --output baseline.qmr --format none --result-stream="scons_tdb.AegisBaselineStream" test --- TEST RESULTS ------------------------------------------------------------- - test/fail.py : FAIL + %(test_fail_py)s : FAIL FAILING TEST STDOUT FAILING TEST STDERR - test/no_result.py : NO_RESULT + %(test_no_result_py)s : NO_RESULT NO RESULT TEST STDOUT NO RESULT TEST STDERR - test/pass.py : PASS + %(test_pass_py)s : PASS --- TESTS WITH UNEXPECTED OUTCOMES ------------------------------------------- - test/no_result.py : NO_RESULT + %(test_no_result_py)s : NO_RESULT - test/pass.py : PASS + %(test_pass_py)s : PASS --- STATISTICS --------------------------------------------------------------- - 1 ( 33%) tests as expected - 1 ( 33%) tests unexpected PASS - 1 ( 33%) tests unexpected NO_RESULT -""" + 1 ( 33%%) tests as expected + 1 ( 33%%) tests unexpected PASS + 1 ( 33%%) tests unexpected NO_RESULT +""" % locals() -test.run(arguments = '--qmtest -b . test', stdout = expect) +test.run(arguments = '-b . test', status = 1, stdout = expect) test.pass_test() diff --git a/test/runtest/baseline/fail.py b/test/runtest/baseline/fail.py index b650a8c..b61e5da 100644 --- a/test/runtest/baseline/fail.py +++ b/test/runtest/baseline/fail.py @@ -38,7 +38,7 @@ test.write_failing_test(['test', 'fail.py']) # NOTE: The "test/fail.py : FAIL" line has spaces at the end. -expect = r"""qmtest.py run --output baseline.qmr --format none --result-stream=scons_tdb.AegisBaselineStream test/fail.py +expect = r"""qmtest.py run --output baseline.qmr --format none --result-stream="scons_tdb.AegisBaselineStream" test/fail.py --- TEST RESULTS ------------------------------------------------------------- test/fail.py : FAIL @@ -57,6 +57,6 @@ expect = r"""qmtest.py run --output baseline.qmr --format none --result-stream=s 1 (100%) tests as expected """ -test.run(arguments = '--qmtest -b . test/fail.py', stdout = expect) +test.run(arguments = '-b . test/fail.py', status = 1, stdout = expect) test.pass_test() diff --git a/test/runtest/baseline/no_result.py b/test/runtest/baseline/no_result.py index dc2586d..9ef815d 100644 --- a/test/runtest/baseline/no_result.py +++ b/test/runtest/baseline/no_result.py @@ -36,7 +36,7 @@ test.subdir('test') test.write_no_result_test(['test', 'no_result.py']) -expect = r"""qmtest.py run --output baseline.qmr --format none --result-stream=scons_tdb.AegisBaselineStream test/no_result.py +expect = r"""qmtest.py run --output baseline.qmr --format none --result-stream="scons_tdb.AegisBaselineStream" test/no_result.py --- TEST RESULTS ------------------------------------------------------------- test/no_result.py : NO_RESULT @@ -55,6 +55,6 @@ expect = r"""qmtest.py run --output baseline.qmr --format none --result-stream=s 1 (100%) tests unexpected NO_RESULT """ -test.run(arguments = '--qmtest -b . test/no_result.py', stdout = expect) +test.run(arguments = '-b . test/no_result.py', status = 1, stdout = expect) test.pass_test() diff --git a/test/runtest/baseline/pass.py b/test/runtest/baseline/pass.py index b32ecbe..f574e57 100644 --- a/test/runtest/baseline/pass.py +++ b/test/runtest/baseline/pass.py @@ -38,7 +38,7 @@ test.write_passing_test(['test', 'pass.py']) # NOTE: The "test/pass.py : PASS" line has spaces at the end. -expect = r"""qmtest.py run --output baseline.qmr --format none --result-stream=scons_tdb.AegisBaselineStream test/pass.py +expect = r"""qmtest.py run --output baseline.qmr --format none --result-stream="scons_tdb.AegisBaselineStream" test/pass.py --- TEST RESULTS ------------------------------------------------------------- test/pass.py : PASS @@ -53,6 +53,6 @@ expect = r"""qmtest.py run --output baseline.qmr --format none --result-stream=s 1 (100%) tests unexpected PASS """ -test.run(arguments = '--qmtest -b . test/pass.py', stdout = expect) +test.run(arguments = '-b . test/pass.py', stdout = expect) test.pass_test() diff --git a/test/runtest/fallback.py b/test/runtest/fallback.py new file mode 100644 index 0000000..90cd4f2 --- /dev/null +++ b/test/runtest/fallback.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test that runtest.py falls back (with a warning) using --noqmtest +if it can't find qmtest.py on the $PATH. +""" + +import os +import os.path +import re +import string + +import TestRuntest + +python = TestRuntest.python +_python_ = TestRuntest._python_ + +test = TestRuntest.TestRuntest(noqmtest=1) + +qmtest_py = test.where_is('qmtest.py') + +if qmtest_py: + dir = os.path.split(qmtest_py)[0] + path = string.split(os.environ['PATH'], os.pathsep) + path.remove(dir) + os.environ['PATH'] = string.join(path, os.pathsep) + +test.subdir('test') + +test_pass_py = os.path.join('test', 'pass.py') +test_fail_py = os.path.join('test', 'fail.py') +test_no_result_py = os.path.join('test', 'no_result.py') + +workpath_pass_py = test.workpath(test_pass_py) +workpath_fail_py = test.workpath(test_fail_py) +workpath_no_result_py = test.workpath(test_no_result_py) + +test.write_failing_test(test_fail_py) +test.write_no_result_test(test_no_result_py) +test.write_passing_test(test_pass_py) + +if re.search('\s', python): + expect_python = _python_ +else: + expect_python = python + +expect_stdout = """\ +%(expect_python)s -tt %(workpath_fail_py)s +FAILING TEST STDOUT +%(expect_python)s -tt %(workpath_no_result_py)s +NO RESULT TEST STDOUT +%(expect_python)s -tt %(workpath_pass_py)s +PASSING TEST STDOUT + +Failed the following test: +\t%(test_fail_py)s + +NO RESULT from the following test: +\t%(test_no_result_py)s +""" % locals() + +expect_stderr = """\ +Warning: qmtest.py not found on $PATH, assuming --noqmtest option. +FAILING TEST STDERR +NO RESULT TEST STDERR +PASSING TEST STDERR +""" + +testlist = [ + test_fail_py, + test_no_result_py, + test_pass_py, +] + +test.run(arguments = string.join(testlist), + status = 1, + stdout = expect_stdout, + stderr = expect_stderr) + +test.pass_test() diff --git a/test/runtest/noqmtest.py b/test/runtest/noqmtest.py new file mode 100644 index 0000000..c442125 --- /dev/null +++ b/test/runtest/noqmtest.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test that the --noqmtest option invokes tests directly via Python, not +using qmtest. +""" + +import os.path +import re +import string + +import TestRuntest + +python = TestRuntest.python +_python_ = TestRuntest._python_ + +test = TestRuntest.TestRuntest(noqmtest=1) + +test.subdir('test') + +test_pass_py = os.path.join('test', 'pass.py') +test_fail_py = os.path.join('test', 'fail.py') +test_no_result_py = os.path.join('test', 'no_result.py') + +workpath_pass_py = test.workpath(test_pass_py) +workpath_fail_py = test.workpath(test_fail_py) +workpath_no_result_py = test.workpath(test_no_result_py) + +test.write_failing_test(test_fail_py) +test.write_no_result_test(test_no_result_py) +test.write_passing_test(test_pass_py) + +if re.search('\s', python): + expect_python = _python_ +else: + expect_python = python + +expect_stdout = """\ +%(expect_python)s -tt %(workpath_fail_py)s +FAILING TEST STDOUT +%(expect_python)s -tt %(workpath_no_result_py)s +NO RESULT TEST STDOUT +%(expect_python)s -tt %(workpath_pass_py)s +PASSING TEST STDOUT + +Failed the following test: +\t%(test_fail_py)s + +NO RESULT from the following test: +\t%(test_no_result_py)s +""" % locals() + +expect_stderr = """\ +FAILING TEST STDERR +NO RESULT TEST STDERR +PASSING TEST STDERR +""" + +testlist = [ + test_fail_py, + test_no_result_py, + test_pass_py, +] + +test.run(arguments = '--noqmtest %s' % string.join(testlist), + status = 1, + stdout = expect_stdout, + stderr = expect_stderr) + +test.pass_test() diff --git a/test/runtest/print_time.py b/test/runtest/print_time.py index 1d86baa..39bf810 100644 --- a/test/runtest/print_time.py +++ b/test/runtest/print_time.py @@ -29,9 +29,16 @@ Test a combination of a passing test, failing test, and no-result test with no argument on the command line. """ +import os.path +import re + import TestCmd import TestRuntest +test_fail_py = re.escape(os.path.join('test', 'fail.py')) +test_no_result_py = re.escape(os.path.join('test', 'no_result.py')) +test_pass_py = re.escape(os.path.join('test', 'pass.py')) + test = TestRuntest.TestRuntest(match = TestCmd.match_re) test.subdir('test') @@ -45,10 +52,10 @@ test.write_passing_test(['test', 'pass.py']) # NOTE: The "test/fail.py : FAIL" and "test/pass.py : PASS" lines both # have spaces at the end. -expect = r"""qmtest.py run --output results.qmr --format none --result-stream=scons_tdb.AegisChangeStream --context print_time=1 test +expect = r"""qmtest.py run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream\(print_time='1'\)" test --- TEST RESULTS ------------------------------------------------------------- - test/fail.py : FAIL + %(test_fail_py)s : FAIL FAILING TEST STDOUT @@ -56,7 +63,7 @@ expect = r"""qmtest.py run --output results.qmr --format none --result-stream=sc Total execution time: \d+\.\d+ seconds - test/no_result.py : NO_RESULT + %(test_no_result_py)s : NO_RESULT NO RESULT TEST STDOUT @@ -64,26 +71,26 @@ expect = r"""qmtest.py run --output results.qmr --format none --result-stream=sc Total execution time: \d+\.\d+ seconds - test/pass.py : PASS + %(test_pass_py)s : PASS Total execution time: \d+\.\d+ seconds --- TESTS THAT DID NOT PASS -------------------------------------------------- - test/fail.py : FAIL + %(test_fail_py)s : FAIL - test/no_result.py : NO_RESULT + %(test_no_result_py)s : NO_RESULT --- STATISTICS --------------------------------------------------------------- 3 tests total - 1 \( 33%\) tests PASS - 1 \( 33%\) tests FAIL - 1 \( 33%\) tests NO_RESULT -""" + 1 \( 33%%\) tests PASS + 1 \( 33%%\) tests FAIL + 1 \( 33%%\) tests NO_RESULT +""" % locals() -test.run(arguments = '--qmtest -t test', stdout = expect) +test.run(arguments = '-t test', status = 1, stdout = expect) test.pass_test() diff --git a/test/runtest/python.py b/test/runtest/python.py index 76eec7d..1af32dd 100644 --- a/test/runtest/python.py +++ b/test/runtest/python.py @@ -28,35 +28,40 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" Test that the -P option lets us specify a Python version to use. """ +import os.path +import re +import sys + +if not hasattr(os.path, 'pardir'): + os.path.pardir = '..' + import TestRuntest test = TestRuntest.TestRuntest() -mypython_py = test.workpath('mypython.py') -mypython_out = test.workpath('mypython.out') +test_pass_py = os.path.join('test', 'pass.py') -test.subdir('test') +head, python = os.path.split(TestRuntest.python) +head, dir = os.path.split(head) -test.write_passing_test(['test', 'pass.py']) +mypython = os.path.join(head, dir, os.path.pardir, dir, python) -test.write(mypython_py, """\ -#!/usr/bin/env python -import os -import sys -import string -open(r'%s', 'a').write(string.join(sys.argv) + '\\n') -os.system(string.join([sys.executable] + sys.argv[1:])) -""" % mypython_out) +if re.search('\s', mypython): + _mypython_ = '"' + mypython + '"' +else: + _mypython_ = mypython -test.chmod(mypython_py, 0755) +test.subdir('test') + +test.write_passing_test(['test', 'pass.py']) # NOTE: The "test/fail.py : FAIL" and "test/pass.py : PASS" lines both # have spaces at the end. -expect = r"""qmtest.py run --output results.qmr --format none --result-stream=scons_tdb.AegisChangeStream --context python=%(mypython_py)s test +expect = r"""qmtest.py run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" --context python="%(mypython)s" test --- TEST RESULTS ------------------------------------------------------------- - test/pass.py : PASS + %(test_pass_py)s : PASS --- TESTS THAT DID NOT PASS -------------------------------------------------- @@ -70,11 +75,6 @@ expect = r"""qmtest.py run --output results.qmr --format none --result-stream=sc 1 (100%%) tests PASS """ % locals() -test.run(arguments = '--qmtest -P %s test' % mypython_py, - stdout = expect) - -test.must_match(mypython_out, """\ -%s ./test/pass.py -""" % mypython_py) +test.run(arguments = '-P %s test' % _mypython_, stdout = expect) test.pass_test() diff --git a/test/runtest/simple/combined.py b/test/runtest/simple/combined.py index 1640d94..58d2f27 100644 --- a/test/runtest/simple/combined.py +++ b/test/runtest/simple/combined.py @@ -30,10 +30,16 @@ Test a combination of a passing test, failing test, and no-result test with no argument on the command line. """ +import os.path + import TestRuntest test = TestRuntest.TestRuntest() +test_fail_py = os.path.join('test', 'fail.py') +test_no_result_py = os.path.join('test', 'no_result.py') +test_pass_py = os.path.join('test', 'pass.py') + test.subdir('test') test.write_failing_test(['test', 'fail.py']) @@ -45,39 +51,39 @@ test.write_passing_test(['test', 'pass.py']) # NOTE: The "test/fail.py : FAIL" and "test/pass.py : PASS" lines both # have spaces at the end. -expect = r"""qmtest.py run --output results.qmr --format none --result-stream=scons_tdb.AegisChangeStream test +expect = r"""qmtest.py run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" test --- TEST RESULTS ------------------------------------------------------------- - test/fail.py : FAIL + %(test_fail_py)s : FAIL FAILING TEST STDOUT FAILING TEST STDERR - test/no_result.py : NO_RESULT + %(test_no_result_py)s : NO_RESULT NO RESULT TEST STDOUT NO RESULT TEST STDERR - test/pass.py : PASS + %(test_pass_py)s : PASS --- TESTS THAT DID NOT PASS -------------------------------------------------- - test/fail.py : FAIL + %(test_fail_py)s : FAIL - test/no_result.py : NO_RESULT + %(test_no_result_py)s : NO_RESULT --- STATISTICS --------------------------------------------------------------- 3 tests total - 1 ( 33%) tests PASS - 1 ( 33%) tests FAIL - 1 ( 33%) tests NO_RESULT -""" + 1 ( 33%%) tests PASS + 1 ( 33%%) tests FAIL + 1 ( 33%%) tests NO_RESULT +""" % locals() -test.run(arguments = '--qmtest test', stdout = expect) +test.run(arguments = 'test', status = 1, stdout = expect) test.pass_test() diff --git a/test/runtest/simple/fail.py b/test/runtest/simple/fail.py index ba2cc97..ec9f532 100644 --- a/test/runtest/simple/fail.py +++ b/test/runtest/simple/fail.py @@ -38,7 +38,7 @@ test.write_failing_test(['test', 'fail.py']) # NOTE: The "test/fail.py : FAIL" line has spaces at the end. -expect = r"""qmtest.py run --output results.qmr --format none --result-stream=scons_tdb.AegisChangeStream test/fail.py +expect = r"""qmtest.py run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" test/fail.py --- TEST RESULTS ------------------------------------------------------------- test/fail.py : FAIL @@ -59,6 +59,6 @@ expect = r"""qmtest.py run --output results.qmr --format none --result-stream=sc 1 (100%) tests FAIL """ -test.run(arguments = '--qmtest test/fail.py', stdout = expect) +test.run(arguments = 'test/fail.py', status = 1, stdout = expect) test.pass_test() diff --git a/test/runtest/simple/no_result.py b/test/runtest/simple/no_result.py index 7c9687e..4ec6e78 100644 --- a/test/runtest/simple/no_result.py +++ b/test/runtest/simple/no_result.py @@ -36,7 +36,7 @@ test.subdir('test') test.write_no_result_test(['test', 'no_result.py']) -expect = r"""qmtest.py run --output results.qmr --format none --result-stream=scons_tdb.AegisChangeStream test/no_result.py +expect = r"""qmtest.py run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" test/no_result.py --- TEST RESULTS ------------------------------------------------------------- test/no_result.py : NO_RESULT @@ -57,6 +57,6 @@ expect = r"""qmtest.py run --output results.qmr --format none --result-stream=sc 1 (100%) tests NO_RESULT """ -test.run(arguments = '--qmtest test/no_result.py', stdout = expect) +test.run(arguments = 'test/no_result.py', status = 1, stdout = expect) test.pass_test() diff --git a/test/runtest/simple/pass.py b/test/runtest/simple/pass.py index 8dfc996..c3a8b02 100644 --- a/test/runtest/simple/pass.py +++ b/test/runtest/simple/pass.py @@ -38,7 +38,7 @@ test.write_passing_test(['test', 'pass.py']) # NOTE: The "test/pass.py : PASS" line has spaces at the end. -expect = r"""qmtest.py run --output results.qmr --format none --result-stream=scons_tdb.AegisChangeStream test/pass.py +expect = r"""qmtest.py run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" test/pass.py --- TEST RESULTS ------------------------------------------------------------- test/pass.py : PASS @@ -55,6 +55,6 @@ expect = r"""qmtest.py run --output results.qmr --format none --result-stream=sc 1 (100%) tests PASS """ -test.run(arguments = '--qmtest test/pass.py', stdout = expect) +test.run(arguments = 'test/pass.py', stdout = expect) test.pass_test() diff --git a/test/runtest/src.py b/test/runtest/src.py index 2f723b4..3063a4e 100644 --- a/test/runtest/src.py +++ b/test/runtest/src.py @@ -29,6 +29,8 @@ Verify that we find tests under the src/ tree only if they end with *Tests.py. """ +import os.path + import TestRuntest test = TestRuntest.TestRuntest(verbose=1) @@ -36,6 +38,9 @@ test = TestRuntest.TestRuntest(verbose=1) test.subdir(['src'], ['src', 'suite']) +src_passTests_py = os.path.join('src', 'passTests.py') +src_suite_passTests_py = os.path.join('src', 'suite', 'passTests.py') + test.write_passing_test(['src', 'pass.py']) test.write_passing_test(['src', 'passTests.py']) @@ -44,15 +49,15 @@ test.write_passing_test(['src', 'suite', 'pass.py']) test.write_passing_test(['src', 'suite', 'passTests.py']) -# NOTE: The "test/fail.py : FAIL" and "test/pass.py : PASS" lines both -# have spaces at the end. +# NOTE: The "test/pass.py : PASS" and "test/passTests.py : PASS" lines +# both have spaces at the end. -expect = r"""qmtest.py run --output results.qmr --format none --result-stream=scons_tdb.AegisChangeStream src +expect = r"""qmtest.py run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" src --- TEST RESULTS ------------------------------------------------------------- - src/passTests.py : PASS + %(src_passTests_py)s : PASS - src/suite/passTests.py : PASS + %(src_suite_passTests_py)s : PASS --- TESTS THAT DID NOT PASS -------------------------------------------------- @@ -63,9 +68,9 @@ expect = r"""qmtest.py run --output results.qmr --format none --result-stream=sc 2 tests total - 2 (100%) tests PASS -""" + 2 (100%%) tests PASS +""" % locals() -test.run(arguments = '--qmtest src', stdout = expect) +test.run(arguments = 'src', stdout = expect) test.pass_test() diff --git a/test/runtest/testlistfile.py b/test/runtest/testlistfile.py new file mode 100644 index 0000000..d738530 --- /dev/null +++ b/test/runtest/testlistfile.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test a list of tests to run in a file specified with the -f option. +""" + +import os.path + +import TestCmd +import TestRuntest + +test_fail_py = os.path.join('test', 'fail.py') +test_no_result_py = os.path.join('test', 'no_result.py') +test_pass_py = os.path.join('test', 'pass.py') + +test = TestRuntest.TestRuntest() + +test.subdir('test') + +test.write_failing_test(['test', 'fail.py']) + +test.write_no_result_test(['test', 'no_result.py']) + +test.write_passing_test(['test', 'pass.py']) + +test.write('t.txt', """\ +#%(test_fail_py)s +%(test_pass_py)s +""" % locals()) + +# NOTE: The "test/fail.py : FAIL" and "test/pass.py : PASS" lines both +# have spaces at the end. + +expect = """qmtest.py run --output results.qmr --format none --result-stream="scons_tdb.AegisChangeStream" %(test_pass_py)s +--- TEST RESULTS ------------------------------------------------------------- + + %(test_pass_py)s : PASS + +--- TESTS THAT DID NOT PASS -------------------------------------------------- + + None. + + +--- STATISTICS --------------------------------------------------------------- + + 1 tests total + + 1 (100%%) tests PASS +""" % locals() + +test.run(arguments = '-f t.txt', stdout = expect) + +test.pass_test() diff --git a/test/runtest/xml/output.py b/test/runtest/xml/output.py new file mode 100644 index 0000000..e1c14cf --- /dev/null +++ b/test/runtest/xml/output.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test writing XML output to a file. +""" + +import os.path +import re +import sys + +import TestRuntest + +test = TestRuntest.TestRuntest() + +test_fail_py = re.escape(os.path.join('test', 'fail.py')) +test_no_result_py = re.escape(os.path.join('test', 'no_result.py')) +test_pass_py = re.escape(os.path.join('test', 'pass.py')) + +# sys.stdout and sys.stderr are open non-binary ('w' instead of 'wb') +# so the lines written on Windows are terminated \r\n, not just \n. The +# expressions below use 'cr' as the optional carriage return character. +if sys.platform in ['win32']: + cr = '\r' +else: + cr = '' + +test.subdir('test') + +test.write_fake_scons_source_tree() + +test.write_failing_test(['test', 'fail.py']) + +test.write_no_result_test(['test', 'no_result.py']) + +test.write_passing_test(['test', 'pass.py']) + +test.run(arguments = '-o xml.out --xml test', status = 1) + +expect_engine = """\ +<annotation key="scons_test\\.engine"> + __build__='D456' + __buildsys__='another_fake_system' + __date__='Dec 31 1999' + __developer__='John Doe' + __version__='4\\.5\\.6' +</annotation> +""" + +expect_script = """\ +<annotation key="scons_test\\.script"> + __build__='D123' + __buildsys__='fake_system' + __date__='Jan 1 1970' + __developer__='Anonymous' + __version__='1\\.2\\.3' +</annotation> +""" + +# The actual values printed for sys and os.environ will be completely +# dependent on the local values. Don't bother trying to match, just +# look to see if the opening tag exists. + +expect_sys = """\ +<annotation key="scons_test\\.sys"> +""" + +expect_os_environ = """\ +<annotation key="scons_test\\.os\\.environ"> +""" + +expect_fail = """\ + <result id="%(test_fail_py)s" kind="test" outcome="FAIL"> + <annotation name="Test\\.exit_code"> + "1" + </annotation> + <annotation name="Test\\.stderr"> + "<pre>FAILING TEST STDERR%(cr)s +</pre>" + </annotation> + <annotation name="Test\\.stdout"> + "<pre>FAILING TEST STDOUT%(cr)s +</pre>" + </annotation> + <annotation name="qmtest\\.cause"> + "Non-zero exit_code\\." + </annotation> + <annotation name="qmtest\\.end_time"> + "[\\d.]+" + </annotation> + <annotation name="qmtest\\.start_time"> + "[\\d.]+" + </annotation> + <annotation name="qmtest\\.target"> + "local" + </annotation> + </result> +""" % locals() + +expect_no_result = """\ + <result id="%(test_no_result_py)s" kind="test" outcome="FAIL"> + <annotation name="Test.exit_code"> + "2" + </annotation> + <annotation name="Test\\.stderr"> + "<pre>NO RESULT TEST STDERR%(cr)s +</pre>" + </annotation> + <annotation name="Test\\.stdout"> + "<pre>NO RESULT TEST STDOUT%(cr)s +</pre>" + </annotation> + <annotation name="qmtest\\.cause"> + "Non-zero exit_code\\." + </annotation> + <annotation name="qmtest\\.end_time"> + "[\\d.]+" + </annotation> + <annotation name="qmtest\\.start_time"> + "[\\d.]+" + </annotation> + <annotation name="qmtest\\.target"> + "local" + </annotation> + </result> +""" % locals() + +expect_pass = """\ + <result id="%(test_pass_py)s" kind="test" outcome="PASS"> + <annotation name="qmtest\\.end_time"> + "[\\d.]+" + </annotation> + <annotation name="qmtest\\.start_time"> + "[\\d.]+" + </annotation> + <annotation name="qmtest\\.target"> + "local" + </annotation> + </result> +""" % locals() + +xml_out = test.read('xml.out', 'r') + +expect = [ + expect_engine, + expect_script, + expect_sys, + expect_os_environ, + expect_fail, + expect_no_result, + expect_pass, +] + +non_matches = [] + +for e in expect: + if not re.search(e, xml_out): + non_matches.append(e) + +if non_matches: + for n in non_matches: + print "DID NOT MATCH " + '='*60 + print n + print "ACTUAL XML OUTPUT " + '='*60 + print xml_out + test.fail_test() + +test.pass_test() |