From 6084c53b521cb14bce1bdbc5fe5c7c0ee6ae158e Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 7 Nov 2020 11:06:29 -0700 Subject: runtest now writes a log of fails failed_tests.log is created if there are any fails (on by default). --retry option added to rerun tests from that file, or use -f listfile to use a file of a different name. --faillog=FILE allows saving the fails to a non-default name (in case don't want to overwrite the existing one, perhaps); --no-faillog disables the writing of the log. Two unneeded tests relating to qmtest were dropped: fallback.py didn't really test anything, and noqmtest.py was a duplicate of simple/combined.py after the qmtest specifics had been stripped out. Added tests for the added three options (git thinks two were renames). Signed-off-by: Mats Wichmann --- CHANGES.txt | 5 ++- runtest.py | 42 +++++++++++++----- test/runtest/SCons.py | 7 +-- test/runtest/faillog.py | 81 ++++++++++++++++++++++++++++++++++ test/runtest/fallback.py | 97 ----------------------------------------- test/runtest/no_faillog.py | 83 +++++++++++++++++++++++++++++++++++ test/runtest/noqmtest.py | 89 ------------------------------------- test/runtest/print_time.py | 4 -- test/runtest/python.py | 2 - test/runtest/retry.py | 69 +++++++++++++++++++++++++++++ test/runtest/simple/combined.py | 25 +++++------ test/runtest/testlistfile.py | 4 +- 12 files changed, 283 insertions(+), 225 deletions(-) create mode 100644 test/runtest/faillog.py delete mode 100644 test/runtest/fallback.py create mode 100644 test/runtest/no_faillog.py delete mode 100644 test/runtest/noqmtest.py create mode 100644 test/runtest/retry.py diff --git a/CHANGES.txt b/CHANGES.txt index e58ea8f..87b20c4 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -72,7 +72,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER MethodWrapper moved to Util to avoid a circular import. Fixes #3028. - Some Python 2 compatibility code dropped - Rework runtest.py to use argparse for arg handling (was a mix - of hand-coded and optparse, with a stated intent to "gradually port") + of hand-coded and optparse, with a stated intent to "gradually port"). + - Add options to runtest to generate/not generate a log of failed tests, + and to rerun such tests. Useful when an error cascades through several + tests, can quickly try if a change improves all the fails. From Simon Tegelid - Fix using TEMPFILE in multiple actions in an action list. Previously a builder, or command diff --git a/runtest.py b/runtest.py index 83a67d5..df7c328 100755 --- a/runtest.py +++ b/runtest.py @@ -61,6 +61,8 @@ testlisting.add_argument('-f', '--file', metavar='FILE', dest='testlistfile', help="Select only tests in FILE") testlisting.add_argument('-a', '--all', action='store_true', help="Select all tests") +testlisting.add_argument('--retry', action='store_true', + help="Rerun the last failed tests in 'failed_tests.log'") testsel.add_argument('--exclude-list', metavar="FILE", dest='excludelistfile', help="""Exclude tests in FILE from current selection""") testtype = testsel.add_mutually_exclusive_group() @@ -87,11 +89,8 @@ parser.add_argument('-n', '--no-exec', action='store_false', help="No execute, just print command lines.") parser.add_argument('--nopipefiles', action='store_false', dest='allow_pipe_files', - help="""Do not use the "file pipe" workaround for Popen() - for starting tests. WARNING: use only when too much - file traffic is giving you trouble AND you can be - sure that none of your tests create output >65K - chars! You might run into some deadlocks else.""") + help="""Do not use the "file pipe" workaround for subprocess + for starting tests. See source code for warnings.""") parser.add_argument('-P', '--python', metavar='PYTHON', help="Use the specified Python interpreter.") parser.add_argument('--quit-on-failure', action='store_true', @@ -102,6 +101,14 @@ parser.add_argument('-X', dest='scons_exec', action='store_true', help="Test script is executable, don't feed to Python.") parser.add_argument('-x', '--exec', metavar="SCRIPT", help="Test using SCRIPT as path to SCons.") +parser.add_argument('--faillog', dest='error_log', metavar="FILE", + default='failed_tests.log', + help="Log failed tests to FILE (enabled by default, " + "default file 'failed_tests.log')") +parser.add_argument('--no-faillog', dest='error_log', + action='store_const', const=None, + default='failed_tests.log', + help="Do not log failed tests to a file") outctl = parser.add_argument_group(description='Output control options:') outctl.add_argument('-k', '--no-progress', action='store_false', @@ -123,6 +130,8 @@ outctl.add_argument('--verbose', metavar='LEVEL', type=int, choices=range(1, 4), 1 = print executed commands, 2 = print commands and non-zero output, 3 = print commands and all output.""") +# maybe add? +# outctl.add_argument('--version', action='version', version='%s 1.0' % script) logctl = parser.add_argument_group(description='Log control options:') logctl.add_argument('-o', '--output', metavar='LOG', help="Save console output to LOG.") @@ -132,14 +141,18 @@ logctl.add_argument('--xml', metavar='XML', help="Save results to XML in SCons X args = parser.parse_args() # we can't do this check with an argparse exclusive group, -# since the cmdline tests (args.testlist) are not optional -if args.testlist and (args.testlistfile or args.all): +# since the cmdline tests (args.testlist) are not optional args, +# exclusive only works with optional args. +if args.testlist and (args.testlistfile or args.all or args.retry): sys.stderr.write( parser.format_usage() - + "error: command line tests cannot be combined with -f/--file or -a/--all\n" + + "error: command line tests cannot be combined with -f/--filei -a/--all or --retry\n" ) sys.exit(1) +if args.retry: + args.testlistfile = 'failed_tests.log' + if args.testlistfile: # args.testlistfile changes from a string to a pathlib Path object try: @@ -196,7 +209,7 @@ if args.exec: scons = args.exec # --- setup stdout/stderr --- -class Unbuffered(): +class Unbuffered: def __init__(self, file): self.file = file @@ -215,7 +228,7 @@ sys.stderr = Unbuffered(sys.stderr) if args.output: logfile = open(args.output, 'w') - class Tee(): + class Tee: def __init__(self, openfile, stream): self.file = openfile self.stream = stream @@ -780,6 +793,7 @@ passed = [t for t in tests if t.status == 0] fail = [t for t in tests if t.status == 1] no_result = [t for t in tests if t.status == 2] +# print summaries, but only if multiple tests were run if len(tests) != 1 and args.execute_tests: if passed and args.print_passed_summary: if len(passed) == 1: @@ -803,6 +817,14 @@ if len(tests) != 1 and args.execute_tests: paths = [x.path for x in no_result] sys.stdout.write("\t" + "\n\t".join(paths) + "\n") +# save the fails to a file +if fail and args.error_log: + paths = [x.path for x in fail] + #print(f"DEBUG: Writing fails to {args.error_log}") + with open(args.error_log, "w") as f: + for test in paths: + print(test, file=f) + if args.xml: if args.output == '-': f = sys.stdout diff --git a/test/runtest/SCons.py b/test/runtest/SCons.py index 9bc86e8..20c4c64 100644 --- a/test/runtest/SCons.py +++ b/test/runtest/SCons.py @@ -34,9 +34,7 @@ import os import TestRuntest test = TestRuntest.TestRuntest() - -test.subdir(['SCons'], - ['SCons', 'suite']) +test.subdir(['SCons'], ['SCons', 'suite']) pythonstring = TestRuntest.pythonstring pythonflags = TestRuntest.pythonflags @@ -44,11 +42,8 @@ src_passTests_py = os.path.join('SCons', 'passTests.py') src_suite_passTests_py = os.path.join('SCons', 'suite', 'passTests.py') test.write_passing_test(['SCons', 'pass.py']) - test.write_passing_test(['SCons', 'passTests.py']) - test.write_passing_test(['SCons', 'suite', 'pass.py']) - test.write_passing_test(['SCons', 'suite', 'passTests.py']) expect_stdout = """\ diff --git a/test/runtest/faillog.py b/test/runtest/faillog.py new file mode 100644 index 0000000..e2ca67e --- /dev/null +++ b/test/runtest/faillog.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test a list of tests in failed_tests.log to run with the --retry option +""" + +import os.path + +import TestRuntest + +pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags +test_fail_py = os.path.join('test', 'fail.py') +test_pass_py = os.path.join('test', 'pass.py') + +test = TestRuntest.TestRuntest() +test.subdir('test') +test.write_failing_test(test_fail_py) +test.write_passing_test(test_pass_py) + +expect_stdout = """\ +%(pythonstring)s%(pythonflags)s %(test_fail_py)s +FAILING TEST STDOUT +%(pythonstring)s%(pythonflags)s %(test_pass_py)s +PASSING TEST STDOUT + +Failed the following test: +\t%(test_fail_py)s +""" % locals() + +expect_stderr = """\ +FAILING TEST STDERR +PASSING TEST STDERR +""" + +testlist = [ + test_fail_py, + test_pass_py, +] + +test.run( + arguments='-k --faillog=fail.log %s' % ' '.join(testlist), + status=1, + stdout=expect_stdout, + stderr=expect_stderr, +) +test.must_exist('fail.log') +test.must_contain('fail.log', test_fail_py) +test.must_not_exist('failed_tests.log') + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/runtest/fallback.py b/test/runtest/fallback.py deleted file mode 100644 index b137307..0000000 --- a/test/runtest/fallback.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -""" -Test that runtest.py falls back (with a warning) using --noqmtest -if it can't find qmtest on the $PATH. -""" - -import os - -import TestRuntest - -pythonstring = TestRuntest.pythonstring -pythonflags = TestRuntest.pythonflags - -test = TestRuntest.TestRuntest() - -# qmtest may be in more than one location in your path -while test.where_is('qmtest'): - qmtest=test.where_is('qmtest') - dir = os.path.split(qmtest)[0] - path = os.environ['PATH'].split(os.pathsep) - path.remove(dir) - os.environ['PATH'] = os.pathsep.join(path) - -test.subdir('test') - -test_fail_py = os.path.join('test', 'fail.py') -test_no_result_py = os.path.join('test', 'no_result.py') -test_pass_py = os.path.join('test', 'pass.py') - -test.write_failing_test(test_fail_py) -test.write_no_result_test(test_no_result_py) -test.write_passing_test(test_pass_py) - -expect_stdout = """\ -%(pythonstring)s%(pythonflags)s %(test_fail_py)s -FAILING TEST STDOUT -%(pythonstring)s%(pythonflags)s %(test_no_result_py)s -NO RESULT TEST STDOUT -%(pythonstring)s%(pythonflags)s %(test_pass_py)s -PASSING TEST STDOUT - -Failed the following test: -\t%(test_fail_py)s - -NO RESULT from the following test: -\t%(test_no_result_py)s -""" % locals() - -expect_stderr = """\ -FAILING TEST STDERR -NO RESULT TEST STDERR -PASSING TEST STDERR -""" - -testlist = [ - test_fail_py, - test_no_result_py, - test_pass_py, -] - -test.run(arguments = '-k '+' '.join(testlist), - status = 1, - stdout = expect_stdout, - stderr = expect_stderr) - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/runtest/no_faillog.py b/test/runtest/no_faillog.py new file mode 100644 index 0000000..db17c8e --- /dev/null +++ b/test/runtest/no_faillog.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test a list of tests in failed_tests.log to run with the --retry option +""" + +import os.path + +import TestRuntest + +pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags +test_fail_py = os.path.join('test', 'fail.py') +test_pass_py = os.path.join('test', 'pass.py') + +test = TestRuntest.TestRuntest() +test.subdir('test') +test.write_failing_test(test_fail_py) +test.write_passing_test(test_pass_py) + +test.write('failed_tests.log', """\ +%(test_fail_py)s +""" % locals()) + +expect_stdout = """\ +%(pythonstring)s%(pythonflags)s %(test_fail_py)s +FAILING TEST STDOUT +%(pythonstring)s%(pythonflags)s %(test_pass_py)s +PASSING TEST STDOUT + +Failed the following test: +\t%(test_fail_py)s +""" % locals() + +expect_stderr = """\ +FAILING TEST STDERR +PASSING TEST STDERR +""" + +testlist = [ + test_fail_py, + test_pass_py, +] + +test.run( + arguments='-k --no-faillog %s' % ' '.join(testlist), + status=1, + stdout=expect_stdout, + stderr=expect_stderr, +) +test.must_not_exist('failing_tests.log') + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/runtest/noqmtest.py b/test/runtest/noqmtest.py deleted file mode 100644 index fcf7ac0..0000000 --- a/test/runtest/noqmtest.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -""" -Test that by default tests are invoked directly via Python, not -using qmtest. -""" - -import os - -import TestRuntest - -pythonstring = TestRuntest.pythonstring -pythonflags = TestRuntest.pythonflags - -test = TestRuntest.TestRuntest() - -test.subdir('test') - -test_fail_py = os.path.join('test', 'fail.py') -test_no_result_py = os.path.join('test', 'no_result.py') -test_pass_py = os.path.join('test', 'pass.py') - -test.write_failing_test(test_fail_py) -test.write_no_result_test(test_no_result_py) -test.write_passing_test(test_pass_py) - -expect_stdout = """\ -%(pythonstring)s%(pythonflags)s %(test_fail_py)s -FAILING TEST STDOUT -%(pythonstring)s%(pythonflags)s %(test_no_result_py)s -NO RESULT TEST STDOUT -%(pythonstring)s%(pythonflags)s %(test_pass_py)s -PASSING TEST STDOUT - -Failed the following test: -\t%(test_fail_py)s - -NO RESULT from the following test: -\t%(test_no_result_py)s -""" % locals() - -expect_stderr = """\ -FAILING TEST STDERR -NO RESULT TEST STDERR -PASSING TEST STDERR -""" - -testlist = [ - test_fail_py, - test_no_result_py, - test_pass_py, -] - -test.run(arguments = '-k %s' % ' '.join(testlist), - status = 1, - stdout = expect_stdout, - stderr = expect_stderr) - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/runtest/print_time.py b/test/runtest/print_time.py index 322b88b..834d2ae 100644 --- a/test/runtest/print_time.py +++ b/test/runtest/print_time.py @@ -42,13 +42,9 @@ test_no_result_py = re.escape(os.path.join('test', 'no_result.py')) test_pass_py = re.escape(os.path.join('test', 'pass.py')) test = TestRuntest.TestRuntest(match = TestCmd.match_re) - test.subdir('test') - test.write_failing_test(['test', 'fail.py']) - test.write_no_result_test(['test', 'no_result.py']) - test.write_passing_test(['test', 'pass.py']) expect_stdout = """\ diff --git a/test/runtest/python.py b/test/runtest/python.py index da62378..499ab77 100644 --- a/test/runtest/python.py +++ b/test/runtest/python.py @@ -37,9 +37,7 @@ if not hasattr(os.path, 'pardir'): import TestRuntest test = TestRuntest.TestRuntest() - test_pass_py = os.path.join('test', 'pass.py') - head, python = os.path.split(TestRuntest.python) head, dir = os.path.split(head) diff --git a/test/runtest/retry.py b/test/runtest/retry.py new file mode 100644 index 0000000..4280152 --- /dev/null +++ b/test/runtest/retry.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test a list of tests in failed_tests.log to run with the --retry option +""" + +import os.path + +import TestRuntest + +pythonstring = TestRuntest.pythonstring +pythonflags = TestRuntest.pythonflags +test_fail_py = os.path.join('test', 'fail.py') +test_no_result_py = os.path.join('test', 'no_result.py') +test_pass_py = os.path.join('test', 'pass.py') + +test = TestRuntest.TestRuntest() + +test.subdir('test') +test.write_failing_test(['test', 'fail.py']) +test.write_no_result_test(['test', 'no_result.py']) +test.write_passing_test(['test', 'pass.py']) + +test.write('failed_tests.log', """\ +%(test_fail_py)s +""" % locals()) + +expect_stdout = """\ +%(pythonstring)s%(pythonflags)s %(test_fail_py)s +FAILING TEST STDOUT +""" % locals() + +expect_stderr = """\ +FAILING TEST STDERR +""" + +test.run(arguments='-k --retry', status=1, stdout=expect_stdout, stderr=expect_stderr) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/runtest/simple/combined.py b/test/runtest/simple/combined.py index ec0a1bb..a54e57c 100644 --- a/test/runtest/simple/combined.py +++ b/test/runtest/simple/combined.py @@ -1,4 +1,3 @@ - #!/usr/bin/env python # # __COPYRIGHT__ @@ -34,21 +33,17 @@ import os import TestRuntest -test = TestRuntest.TestRuntest() - pythonstring = TestRuntest.pythonstring pythonflags = TestRuntest.pythonflags test_fail_py = os.path.join('test', 'fail.py') test_no_result_py = os.path.join('test', 'no_result.py') test_pass_py = os.path.join('test', 'pass.py') +test = TestRuntest.TestRuntest() test.subdir('test') - -test.write_failing_test(['test', 'fail.py']) - -test.write_no_result_test(['test', 'no_result.py']) - -test.write_passing_test(['test', 'pass.py']) +test.write_failing_test(test_fail_py) +test.write_no_result_test(test_no_result_py) +test.write_passing_test(test_pass_py) expect_stdout = """\ %(pythonstring)s%(pythonflags)s %(test_fail_py)s @@ -71,10 +66,14 @@ NO RESULT TEST STDERR PASSING TEST STDERR """ -test.run(arguments='-k test', - status=1, - stdout=expect_stdout, - stderr=expect_stderr) +test.run( + arguments='-k test', + status=1, + stdout=expect_stdout, + stderr=expect_stderr +) +test.must_exist('failed_tests.log') +test.must_contain('failed_tests.log', test_fail_py) test.pass_test() diff --git a/test/runtest/testlistfile.py b/test/runtest/testlistfile.py index ba034e8..5c956b8 100644 --- a/test/runtest/testlistfile.py +++ b/test/runtest/testlistfile.py @@ -26,6 +26,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Test a list of tests to run in a file specified with the -f option. +The commented-out test should not run. """ import os.path @@ -41,11 +42,8 @@ test_pass_py = os.path.join('test', 'pass.py') test = TestRuntest.TestRuntest() test.subdir('test') - test.write_failing_test(['test', 'fail.py']) - test.write_no_result_test(['test', 'no_result.py']) - test.write_passing_test(['test', 'pass.py']) test.write('t.txt', """\ -- cgit v0.12 From 82bdb9c3423ae80461af2c5e224456e0ba181972 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 8 Nov 2020 06:56:13 -0700 Subject: [PR #3822] fix typo, reword comment Signed-off-by: Mats Wichmann --- CHANGES.txt | 3 ++- runtest.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 87b20c4..77fb110 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -75,7 +75,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER of hand-coded and optparse, with a stated intent to "gradually port"). - Add options to runtest to generate/not generate a log of failed tests, and to rerun such tests. Useful when an error cascades through several - tests, can quickly try if a change improves all the fails. + tests, can quickly try if a change improves all the fails. Dropped + runtest test for fallback from qmtest, not needed; added new tests. From Simon Tegelid - Fix using TEMPFILE in multiple actions in an action list. Previously a builder, or command diff --git a/runtest.py b/runtest.py index df7c328..9e7cb2c 100755 --- a/runtest.py +++ b/runtest.py @@ -140,13 +140,13 @@ logctl.add_argument('--xml', metavar='XML', help="Save results to XML in SCons X # process args and handle a few specific cases: args = parser.parse_args() -# we can't do this check with an argparse exclusive group, -# since the cmdline tests (args.testlist) are not optional args, -# exclusive only works with optional args. +# we can't do this check with an argparse exclusive group, since those +# only work with optional args, and the cmdline tests (args.testlist) +# are not optional args, if args.testlist and (args.testlistfile or args.all or args.retry): sys.stderr.write( parser.format_usage() - + "error: command line tests cannot be combined with -f/--filei -a/--all or --retry\n" + + "error: command line tests cannot be combined with -f/--file, -a/--all or --retry\n" ) sys.exit(1) -- cgit v0.12