From 2e3fb8d5b786eec70b69ae2a058581d6fa892bf9 Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Wed, 12 Dec 2012 18:02:28 +0300 Subject: Remove subprocess compatibility code used for Python < 2.4 --- QMTest/TestCmd.py | 70 +++----------- QMTest/TestCmdTests.py | 70 +++----------- runtest.py | 255 +++++++++++++++++++++---------------------------- 3 files changed, 129 insertions(+), 266 deletions(-) diff --git a/QMTest/TestCmd.py b/QMTest/TestCmd.py index 708fdc4..0c42ab5 100644 --- a/QMTest/TestCmd.py +++ b/QMTest/TestCmd.py @@ -654,68 +654,20 @@ else: +import subprocess + try: - import subprocess -except ImportError: - # The subprocess module doesn't exist in this version of Python, - # so we're going to cobble up something that looks just enough - # like its API for our purposes below. - import popen2 - subprocess = types.ModuleType('subprocess') - - subprocess.PIPE = 'PIPE' - subprocess.STDOUT = 'STDOUT' - subprocess.mswindows = (sys.platform == 'win32') - - class Popen(popen2.Popen3, popen2.Popen4): - universal_newlines = 1 - def __init__(self, command, **kw): - if kw.get('stderr') == 'STDOUT': - popen2.Popen4.__init__(self, command, 1) - else: - popen2.Popen3.__init__(self, command, 1) - self.stdin = self.tochild - self.stdout = self.fromchild - self.stderr = self.childerr - def communicate(self, input=None): - if input: - self.stdin.write(input) - self.stdin.close() - out = self.stdout.read() - if self.stderr is None: - err = None - else: - err = self.stderr.read() - self.stdout.close() - if self.stderr is not None: - self.stderr.close() - self.returncode = self.wait() - return (out, err) + subprocess.Popen.terminate +except AttributeError: + if sys.platform == 'win32': + import win32process + def terminate(self): + win32process.TerminateProcess(self._handle, 1) + else: def terminate(self): os.kill(self.pid, signal.SIGTERM) - def wait(self, *args, **kw): - resultcode = popen2.Popen3.wait(self, *args, **kw) - if os.WIFSIGNALED(resultcode): - return (- os.WTERMSIG(resultcode)) - elif os.WIFEXITED(resultcode): - return os.WEXITSTATUS(resultcode) - else: - return None - - subprocess.Popen = Popen -else: - try: - subprocess.Popen.terminate - except AttributeError: - if sys.platform == 'win32': - import win32process - def terminate(self): - win32process.TerminateProcess(self._handle, 1) - else: - def terminate(self): - os.kill(self.pid, signal.SIGTERM) - method = types.MethodType(terminate, None, subprocess.Popen) - setattr(subprocess.Popen, 'terminate', method) + method = types.MethodType(terminate, None, subprocess.Popen) + setattr(subprocess.Popen, 'terminate', method) diff --git a/QMTest/TestCmdTests.py b/QMTest/TestCmdTests.py index 1fe328c..1044ed1 100644 --- a/QMTest/TestCmdTests.py +++ b/QMTest/TestCmdTests.py @@ -58,68 +58,20 @@ def _clear_dict(dict, *keys): except KeyError: pass +import subprocess + try: - import subprocess -except ImportError: - # The subprocess module doesn't exist in this version of Python, - # so we're going to cobble up something that looks just enough - # like its API for our purposes below. - import popen2 - subprocess = types.ModuleType('subprocess') - - subprocess.PIPE = 'PIPE' - subprocess.STDOUT = 'STDOUT' - subprocess.mswindows = (sys.platform == 'win32') - - class Popen(popen2.Popen3, popen2.Popen4): - universal_newlines = 1 - def __init__(self, command, **kw): - if kw.get('stderr') == 'STDOUT': - popen2.Popen4.__init__(self, command, 1) - else: - popen2.Popen3.__init__(self, command, 1) - self.stdin = self.tochild - self.stdout = self.fromchild - self.stderr = self.childerr - def communicate(self, input=None): - if input: - self.stdin.write(input) - self.stdin.close() - out = self.stdout.read() - if self.stderr is None: - err = None - else: - err = self.stderr.read() - self.stdout.close() - if self.stderr is not None: - self.stderr.close() - self.returncode = self.wait() - return (out, err) + subprocess.Popen.terminate +except AttributeError: + if sys.platform == 'win32': + import win32process + def terminate(self): + win32process.TerminateProcess(self._handle, 1) + else: def terminate(self): os.kill(self.pid, signal.SIGTERM) - def wait(self, *args, **kw): - resultcode = popen2.Popen3.wait(self, *args, **kw) - if os.WIFEXITED(resultcode): - return os.WEXITSTATUS(resultcode) - elif os.WIFSIGNALED(resultcode): - return os.WTERMSIG(resultcode) - else: - return None - - subprocess.Popen = Popen -else: - try: - subprocess.Popen.terminate - except AttributeError: - if sys.platform == 'win32': - import win32process - def terminate(self): - win32process.TerminateProcess(self._handle, 1) - else: - def terminate(self): - os.kill(self.pid, signal.SIGTERM) - method = types.MethodType(terminate, None, subprocess.Popen) - setattr(subprocess.Popen, 'terminate', method) + method = types.MethodType(terminate, None, subprocess.Popen) + setattr(subprocess.Popen, 'terminate', method) class ExitError(Exception): pass diff --git a/runtest.py b/runtest.py index 55a16f5..3af7766 100644 --- a/runtest.py +++ b/runtest.py @@ -308,93 +308,79 @@ def escape(s): s = s.replace('\\', '\\\\') return s -# Try to use subprocess instead of the more low-level -# spawn command... -use_subprocess = True -try: - import subprocess -except: - use_subprocess = False - -if use_subprocess: - if not suppress_stdout and not suppress_stderr: - # Without any output suppressed, we let the subprocess - # write its stuff freely to stdout/stderr. + +import subprocess + +if not suppress_stdout and not suppress_stderr: + # Without any output suppressed, we let the subprocess + # write its stuff freely to stdout/stderr. + def spawn_it(command_args): + p = subprocess.Popen(' '.join(command_args), + shell=True) + return (None, None, p.wait()) +else: + # Else, we catch the output of both pipes... + if allow_pipe_files: + # The subprocess.Popen() suffers from a well-known + # problem. Data for stdout/stderr is read into a + # memory buffer of fixed size, 65K which is not very much. + # When it fills up, it simply stops letting the child process + # write to it. The child will then sit and patiently wait to + # be able to write the rest of its output. Hang! + # In order to work around this, we follow a suggestion + # by Anders Pearson in + # http://http://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-is-your-enemy/ + # and pass temp file objects to Popen() instead of the ubiquitous + # subprocess.PIPE. def spawn_it(command_args): + # Create temporary files + import tempfile + tmp_stdout = tempfile.TemporaryFile(mode='w+t') + tmp_stderr = tempfile.TemporaryFile(mode='w+t') + # Start subprocess... p = subprocess.Popen(' '.join(command_args), + stdout=tmp_stdout, + stderr=tmp_stderr, shell=True) - return (None, None, p.wait()) - else: - # Else, we catch the output of both pipes... - if allow_pipe_files: - # The subprocess.Popen() suffers from a well-known - # problem. Data for stdout/stderr is read into a - # memory buffer of fixed size, 65K which is not very much. - # When it fills up, it simply stops letting the child process - # write to it. The child will then sit and patiently wait to - # be able to write the rest of its output. Hang! - # In order to work around this, we follow a suggestion - # by Anders Pearson in - # http://http://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-is-your-enemy/ - # and pass temp file objects to Popen() instead of the ubiquitous - # subprocess.PIPE. - def spawn_it(command_args): - # Create temporary files - import tempfile - tmp_stdout = tempfile.TemporaryFile(mode='w+t') - tmp_stderr = tempfile.TemporaryFile(mode='w+t') - # Start subprocess... - p = subprocess.Popen(' '.join(command_args), - stdout=tmp_stdout, - stderr=tmp_stderr, - shell=True) - # ... and wait for it to finish. - ret = p.wait() - - try: - # Rewind to start of files - tmp_stdout.seek(0) - tmp_stderr.seek(0) - # Read output - spawned_stdout = tmp_stdout.read() - spawned_stderr = tmp_stderr.read() - finally: - # Remove temp files by closing them - tmp_stdout.close() - tmp_stderr.close() - - # Return values - return (spawned_stderr, spawned_stdout, ret) + # ... and wait for it to finish. + ret = p.wait() - else: - # We get here only if the user gave the '--nopipefiles' - # option, meaning the "temp file" approach for - # subprocess.communicate() above shouldn't be used. - # He hopefully knows what he's doing, but again we have a - # potential deadlock situation in the following code: - # If the subprocess writes a lot of data to its stderr, - # the pipe will fill up (nobody's reading it yet) and the - # subprocess will wait for someone to read it. - # But the parent process is trying to read from stdin - # (but the subprocess isn't writing anything there). - # Hence a deadlock. - # Be dragons here! Better don't use this! - def spawn_it(command_args): - p = subprocess.Popen(' '.join(command_args), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True) - spawned_stdout = p.stdout.read() - spawned_stderr = p.stderr.read() - return (spawned_stderr, spawned_stdout, p.wait()) -else: - has_subprocess = False - # Set up lowest-common-denominator spawning of a process on both Windows - # and non-Windows systems that works all the way back to Python 1.6 - def spawn_it(command_args): - command = command_args[0] - command_args = [escape(c) for c in command_args] - return (None, None, os.spawnv(os.P_WAIT, command, command_args)) + try: + # Rewind to start of files + tmp_stdout.seek(0) + tmp_stderr.seek(0) + # Read output + spawned_stdout = tmp_stdout.read() + spawned_stderr = tmp_stderr.read() + finally: + # Remove temp files by closing them + tmp_stdout.close() + tmp_stderr.close() + + # Return values + return (spawned_stderr, spawned_stdout, ret) + + else: + # We get here only if the user gave the '--nopipefiles' + # option, meaning the "temp file" approach for + # subprocess.communicate() above shouldn't be used. + # He hopefully knows what he's doing, but again we have a + # potential deadlock situation in the following code: + # If the subprocess writes a lot of data to its stderr, + # the pipe will fill up (nobody's reading it yet) and the + # subprocess will wait for someone to read it. + # But the parent process is trying to read from stdin + # (but the subprocess isn't writing anything there). + # Hence a deadlock. + # Be dragons here! Better don't use this! + def spawn_it(command_args): + p = subprocess.Popen(' '.join(command_args), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) + spawned_stdout = p.stdout.read() + spawned_stderr = p.stderr.read() + return (spawned_stderr, spawned_stdout, p.wait()) class Base(object): def __init__(self, path, spe=None): @@ -415,71 +401,44 @@ class SystemExecutor(Base): if s < 0 or s > 2: sys.stdout.write("Unexpected exit status %d\n" % s) -if not use_subprocess: - import popen2 - try: - popen2.Popen3 - except AttributeError: - class PopenExecutor(Base): - def execute(self): - (tochild, fromchild, childerr) = os.popen3(self.command_str) - tochild.close() - self.stderr = childerr.read() - self.stdout = fromchild.read() - fromchild.close() - self.status = childerr.close() - if not self.status: - self.status = 0 - else: - self.status = self.status >> 8 - else: - class PopenExecutor(Base): - def execute(self): - p = popen2.Popen3(self.command_str, 1) - p.tochild.close() - self.stdout = p.fromchild.read() - self.stderr = p.childerr.read() - self.status = p.wait() - self.status = self.status >> 8 -else: - class PopenExecutor(Base): - # For an explanation of the following 'if ... else' - # and the 'allow_pipe_files' option, please check out the - # use_subprocess path in the definition of spawn_it() above. - if allow_pipe_files: - def execute(self): - # Create temporary files - import tempfile - tmp_stdout = tempfile.TemporaryFile(mode='w+t') - tmp_stderr = tempfile.TemporaryFile(mode='w+t') - # Start subprocess... - p = subprocess.Popen(self.command_str, - stdout=tmp_stdout, - stderr=tmp_stderr, - shell=True) - # ... and wait for it to finish. - self.status = p.wait() - - try: - # Rewind to start of files - tmp_stdout.seek(0) - tmp_stderr.seek(0) - # Read output - self.stdout = tmp_stdout.read() - self.stderr = tmp_stderr.read() - finally: - # Remove temp files by closing them - tmp_stdout.close() - tmp_stderr.close() - else: - def execute(self): - p = subprocess.Popen(self.command_str, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True) - self.stdout = p.stdout.read() - self.stderr = p.stderr.read() - self.status = p.wait() +class PopenExecutor(Base): + # For an explanation of the following 'if ... else' + # and the 'allow_pipe_files' option, please check out the + # definition of spawn_it() above. + if allow_pipe_files: + def execute(self): + # Create temporary files + import tempfile + tmp_stdout = tempfile.TemporaryFile(mode='w+t') + tmp_stderr = tempfile.TemporaryFile(mode='w+t') + # Start subprocess... + p = subprocess.Popen(self.command_str, + stdout=tmp_stdout, + stderr=tmp_stderr, + shell=True) + # ... and wait for it to finish. + self.status = p.wait() + + try: + # Rewind to start of files + tmp_stdout.seek(0) + tmp_stderr.seek(0) + # Read output + self.stdout = tmp_stdout.read() + self.stderr = tmp_stderr.read() + finally: + # Remove temp files by closing them + tmp_stdout.close() + tmp_stderr.close() + else: + def execute(self): + p = subprocess.Popen(self.command_str, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) + self.stdout = p.stdout.read() + self.stderr = p.stderr.read() + self.status = p.wait() class XML(PopenExecutor): def header(self, f): -- cgit v0.12 From b70731ca1c406687d1f53d7ff2ed357072e40e85 Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Fri, 14 Dec 2012 16:09:03 +0300 Subject: CHANGES.txt: clean up after which Python 2.3 is unlikely to work --- src/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 6136f81..18b2c18 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -10,6 +10,8 @@ RELEASE 2.X.X - - Added ability to run scripts/scons.py directly from source checkout - Hide deprecated --debug={dtree,stree,tree} from --help output - Error messages from option parser now include hints about valid choices + - Cleaned up some Python 1.5 and pre-2.3 code, so don't expect SCons + to run on anything less than Python 2.4 anymore From Juan Lang: - Fix WiX Tool to use .wixobj rather than .wxiobj for compiler output -- cgit v0.12 From cab504dd1de67c0eaeb8f7f4553826937303a3e6 Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Fri, 14 Dec 2012 16:09:38 +0300 Subject: runtest.py: Gradually moving from getopt to optparse --- runtest.py | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/runtest.py b/runtest.py index 3af7766..d6af07e 100644 --- a/runtest.py +++ b/runtest.py @@ -94,7 +94,6 @@ import time cwd = os.getcwd() -all = 0 baseline = 0 builddir = os.path.join(cwd, 'build') external = 0 @@ -171,9 +170,37 @@ Environment Variables: PRESERVE, PRESERVE_{PASS,FAIL,NO_RESULT}: preserve test subdirs TESTCMD_VERBOSE: turn on verbosity in TestCommand """ - -opts, args = getopt.getopt(sys.argv[1:], "3ab:def:hklno:P:p:qsv:Xx:t", - ['all', 'baseline=', 'builddir=', + + +# "Pass-through" option parsing -- an OptionParser that ignores +# unknown options and lets them pile up in the leftover argument +# list. Useful to gradually port getopt to optparse. + +from optparse import OptionParser, BadOptionError + +class PassThroughOptionParser(OptionParser): + def _process_long_opt(self, rargs, values): + try: + OptionParser._process_long_opt(self, rargs, values) + except BadOptionError, err: + self.largs.append(err.opt_str) + def _process_short_opts(self, rargs, values): + try: + OptionParser._process_short_opts(self, rargs, values) + except BadOptionError, err: + self.largs.append(err.opt_str) + +parser = PassThroughOptionParser(add_help_option=False) +parser.add_option('-a', '--all', action='store_true', + help="Run all tests.") +(options, args) = parser.parse_args() + +#print "options:", options +#print "args:", args + + +opts, args = getopt.getopt(args, "3b:def:hklno:P:p:qsv:Xx:t", + ['baseline=', 'builddir=', 'debug', 'external', 'file=', 'help', 'no-progress', 'list', 'no-exec', 'noqmtest', 'nopipefiles', 'output=', 'package=', 'passed', 'python=', 'qmtest', @@ -184,8 +211,6 @@ opts, args = getopt.getopt(sys.argv[1:], "3ab:def:hklno:P:p:qsv:Xx:t", for o, a in opts: if o in ['-3']: python3incompatibilities = 1 - elif o in ['-a', '--all']: - all = 1 elif o in ['-b', '--baseline']: baseline = a elif o in ['--builddir']: @@ -254,7 +279,7 @@ for o, a in opts: elif o in ['--xml']: format = o -if not args and not all and not testlistfile: +if not args and not options.all and not testlistfile: sys.stderr.write("""\ runtest.py: No tests were specified. List one or more tests on the command line, use the @@ -649,7 +674,7 @@ elif testlistfile: tests = [x for x in tests if x[0] != '#'] tests = [x[:-1] for x in tests] tests = [x.strip() for x in tests] -elif all and not qmtest: +elif options.all and not qmtest: # Find all of the SCons functional tests in the local directory # tree. This is anything under the 'src' subdirectory that ends # with 'Tests.py', or any Python script (*.py) under the 'test' -- cgit v0.12 From 7911aed366660c73c2d348d31e1657ed73372540 Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Sat, 15 Dec 2012 16:51:47 +0300 Subject: runtest.py: Turn on unbuffered output ASAP. --- runtest.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/runtest.py b/runtest.py index d6af07e..e482dc6 100644 --- a/runtest.py +++ b/runtest.py @@ -289,6 +289,23 @@ runtest.py: No tests were specified. """) sys.exit(1) + +# --- setup stdout/stderr --- +class Unbuffered(object): + def __init__(self, file): + self.file = file + self.softspace = 0 ## backward compatibility; not supported in Py3k + def write(self, arg): + self.file.write(arg) + self.file.flush() + def __getattr__(self, attr): + return getattr(self.file, attr) + +sys.stdout = Unbuffered(sys.stdout) +sys.stderr = Unbuffered(sys.stderr) + + +# --- define helpers ---- if sys.platform in ('win32', 'cygwin'): def whereis(file): @@ -488,6 +505,7 @@ format_class = { Test = format_class[format] +# --- start processing --- if package: dir = { @@ -746,19 +764,6 @@ if qmtest: tests = [Test(t) for t in tests] -class Unbuffered(object): - def __init__(self, file): - self.file = file - self.softspace = 0 ## backward compatibility; not supported in Py3k - def write(self, arg): - self.file.write(arg) - self.file.flush() - def __getattr__(self, attr): - return getattr(self.file, attr) - -sys.stdout = Unbuffered(sys.stdout) -sys.stderr = Unbuffered(sys.stderr) - if list_only: for t in tests: sys.stdout.write(t.path + "\n") -- cgit v0.12 From 78785a796c5cb080dc6a0e7bc99c720a349b3859 Mon Sep 17 00:00:00 2001 From: Dirk Baechle Date: Sun, 16 Dec 2012 18:56:16 +0100 Subject: - extended the must_contain* methods of the test framework, such that they all support user-defined find/search functions - relaxed several of the regex comparisons for better cross-platform compatibility --- QMTest/TestCommon.py | 31 ++++++++++++++------ QMTest/TestSCons.py | 28 ++++++++++++++++++ test/Delete.py | 1 + test/Errors/execute-a-directory.py | 47 ++++++++++++++----------------- test/Errors/non-executable-file.py | 38 ++++++++++++------------- test/Errors/nonexistent-executable.py | 53 +++++++++-------------------------- test/Execute.py | 20 ++++++------- test/Install/Install.py | 1 + test/Interactive/shell.py | 16 +++++------ test/Object.py | 6 +--- test/Repository/StaticLibrary.py | 11 ++------ test/scons-time/func/file.py | 11 ++++---- 12 files changed, 133 insertions(+), 130 deletions(-) diff --git a/QMTest/TestCommon.py b/QMTest/TestCommon.py index 90aaed2..6d47149 100644 --- a/QMTest/TestCommon.py +++ b/QMTest/TestCommon.py @@ -286,11 +286,17 @@ class TestCommon(TestCmd): print "Unwritable files: `%s'" % "', `".join(unwritable) self.fail_test(missing + unwritable) - def must_contain(self, file, required, mode = 'rb'): + def must_contain(self, file, required, mode = 'rb', find = None): """Ensures that the specified file contains the required text. """ file_contents = self.read(file, mode) - contains = (file_contents.find(required) != -1) + if find is None: + def find(o, l): + try: + return o.index(l) + except ValueError: + return None + contains = find(file_contents, required) if not contains: print "File `%s' does not contain required string." % file print self.banner('Required string ') @@ -317,6 +323,9 @@ class TestCommon(TestCmd): except ValueError: return None missing = [] + if is_List(output): + output = '\n'.join(output) + for line in lines: if find(output, line) is None: missing.append(line) @@ -415,9 +424,9 @@ class TestCommon(TestCmd): sys.stdout.flush() self.fail_test() - def must_contain_lines(self, lines, output, title=None): + def must_contain_lines(self, lines, output, title=None, find = None): # Deprecated; retain for backwards compatibility. - return self.must_contain_all_lines(output, lines, title) + return self.must_contain_all_lines(output, lines, title, find) def must_exist(self, *files): """Ensures that the specified file(s) must exist. An individual @@ -467,11 +476,17 @@ class TestCommon(TestCmd): self.diff(expect, file_contents, 'contents ') raise - def must_not_contain(self, file, banned, mode = 'rb'): + def must_not_contain(self, file, banned, mode = 'rb', find = None): """Ensures that the specified file doesn't contain the banned text. """ file_contents = self.read(file, mode) - contains = (file_contents.find(banned) != -1) + if find is None: + def find(o, l): + try: + return o.index(l) + except ValueError: + return None + contains = find(file_contents, banned) if contains: print "File `%s' contains banned string." % file print self.banner('Banned string ') @@ -512,8 +527,8 @@ class TestCommon(TestCmd): sys.stdout.write(output) self.fail_test() - def must_not_contain_lines(self, lines, output, title=None): - return self.must_not_contain_any_line(output, lines, title) + def must_not_contain_lines(self, lines, output, title=None, find=None): + return self.must_not_contain_any_line(output, lines, title, find) def must_not_exist(self, *files): """Ensures that the specified file(s) must not exist. diff --git a/QMTest/TestSCons.py b/QMTest/TestSCons.py index 296af67..ea18757 100644 --- a/QMTest/TestSCons.py +++ b/QMTest/TestSCons.py @@ -108,7 +108,35 @@ def re_escape(str): str = str.replace(c, '\\' + c) return str +# +# Helper functions that we use as a replacement to the default re.match +# when searching for special strings in stdout/stderr. +# +def search_re(out, l): + """ Search the regular expression 'l' in the output 'out' + and return the start index when successful. + """ + m = re.search(l, out) + if m: + return m.start() + + return None + +def search_re_in_list(out, l): + """ Search the regular expression 'l' in each line of + the given string list 'out' and return the line's index + when successful. + """ + for idx, o in enumerate(out): + m = re.search(l, o) + if m: + return idx + + return None +# +# Helpers for handling Python version numbers +# def python_version_string(): return sys.version.split()[0] diff --git a/test/Delete.py b/test/Delete.py index 49b4600..94ba24a 100644 --- a/test/Delete.py +++ b/test/Delete.py @@ -212,6 +212,7 @@ fail_strings = [ "No such file or directory", "The system cannot find the file specified", "The system cannot find the path specified", + "Das System kann die angegebene Datei nicht finden", ] test.must_contain_any_line(test.stderr(), fail_strings) diff --git a/test/Errors/execute-a-directory.py b/test/Errors/execute-a-directory.py index 1b679c6..1d2036e 100644 --- a/test/Errors/execute-a-directory.py +++ b/test/Errors/execute-a-directory.py @@ -53,63 +53,58 @@ Bad command or file name """ unrecognized = """\ -'%s' is not recognized as an internal or external command, +'.+' is not recognized as an internal or external command, operable program or batch file. -scons: *** [%s] Error 1 +scons: \*\*\* \[%s\] Error 1 """ unspecified = """\ The name specified is not recognized as an internal or external command, operable program or batch file. -scons: *** [%s] Error 1 +scons: \*\*\* \[%s\] Error 1 """ cannot_execute = """\ -%s: cannot execute -scons: *** [%s] Error %s -""" - -Permission_denied = """\ -%s: Permission denied -scons: *** [%s] Error %s +(sh: )*.+: cannot execute +scons: \*\*\* \[%s\] Error %s """ permission_denied = """\ -%s: permission denied -scons: *** [%s] Error %s +.+: (p|P)ermission denied +scons: \*\*\* \[%s\] Error %s """ is_a_directory = """\ -%s: is a directory -scons: *** [%s] Error %s +.+: (i|I)s a directory +scons: \*\*\* \[%s\] Error %s """ -Is_a_directory = """\ -%s: Is a directory -scons: *** [%s] Error %s +konnte_nicht_gefunden_werden = """\ +Der Befehl ".+" ist entweder falsch geschrieben oder +konnte nicht gefunden werden. +scons: \*\*\* \[%s\] Error %s """ test.description_set("Incorrect STDERR:\n%s\n" % test.stderr()) if os.name == 'nt': errs = [ bad_command, - unrecognized % (test.workdir, 'f3'), + unrecognized % 'f3', + konnte_nicht_gefunden_werden % ('f3', 1), unspecified % 'f3' ] - test.fail_test(not test.stderr() in errs) elif sys.platform.find('sunos') != -1: errs = [ - cannot_execute % ('sh: %s' % test.workdir, 'f3', 1), + cannot_execute % ('f3', 1), ] - test.fail_test(not test.stderr() in errs) else: errs = [ - cannot_execute % (not_executable, 'f3', 126), - is_a_directory % (test.workdir, 'f3', 126), - Is_a_directory % (test.workdir, 'f3', 126), - Permission_denied % (test.workdir, 'f3', 126), + cannot_execute % ('f3', 126), + is_a_directory % ('f3', 126), + permission_denied % ('f3', 126), ] - test.must_contain_any_line(test.stderr(), errs) + +test.must_contain_any_line(test.stderr(), errs, find=TestSCons.search_re) test.pass_test() diff --git a/test/Errors/non-executable-file.py b/test/Errors/non-executable-file.py index db7c88a..e1b8f4e 100644 --- a/test/Errors/non-executable-file.py +++ b/test/Errors/non-executable-file.py @@ -42,30 +42,31 @@ Bad command or file name """ unrecognized = """\ -'%s' is not recognized as an internal or external command, +'.+' is not recognized as an internal or external command, operable program or batch file. -scons: *** [%s] Error 1 +scons: \*\*\* \[%s\] Error 1 """ unspecified = """\ The name specified is not recognized as an internal or external command, operable program or batch file. -scons: *** [%s] Error 1 +scons: \*\*\* \[%s\] Error 1 """ cannot_execute = """\ -%s: cannot execute -scons: *** [%s] Error %s +(sh: )*.+: cannot execute +scons: \*\*\* \[%s\] Error %s """ -Permission_denied = """\ -%s: Permission denied -scons: *** [%s] Error %s +permission_denied = """\ +.+: (p|P)ermission denied +scons: \*\*\* \[%s\] Error %s """ -permission_denied = """\ -%s: permission denied -scons: *** [%s] Error %s +konnte_nicht_gefunden_werden = """\ +Der Befehl ".+" ist entweder falsch geschrieben oder +konnte nicht gefunden werden. +scons: \*\*\* \[%s\] Error %s """ test.write('SConstruct', r""" @@ -83,22 +84,21 @@ test.description_set("Incorrect STDERR:\n%s\n" % test.stderr()) if os.name == 'nt': errs = [ bad_command, - unrecognized % (not_executable, 'f1'), + unrecognized % 'f1', + konnte_nicht_gefunden_werden % ('f1', 1), unspecified % 'f1' ] - test.fail_test(not test.stderr() in errs) elif sys.platform.find('sunos') != -1: errs = [ - cannot_execute % ('sh: %s' % not_executable, 'f1', 1), + cannot_execute % ('f1', 1), ] - test.fail_test(not test.stderr() in errs) else: errs = [ - cannot_execute % (not_executable, 'f1', 126), - Permission_denied % (not_executable, 'f1', 126), - permission_denied % (not_executable, 'f1', 126), + cannot_execute % ('f1', 126), + permission_denied % ('f1', 126), ] - test.must_contain_any_line(test.stderr(), errs) + +test.must_contain_any_line(test.stderr(), errs, find=TestSCons.search_re) test.pass_test() diff --git a/test/Errors/nonexistent-executable.py b/test/Errors/nonexistent-executable.py index b9deea1..acaaaeb 100644 --- a/test/Errors/nonexistent-executable.py +++ b/test/Errors/nonexistent-executable.py @@ -46,58 +46,33 @@ test.run(arguments='.', stderr = None, status = 2) -bad_command = """\ -Bad command or file name -""" - -unrecognized = """\ -'%s' is not recognized as an internal or external command, -operable program or batch file. -scons: *** [%s] Error 1 -""" - -unspecified = """\ -The name specified is not recognized as an -internal or external command, operable program or batch file. -scons: *** [%s] Error 1 -""" - -not_found_1_space = """\ -sh: %s: not found -scons: *** [%s] Error %s -""" - -not_found_2_spaces = """\ -sh: %s: not found -scons: *** [%s] Error %s -""" - -No_such = """\ -%s: No such file or directory -scons: *** [%s] Error %s -""" +bad_command = """Bad command or file name""" +unrecognized = r"""'.+' is not recognized as an internal or external command,\s+operable program or batch file.\sscons: \*\*\* \[%s\] Error 1""" +unspecified = r"""The name specified is not recognized as an\s+internal or external command, operable program or batch file.\s+scons: \*\*\* \[%s\] Error 1""" +not_found_space = r"""sh: (\d: )*.+: \s*not found\s+scons: \*\*\* \[%s\] Error %s""" +No_such = r""".+: No such file or directory\s+scons: \*\*\* \[%s\] Error %s""" +konnte_nicht_gefunden_werden = r"""Der Befehl ".+" ist entweder falsch geschrieben oder\s+konnte nicht gefunden werden.\s+scons: \*\*\* \[%s\] Error %s""" test.description_set("Incorrect STDERR:\n%s\n" % test.stderr()) if os.name == 'nt': errs = [ bad_command, - unrecognized % (no_such_file, 'f1'), + unrecognized % 'f1', + konnte_nicht_gefunden_werden % ('f1', 1), unspecified % 'f1' ] - test.fail_test(not test.stderr() in errs) elif sys.platform.find('sunos') != -1: errs = [ - not_found_1_space % (no_such_file, 'f1', 1), + not_found_space % ('f1', 1), ] - test.fail_test(not test.stderr() in errs) else: errs = [ - not_found_1_space % (no_such_file, 'f1', 1), - not_found_2_spaces % (no_such_file, 'f1', 1), - not_found_1_space % (no_such_file, 'f1', 127), - No_such % (no_such_file, 'f1', 127), + not_found_space % ('f1', 1), + not_found_space % ('f1', 127), + No_such % ('f1', 127), ] - test.must_contain_any_line(test.stderr(), errs) + +test.must_contain_any_line(test.stderr(), errs, find=TestSCons.search_re) test.pass_test() diff --git a/test/Execute.py b/test/Execute.py index 4caa4c4..2e53444 100644 --- a/test/Execute.py +++ b/test/Execute.py @@ -84,19 +84,17 @@ test.write('m.in', "m.in\n") import sys if sys.platform == 'win32': - expect = """\ -scons: *** Error 1 -scons: *** Error 2 -scons: *** nonexistent.in/*.*: The system cannot find the path specified -""" + expect = r"""scons: \*\*\* Error 1 +scons: \*\*\* Error 2 +scons: \*\*\* nonexistent.in/\*\.\*: (The system cannot find the path specified|Das System kann den angegebenen Pfad nicht finden)""" else: - expect = """\ -scons: *** Error 1 -scons: *** Error 2 -scons: *** nonexistent.in: No such file or directory -""" + expect = r"""scons: \*\*\* Error 1 +scons: \*\*\* Error 2 +scons: \*\*\* nonexistent\.in: No such file or directory""" + +test.run(arguments = '.', stdout = None, stderr = None) -test.run(arguments = '.', stderr=expect) +test.must_contain_all_lines(test.stderr(), expect.splitlines(), find=TestSCons.search_re) test.must_match('a.out', "a.in\n") test.must_match('b.out', "b.in\n") diff --git a/test/Install/Install.py b/test/Install/Install.py index 29a8276..adadfd9 100644 --- a/test/Install/Install.py +++ b/test/Install/Install.py @@ -136,6 +136,7 @@ f = open(f1_out, 'rb') expect = [ "Permission denied", "The process cannot access the file because it is being used by another process", + "Der Prozess kann nicht auf die Datei zugreifen, da sie von einem anderen Prozess verwendet wird", ] test.run(chdir = 'work', arguments = f1_out, stderr=None, status=2) diff --git a/test/Interactive/shell.py b/test/Interactive/shell.py index f4e89bd..842a12e 100644 --- a/test/Interactive/shell.py +++ b/test/Interactive/shell.py @@ -83,21 +83,21 @@ scons.send("build foo.out\n") scons.send("\n") if sys.platform == 'win32': - no_such_error = "'no_such_command' is not recognized as an internal or external command,\noperable program or batch file." + no_such_error = r"('no_such_command' is not recognized as an internal or external command,\s+operable program or batch file\.|Der Befehl \"no_such_command\" ist entweder falsch geschrieben oder\s+konnte nicht gefunden werden\.)" else: no_such_error = 'scons: no_such_command: No such file or directory' expect_stdout = """\ -scons>>> Copy("foo.out", "foo.in") -Touch("1") +scons>>> Copy\("foo.out", "foo.in"\) +Touch\("1"\) scons>>> hello from shell_command.py -scons>>> !%(_python_)s %(_shell_command_py_)s +scons>>> ![^"]+ ".*" hello from shell_command.py scons>>> hello from shell_command.py -scons>>> shell %(_python_)s %(_shell_command_py_)s +scons>>> shell [^"]+ ".*" hello from shell_command.py scons>>> hello from shell_command.py -scons>>> sh %(_python_)s %(_shell_command_py_)s +scons>>> sh [^"]+ ".*" hello from shell_command.py scons>>> %(no_such_error)s scons>>> !no_such_command arg1 arg2 @@ -108,9 +108,9 @@ scons: `foo.out' is up to date. scons>>> """ % locals() -test.finish(scons, stdout = expect_stdout) - +test.finish(scons, stdout = None) +test.must_contain_all_lines(test.stdout(), expect_stdout.splitlines(), find=TestSCons.search_re) test.pass_test() diff --git a/test/Object.py b/test/Object.py index 406c2c1..c07da46 100644 --- a/test/Object.py +++ b/test/Object.py @@ -24,13 +24,9 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" -import sys import TestSCons -if sys.platform == 'win32': - _obj = '.obj' -else: - _obj = '.o' +_obj = TestSCons._obj test = TestSCons.TestSCons() diff --git a/test/Repository/StaticLibrary.py b/test/Repository/StaticLibrary.py index e5c76f9..4f8160c 100644 --- a/test/Repository/StaticLibrary.py +++ b/test/Repository/StaticLibrary.py @@ -25,17 +25,10 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path -import sys import TestSCons -if sys.platform == 'win32': - _obj = '.obj' - _exe = '.exe' -else: - _obj = '.o' - _exe = '' - - +_obj = TestSCons._obj +_exe = TestSCons._exe test = TestSCons.TestSCons() diff --git a/test/scons-time/func/file.py b/test/scons-time/func/file.py index fa9d36e..1fe3513 100644 --- a/test/scons-time/func/file.py +++ b/test/scons-time/func/file.py @@ -76,13 +76,13 @@ r"""set title "ST2.CONF TITLE" set key bottom left set label 3 "label 1.5" at 0.5,0.5 right set label 4 "label 1.6" at 0.6,0.4 right -plot '-' title "Startup" with lines lt 1, \ - '-' notitle with lines lt 7, \ - '-' title "label 1.5" with lines lt 7, \ +plot '-' title "Startup" with lines lt 1, \\ + '-' notitle with lines lt 7, \\ + '-' title "label 1.5" with lines lt 7, \\ '-' title "label 1.6" with lines lt 7 # Startup 1 0.000 -2 0.000 +2 0.\d* e 1.4 0 1.4 1 @@ -95,8 +95,9 @@ e e """ -test.run(arguments = 'func --file st2.conf --fmt gnuplot', stdout = expect2) +test.run(arguments = 'func --file st2.conf --fmt gnuplot') +test.must_contain_exactly_lines(test.stdout(), expect2, find=TestSCons.search_re_in_list) test.pass_test() -- cgit v0.12 From 01bc93af6597b256ba2545e248289b82a18e9d5e Mon Sep 17 00:00:00 2001 From: Gary Oberbrunner Date: Sun, 16 Dec 2012 20:30:08 +0000 Subject: Fixed tests/scons-time/file.py. --- QMTest/TestSCons_time.py | 2 ++ test/scons-time/func/file.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/QMTest/TestSCons_time.py b/QMTest/TestSCons_time.py index 4a759c0..abe8ccf 100644 --- a/QMTest/TestSCons_time.py +++ b/QMTest/TestSCons_time.py @@ -21,6 +21,8 @@ import sys from TestCommon import * from TestCommon import __all__ +# some of the scons_time tests may need regex-based matching: +from TestSCons import search_re, search_re_in_list __all__.extend([ 'TestSCons_time', ]) diff --git a/test/scons-time/func/file.py b/test/scons-time/func/file.py index 1fe3513..8e12123 100644 --- a/test/scons-time/func/file.py +++ b/test/scons-time/func/file.py @@ -97,7 +97,7 @@ e test.run(arguments = 'func --file st2.conf --fmt gnuplot') -test.must_contain_exactly_lines(test.stdout(), expect2, find=TestSCons.search_re_in_list) +test.must_contain_exactly_lines(test.stdout(), expect2, find=TestSCons_time.search_re_in_list) test.pass_test() -- cgit v0.12 From 0b692e2a31fad677659ad08cd3241051ebba47b1 Mon Sep 17 00:00:00 2001 From: Gary Oberbrunner Date: Sun, 16 Dec 2012 21:09:30 -0500 Subject: Add -jN support to runtest.py to run tests in parallel --- runtest.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++++-------- src/CHANGES.txt | 1 + 2 files changed, 73 insertions(+), 11 deletions(-) diff --git a/runtest.py b/runtest.py index 3af7766..de36330 100644 --- a/runtest.py +++ b/runtest.py @@ -92,6 +92,14 @@ import stat import sys import time +try: + import threading + import Queue # 2to3: rename to queue + threading_ok = True +except ImportError: + print "Can't import threading or queue" + threading_ok = False + cwd = os.getcwd() all = 0 @@ -101,6 +109,7 @@ external = 0 debug = '' execute_tests = 1 format = None +jobs = 1 list_only = None printcommand = 1 package = None @@ -172,9 +181,10 @@ Environment Variables: TESTCMD_VERBOSE: turn on verbosity in TestCommand """ -opts, args = getopt.getopt(sys.argv[1:], "3ab:def:hklno:P:p:qsv:Xx:t", +opts, args = getopt.getopt(sys.argv[1:], "3ab:def:hj:klno:P:p:qsv:Xx:t", ['all', 'baseline=', 'builddir=', 'debug', 'external', 'file=', 'help', 'no-progress', + 'jobs=', 'list', 'no-exec', 'noqmtest', 'nopipefiles', 'output=', 'package=', 'passed', 'python=', 'qmtest', 'quiet', 'short-progress', 'time', @@ -207,6 +217,8 @@ for o, a in opts: elif o in ['-h', '--help']: print helpstr sys.exit(0) + elif o in ['-j', '--jobs']: + jobs = int(a) elif o in ['-k', '--no-progress']: print_progress = 0 elif o in ['-l', '--list']: @@ -763,7 +775,11 @@ else: total_start_time = time_func() total_num_tests = len(tests) -for idx,t in enumerate(tests): +tests_completed = 0 + +def run_test(t, io_lock, async=True): + global tests_completed + header = "" command_args = ['-tt'] if python3incompatibilities: command_args.append('-3') @@ -774,12 +790,16 @@ for idx,t in enumerate(tests): t.command_str = " ".join([escape(python)] + command_args) if printcommand: if print_progress: - sys.stdout.write("%d/%d (%.2f%s) %s\n" % (idx+1, total_num_tests, - float(idx+1)*100.0/float(total_num_tests), - '%', - t.command_str)) + tests_completed += 1 + n = tests_completed # approx indicator of where we are + header += ("%d/%d (%.2f%s) %s\n" % (n, total_num_tests, + float(n)*100.0/float(total_num_tests), + '%', + t.command_str)) else: - sys.stdout.write(t.command_str + "\n") + header += t.command_str + "\n" + if not suppress_stdout and not suppress_stderr: + sys.stdout.write(header) head, tail = os.path.split(t.abspath) if head: os.environ['PYTHON_SCRIPT_DIR'] = head @@ -788,13 +808,54 @@ for idx,t in enumerate(tests): test_start_time = time_func() if execute_tests: t.execute() - if not suppress_stdout and t.stdout: - print t.stdout - if not suppress_stderr and t.stderr: - print t.stderr t.test_time = time_func() - test_start_time + if io_lock: + io_lock.acquire() + if suppress_stdout or suppress_stderr: + sys.stdout.write(header) + if not suppress_stdout and t.stdout: + print t.stdout + if not suppress_stderr and t.stderr: + print t.stderr print_time_func("Test execution time: %.1f seconds\n", t.test_time) + if io_lock: + io_lock.release() + +class RunTest(threading.Thread): + def __init__(self, queue, io_lock): + threading.Thread.__init__(self) + self.queue = queue + self.io_lock = io_lock + + def run(self): + while True: + t = self.queue.get() + run_test(t, io_lock, True) + self.queue.task_done() + +if jobs > 1 and threading_ok: + print "Running tests using %d jobs"%jobs + # Start worker threads + queue = Queue.Queue() + io_lock = threading.Lock() + for i in range(1, jobs): + t = RunTest(queue, io_lock) + t.daemon = True + t.start() + # Give tasks to workers + for t in tests: + queue.put(t) + queue.join() +else: + # Run tests serially + if jobs > 1: + print "Ignoring -j%d option; no python threading module available."%jobs + for t in tests: + run_test(t, None, False) + +# all tests are complete by the time we get here + if len(tests) > 0: tests[0].total_time = time_func() - total_start_time print_time_func("Total execution time for all tests: %.1f seconds\n", tests[0].total_time) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 6136f81..05a2aa8 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -43,6 +43,7 @@ RELEASE 2.X.X - * removed Aegis support from runtest.py. (#2872) From Gary Oberbrunner: + - Add -jN support to runtest.py to run tests in parallel - Add MSVC10 and MSVC11 support to get_output low-level bat script runner. - Fix MSVS solution generation for VS11, and fixed tests. -- cgit v0.12 From 48b372251b919b28c80d3c2a7ccf0e2eb5a1a960 Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 17 Dec 2012 14:09:32 +0300 Subject: runtest.py: remove --noqmtest option --- runtest.py | 14 +++----------- src/CHANGES.txt | 1 + 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/runtest.py b/runtest.py index e19201c..382f10a 100644 --- a/runtest.py +++ b/runtest.py @@ -117,6 +117,7 @@ python3incompatibilities = None scons = None scons_exec = None outputfile = None +qmtest = None testlistfile = None version = '' print_times = None @@ -141,7 +142,6 @@ Options: -k, --no-progress Suppress count and percent progress messages. -l, --list List available tests and exit. -n, --no-exec No execute, just print command lines. - --noqmtest Execute tests directly, not using QMTest. --nopipefiles Doesn't use the "file pipe" workaround for subprocess.Popen() for starting tests. WARNING: Only use this when too much file traffic is giving you trouble AND you can be sure that none of @@ -160,7 +160,7 @@ Options: tar-gz .tar.gz distribution zip .zip distribution --passed Summarize which tests passed. - --qmtest Run using the QMTest harness. + --qmtest Run using the QMTest harness (deprecated). -q, --quiet Don't print the test being executed. -s, --short-progress Short progress, prints only the command line and a percentage value, based on the total and @@ -212,7 +212,7 @@ opts, args = getopt.getopt(args, "3b:def:hj:klno:P:p:qsv:Xx:t", ['baseline=', 'builddir=', 'debug', 'external', 'file=', 'help', 'no-progress', 'jobs=', - 'list', 'no-exec', 'noqmtest', 'nopipefiles', 'output=', + 'list', 'no-exec', 'nopipefiles', 'output=', 'package=', 'passed', 'python=', 'qmtest', 'quiet', 'short-progress', 'time', 'version=', 'exec=', @@ -250,8 +250,6 @@ for o, a in opts: list_only = 1 elif o in ['-n', '--no-exec']: execute_tests = None - elif o in ['--noqmtest']: - qmtest = None elif o in ['--nopipefiles']: allow_pipe_files = False elif o in ['-o', '--output']: @@ -344,12 +342,6 @@ else: return f return None -# See if --qmtest or --noqmtest specified -try: - qmtest -except NameError: - qmtest = None - sp.append(builddir) sp.append(cwd) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 9084694..6aad0ea 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -12,6 +12,7 @@ RELEASE 2.X.X - - Error messages from option parser now include hints about valid choices - Cleaned up some Python 1.5 and pre-2.3 code, so don't expect SCons to run on anything less than Python 2.4 anymore + - runtest.py: Removed --noqmtest option - this behavior is by default. From Juan Lang: - Fix WiX Tool to use .wixobj rather than .wxiobj for compiler output -- cgit v0.12 From 1aa43a4a975373c032c2fe00a566c2a618d24dbd Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 17 Dec 2012 19:51:00 +0300 Subject: runtest.py: Exit with an error if no tests were found --- runtest.py | 9 +++++++-- src/CHANGES.txt | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/runtest.py b/runtest.py index 382f10a..d193978 100644 --- a/runtest.py +++ b/runtest.py @@ -712,6 +712,12 @@ elif options.all and not qmtest: tests.extend(find_py('test')) tests.sort() +if not tests: + sys.stderr.write("""\ +runtest.py: No tests were found. +""") + sys.exit(1) + if qmtest: if baseline: aegis_result_stream = 'scons_tdb.AegisBaselineStream' @@ -876,8 +882,7 @@ else: for t in tests: run_test(t, None, False) -# all tests are complete by the time we get here - +# --- all tests are complete by the time we get here --- if len(tests) > 0: tests[0].total_time = time_func() - total_start_time print_time_func("Total execution time for all tests: %.1f seconds\n", tests[0].total_time) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 6aad0ea..9f67f3b 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -12,7 +12,9 @@ RELEASE 2.X.X - - Error messages from option parser now include hints about valid choices - Cleaned up some Python 1.5 and pre-2.3 code, so don't expect SCons to run on anything less than Python 2.4 anymore - - runtest.py: Removed --noqmtest option - this behavior is by default. + - Several fixes for runtest.py: + * now exits with an error if no tests were found + * removed --noqmtest option - this behavior is by default From Juan Lang: - Fix WiX Tool to use .wixobj rather than .wxiobj for compiler output -- cgit v0.12 From 7204d98634a9a695a0c2f818ffafb6f79ad1bfab Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Mon, 17 Dec 2012 19:56:01 +0300 Subject: runtest.py: Replace `-o file --xml` with `--xml file` `runtest.py -o file ...` never worked right throwing exception: Traceback (most recent call last): File "...\runtest.py", line 925, in tests[0].header(f) IndexError: list index out of range Leaving -o argument to gather all output into a file (work in process) also allows to capture output and format test results in parallel. --- runtest.py | 46 +++++++++++++++++++--------------------------- src/CHANGES.txt | 3 ++- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/runtest.py b/runtest.py index d193978..5331ba8 100644 --- a/runtest.py +++ b/runtest.py @@ -44,9 +44,7 @@ # # -n No execute, just print command lines. # -# -o file Print test results to the specified file. -# The --xml option specifies the -# output format. +# -o file Save screen output to the specified log file. # # -P Python Use the specified Python interpreter. # @@ -73,8 +71,8 @@ # # -x scons The scons script to use for tests. # -# --xml Print test results to an output file (specified -# by the -o option) in an SCons-specific XML format. +# --xml file Save test results to the specified file in an +# SCons-specific XML format. # This is (will be) used for reporting results back # to a central SCons test monitoring infrastructure. # @@ -107,7 +105,6 @@ builddir = os.path.join(cwd, 'build') external = 0 debug = '' execute_tests = 1 -format = None jobs = 1 list_only = None printcommand = 1 @@ -147,7 +144,7 @@ Options: traffic is giving you trouble AND you can be sure that none of your tests create output that exceed 65K chars! You might run into some deadlocks else. - -o FILE, --output FILE Print test results to FILE. + -o FILE, --output FILE Save the output from a test run to the log file. -P Python Use the specified Python interpreter. -p PACKAGE, --package PACKAGE Test against the specified PACKAGE: @@ -172,7 +169,7 @@ Options: 3 = print commands and all output. -X Test script is executable, don't feed to Python. -x SCRIPT, --exec SCRIPT Test SCRIPT. - --xml Print results in SCons XML format. + --xml file Save results to file in SCons XML format. Environment Variables: @@ -202,6 +199,8 @@ class PassThroughOptionParser(OptionParser): parser = PassThroughOptionParser(add_help_option=False) parser.add_option('-a', '--all', action='store_true', help="Run all tests.") +parser.add_option('--xml', + help="Save results to file in SCons XML format.") (options, args) = parser.parse_args() #print "options:", options @@ -216,7 +215,7 @@ opts, args = getopt.getopt(args, "3b:def:hj:klno:P:p:qsv:Xx:t", 'package=', 'passed', 'python=', 'qmtest', 'quiet', 'short-progress', 'time', 'version=', 'exec=', - 'verbose=', 'xml']) + 'verbose=']) for o, a in opts: if o in ['-3']: @@ -286,8 +285,6 @@ for o, a in opts: scons_exec = 1 elif o in ['-x', '--exec']: scons = a - elif o in ['--xml']: - format = o if not args and not options.all and not testlistfile: sys.stderr.write("""\ @@ -502,12 +499,10 @@ class XML(PopenExecutor): f.write(' \n' % self.total_time) f.write(' \n') -format_class = { - None : SystemExecutor, - '--xml' : XML, -} - -Test = format_class[format] +if options.xml: + Test = XML +else: + Test = SystemExecutor # --- start processing --- if package: @@ -741,12 +736,9 @@ if qmtest: if python: qmtest_args.append('--context python="%s"' % python) - if outputfile: - if format == '--xml': - rsclass = 'scons_tdb.SConsXMLResultStream' - else: - rsclass = 'scons_tdb.AegisBatchStream' - qof = "r'" + outputfile + "'" + if options.xml: + rsclass = 'scons_tdb.SConsXMLResultStream' + qof = "r'" + options.xml + "'" rs = '--result-stream="%s(filename=%s)"' % (rsclass, qof) qmtest_args.append(rs) @@ -914,18 +906,18 @@ if len(tests) != 1 and execute_tests: paths = [x.path for x in no_result] sys.stdout.write("\t" + "\n\t".join(paths) + "\n") -if outputfile: - if outputfile == '-': +if options.xml: + if options.xml == '-': f = sys.stdout else: - f = open(outputfile, 'w') + f = open(options.xml, 'w') tests[0].header(f) #f.write("test_result = [\n") for t in tests: t.write(f) tests[0].footer(f) #f.write("];\n") - if outputfile != '-': + if options.xml != '-': f.close() if len(fail): diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 9f67f3b..736160f 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -13,8 +13,9 @@ RELEASE 2.X.X - - Cleaned up some Python 1.5 and pre-2.3 code, so don't expect SCons to run on anything less than Python 2.4 anymore - Several fixes for runtest.py: - * now exits with an error if no tests were found + * exit with an error if no tests were found * removed --noqmtest option - this behavior is by default + * replaced `-o FILE --xml` combination with `--xml FILE` From Juan Lang: - Fix WiX Tool to use .wixobj rather than .wxiobj for compiler output -- cgit v0.12 From 15d633a7931591f1829bfb9e0d29c79dfb26a1be Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Tue, 18 Dec 2012 13:24:19 +0300 Subject: runtest.py: Fix CRLF --- runtest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtest.py b/runtest.py index 5331ba8..4c12298 100644 --- a/runtest.py +++ b/runtest.py @@ -199,8 +199,8 @@ class PassThroughOptionParser(OptionParser): parser = PassThroughOptionParser(add_help_option=False) parser.add_option('-a', '--all', action='store_true', help="Run all tests.") -parser.add_option('--xml', - help="Save results to file in SCons XML format.") +parser.add_option('--xml', + help="Save results to file in SCons XML format.") (options, args) = parser.parse_args() #print "options:", options -- cgit v0.12 From 02cae725ed116ae6cd996fe3bf5888f7f8f61b97 Mon Sep 17 00:00:00 2001 From: anatoly techtonik Date: Tue, 18 Dec 2012 14:40:36 +0300 Subject: runtest.py: Enable -o, --output to save stdout/stderr into a file --- runtest.py | 22 +++++++++++++++------- src/CHANGES.txt | 2 ++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/runtest.py b/runtest.py index 4c12298..6beb4ba 100644 --- a/runtest.py +++ b/runtest.py @@ -113,7 +113,6 @@ print_passed_summary = None python3incompatibilities = None scons = None scons_exec = None -outputfile = None qmtest = None testlistfile = None version = '' @@ -199,6 +198,8 @@ class PassThroughOptionParser(OptionParser): parser = PassThroughOptionParser(add_help_option=False) parser.add_option('-a', '--all', action='store_true', help="Run all tests.") +parser.add_option('-o', '--output', + help="Save the output from a test run to the log file.") parser.add_option('--xml', help="Save results to file in SCons XML format.") (options, args) = parser.parse_args() @@ -207,11 +208,11 @@ parser.add_option('--xml', #print "args:", args -opts, args = getopt.getopt(args, "3b:def:hj:klno:P:p:qsv:Xx:t", +opts, args = getopt.getopt(args, "3b:def:hj:klnP:p:qsv:Xx:t", ['baseline=', 'builddir=', 'debug', 'external', 'file=', 'help', 'no-progress', 'jobs=', - 'list', 'no-exec', 'nopipefiles', 'output=', + 'list', 'no-exec', 'nopipefiles', 'package=', 'passed', 'python=', 'qmtest', 'quiet', 'short-progress', 'time', 'version=', 'exec=', @@ -251,10 +252,6 @@ for o, a in opts: execute_tests = None elif o in ['--nopipefiles']: allow_pipe_files = False - elif o in ['-o', '--output']: - if a != '-' and not os.path.isabs(a): - a = os.path.join(cwd, a) - outputfile = a elif o in ['-p', '--package']: package = a elif o in ['--passed']: @@ -311,6 +308,17 @@ class Unbuffered(object): sys.stdout = Unbuffered(sys.stdout) sys.stderr = Unbuffered(sys.stderr) +if options.output: + logfile = open(options.output, 'w') + class Tee(object): + def __init__(self, openfile, stream): + self.file = openfile + self.stream = stream + def write(self, data): + self.file.write(data) + self.stream.write(data) + sys.stdout = Tee(logfile, sys.stdout) + sys.stderr = Tee(logfile, sys.stderr) # --- define helpers ---- if sys.platform in ('win32', 'cygwin'): diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 736160f..e8aa5ee 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -16,6 +16,8 @@ RELEASE 2.X.X - * exit with an error if no tests were found * removed --noqmtest option - this behavior is by default * replaced `-o FILE --xml` combination with `--xml FILE` + * changed `-o, --output FILE` option to capture stdout/stderr output + from runtest.py From Juan Lang: - Fix WiX Tool to use .wixobj rather than .wxiobj for compiler output -- cgit v0.12