Enhance runtest.py and add a script for automated regression-test runs.

4 files changed, 461 insertions, 51 deletions

diff --git a/bin/scons-test.py b/bin/scons-test.py
new file mode 100644
index 0000000..ea75607
--- /dev/null
+++ b/bin/scons-test.py
@@ -0,0 +1,241 @@
+#!/usr/bin/env python
+#
+# A script that takes an scons-src-{version}.zip file, unwraps it in
+# a temporary location, and calls runtest.py to execute one or more of
+# its tests.
+#
+# The default is to download the latest scons-src archive from the SCons
+# web site, and to execute all of the tests.
+#
+# With a little more work, this will become the basis of an automated
+# testing and reporting system that anyone will be able to use to
+# participate in testing SCons on their system and regularly reporting
+# back the results.  A --xml option is a stab at gathering a lot of
+# relevant information about the system, the Python version, etc.,
+# so that problems on different platforms can be identified sooner.
+#
+
+import getopt
+import imp
+import os
+import os.path
+import string
+import sys
+import tempfile
+import time
+import urllib
+import zipfile
+
+helpstr = """\
+Usage: scons-test.py [-f zipfile] [-o outdir] [-v] [--xml] [runtest arguments]
+Options:
+  -f FILE                     Specify input .zip FILE name
+  -o DIR, --out DIR           Change output directory name to DIR
+  -v, --verbose               Print file names when extracting
+  --xml                       XML output
+"""
+
+opts, args = getopt.getopt(sys.argv[1:],
+                           "f:o:v",
+                           ['file=', 'out=', 'verbose', 'xml'])
+
+format = None
+outdir = None
+printname = lambda x: x
+inputfile = 'http://scons.sourceforge.net/scons-src-latest.zip'
+
+for o, a in opts:
+    if o == '-f' or o == '--file':
+        inputfile = a
+    elif o == '-o' or o == '--out':
+        outdir = a
+    elif o == '-v' or o == '--verbose':
+        def printname(x):
+            print x
+    elif o == '--xml':
+        format = o
+
+startdir = os.getcwd()
+
+tempfile.template = 'scons-test.'
+tempdir = tempfile.mktemp()
+
+if not os.path.exists(tempdir):
+    os.mkdir(tempdir)
+    def cleanup(tempdir=tempdir):
+        import shutil
+        os.chdir(startdir)
+        shutil.rmtree(tempdir)
+    sys.exitfunc = cleanup
+
+# Fetch the input file if it happens to be across a network somewhere.
+# Ohmigod, does Python make this simple...
+inputfile, headers = urllib.urlretrieve(inputfile)
+
+# Unzip the header file in the output directory.  We use our own code
+# (lifted from scons-unzip.py) to make the output subdirectory name
+# match the basename of the .zip file.
+zf = zipfile.ZipFile(inputfile, 'r')
+
+if outdir is None:
+    name, _ = os.path.splitext(os.path.basename(inputfile))
+    outdir = os.path.join(tempdir, name)
+
+def outname(n, outdir=outdir):
+    l = []
+    while 1:
+        n, tail = os.path.split(n)
+        if not n:
+            break
+        l.append(tail)
+    l.append(outdir)
+    l.reverse()
+    return apply(os.path.join, l)
+
+for name in zf.namelist():
+    dest = outname(name)
+    dir = os.path.dirname(dest)
+    try:
+        os.makedirs(dir)
+    except:
+        pass
+    printname(dest)
+    # if the file exists, then delete it before writing
+    # to it so that we don't end up trying to write to a symlink:
+    if os.path.isfile(dest) or os.path.islink(dest):
+        os.unlink(dest)
+    if not os.path.isdir(dest):
+        open(dest, 'w').write(zf.read(name))
+
+os.chdir(outdir)
+
+# Load (by hand) the SCons modules we just unwrapped so we can
+# extract their version information.  Note that we have to override
+# SCons.Script.main() with a do_nothing() function, because loading up
+# the 'scons' script will actually try to execute SCons...
+src_script = os.path.join(outdir, 'src', 'script')
+src_engine = os.path.join(outdir, 'src', 'engine')
+src_engine_SCons = os.path.join(src_engine, 'SCons')
+
+fp, pname, desc = imp.find_module('SCons', [src_engine])
+SCons = imp.load_module('SCons', fp, pname, desc)
+
+fp, pname, desc = imp.find_module('Script', [src_engine_SCons])
+SCons.Script = imp.load_module('Script', fp, pname, desc)
+
+def do_nothing():
+    pass
+SCons.Script.main = do_nothing
+
+fp, pname, desc = imp.find_module('scons', [src_script])
+scons = imp.load_module('scons', fp, pname, desc)
+fp.close()
+
+# Default is to run all the tests by passing the -a flags to runtest.py.
+if not args:
+    runtest_args = '-a'
+else:
+    runtest_args = string.join(args)
+
+if format == '--xml':
+
+    print "<scons_test_run>"
+    print "  <sys>"
+    sys_keys = ['byteorder', 'exec_prefix', 'executable', 'maxint', 'maxunicode', 'platform', 'prefix', 'version', 'version_info']
+    for k in sys_keys:
+        print "    <%s>%s</%s>" % (k, sys.__dict__[k], k)
+    print "  </sys>"
+
+    fmt = '%a %b %d %H:%M:%S %Y'
+    print "  <time>"
+    print "    <gmtime>%s</gmtime>" % time.strftime(fmt, time.gmtime())
+    print "    <localtime>%s</localtime>" % time.strftime(fmt, time.localtime())
+    print "  </time>"
+
+    print "  <tempdir>%s</tempdir>" % tempdir
+
+    def print_version_info(tag, module):
+        print "    <%s>" % tag
+        print "      <version>%s</version>" % module.__version__
+        print "      <build>%s</build>" % module.__build__
+        print "      <buildsys>%s</buildsys>" % module.__buildsys__
+        print "      <date>%s</date>" % module.__date__
+        print "      <developer>%s</developer>" % module.__developer__
+        print "    </%s>" % tag
+
+    print "  <scons>"
+    print_version_info("script", scons)
+    print_version_info("engine", SCons)
+    print "  </scons>"
+
+    environ_keys = [
+        'PATH',
+        'SCONSFLAGS',
+        'SCONS_LIB_DIR',
+        'PYTHON_ROOT',
+        'QTDIR',
+
+        'COMSPEC',
+        'INTEL_LICENSE_FILE',
+        'INCLUDE',
+        'LIB',
+        'MSDEVDIR',
+        'OS',
+        'PATHEXT',
+        'SYSTEMROOT',
+        'TEMP',
+        'TMP',
+        'USERNAME',
+        'VXDOMNTOOLS',
+        'WINDIR',
+        'XYZZY'
+
+        'ENV',
+        'HOME',
+        'LANG',
+        'LANGUAGE',
+        'LOGNAME',
+        'MACHINE',
+        'OLDPWD',
+        'PWD',
+        'OPSYS',
+        'SHELL',
+        'TMPDIR',
+        'USER',
+    ]
+
+    print "  <environment>"
+    #keys = os.environ.keys()
+    keys = environ_keys
+    keys.sort()
+    for key in keys:
+        value = os.environ.get(key)
+        if value:
+            print "    <variable>"
+            print "      <name>%s</name>" % key
+            print "      <value>%s</value>" % value
+            print "    </variable>"
+    print "  </environment>"
+
+    command = '"%s" runtest.py -q -o - --xml %s' % (sys.executable, runtest_args)
+    #print command
+    os.system(command)
+    print "</scons_test_run>"
+
+else:
+
+    def print_version_info(tag, module):
+        print "\t%s: v%s.%s, %s, by %s on %s" % (tag,
+                                                 module.__version__,
+                                                 module.__build__,
+                                                 module.__date__,
+                                                 module.__developer__,
+                                                 module.__buildsys__)
+
+    print "SCons by Steven Knight et al.:"
+    print_version_info("script", scons)
+    print_version_info("engine", SCons)
+
+    command = '"%s" runtest.py %s' % (sys.executable, runtest_args)
+    #print command
+    os.system(command)
diff --git a/bin/scons-unzip.py b/bin/scons-unzip.py
new file mode 100644
index 0000000..f772dd5
--- /dev/null
+++ b/bin/scons-unzip.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+#
+# A quick script to unzip a .zip archive and put the files in a
+# subdirectory that matches the basename of the .zip file.
+#
+# This is actually generic functionality, it's not SCons-specific, but
+# I'm using this to make it more convenient to manage working on multiple
+# changes on Windows, where I don't have access to my Aegis tools.
+#
+
+import getopt
+import os.path
+import sys
+import zipfile
+
+helpstr = """\
+Usage: scons-unzip.py [-o outdir] zipfile
+Options:
+  -o DIR, --out DIR           Change output directory name to DIR
+  -v, --verbose               Print file names when extracting
+"""
+
+opts, args = getopt.getopt(sys.argv[1:],
+                           "o:v",
+                           ['out=', 'verbose'])
+
+outdir = None
+printname = lambda x: x
+
+for o, a in opts:
+    if o == '-o' or o == '--out':
+        outdir = a
+    elif o == '-v' or o == '--verbose':
+        def printname(x):
+            print x
+
+if len(args) != 1:
+    sys.stderr.write("scons-unzip.py:  \n")
+    sys.exit(1)
+
+zf = zipfile.ZipFile(str(args[0]), 'r')
+
+if outdir is None:
+    outdir, _ = os.path.splitext(os.path.basename(args[0]))
+
+def outname(n, outdir=outdir):
+    l = []
+    while 1:
+        n, tail = os.path.split(n)
+        if not n:
+            break
+        l.append(tail)
+    l.append(outdir)
+    l.reverse()
+    return apply(os.path.join, l)
+
+for name in zf.namelist():
+    dest = outname(name)
+    dir = os.path.dirname(dest)
+    try:
+        os.makedirs(dir)
+    except:
+        pass
+    printname(dest)
+    # if the file exists, then delete it before writing
+    # to it so that we don't end up trying to write to a symlink:
+    if os.path.isfile(dest) or os.path.islink(dest):
+        os.unlink(dest)
+    if not os.path.isdir(dest):
+        open(dest, 'w').write(zf.read(name))
diff --git a/config b/config
index 8df5dba..17aa2d3 100644
--- a/config
+++ b/config
@@ -260,7 +260,7 @@ diff_command =
  */
 test_command = "python1.5 ${Source runtest.py Absolute} -p tar-gz -v ${SUBSTitute '\\.[CD][0-9]+$' '' ${VERsion}} -q ${File_Name}";
 
-batch_test_command = "python1.5 ${Source runtest.py Absolute} -p tar-gz -v ${SUBSTitute '\\.[CD][0-9]+$' '' ${VERsion}} -o ${Output} ${File_Names} ${COMment $spe}";
+batch_test_command = "python1.5 ${Source runtest.py Absolute} -p tar-gz -v ${SUBSTitute '\\.[CD][0-9]+$' '' ${VERsion}} -o ${Output} --aegis ${File_Names} ${COMment $spe}";
 
 new_test_filename = "test/CHANGETHIS.py";
 
diff --git a/runtest.py b/runtest.py
index 7b08b08..a93789d 100644
--- a/runtest.py
+++ b/runtest.py
@@ -22,33 +22,50 @@
 #
 # Options:
 #
-#	-a		Run all tests; does a virtual 'find' for
-#			all SCons tests under the current directory.
+#       -a              Run all tests; does a virtual 'find' for
+#                       all SCons tests under the current directory.
 #
-#	-d		Debug.  Runs the script under the Python
-#			debugger (pdb.py) so you don't have to
-#			muck with PYTHONPATH yourself.
+#       --aegis         Print test results to an output file (specified
+#                       by the -o option) in the format expected by
+#                       aetest(5).  This is intended for use in the
+#                       batch_test_command field in the Aegis project
+#                       config file.
+#
+#       -d              Debug.  Runs the script under the Python
+#                       debugger (pdb.py) so you don't have to
+#                       muck with PYTHONPATH yourself.
+#
+#       -f file         Only execute the tests listed in the specified
+#                       file.
 #
 #       -h              Print the help and exit.
 #
-#       -o file         Print test results to the specified file
-#                       in the format expected by aetest(5).  This
-#                       is intended for use in the batch_test_command
-#                       field in the Aegis project config file.
+#       -o file         Print test results to the specified file.
+#                       The --aegis and --xml options specify the
+#                       output format.
+#
+#       -P Python       Use the specified Python interpreter.
 #
-#	-P Python	Use the specified Python interpreter.
+#       -p package      Test against the specified package.
 #
-#	-p package	Test against the specified package.
+#       --passed        In the final summary, also report which tests
+#                       passed.  The default is to only report tests
+#                       which failed or returned NO RESULT.
 #
-#	-q		Quiet.  By default, runtest.py prints the
-#			command line it will execute before
-#			executing it.  This suppresses that print.
+#       -q              Quiet.  By default, runtest.py prints the
+#                       command line it will execute before
+#                       executing it.  This suppresses that print.
 #
-#	-X		The scons "script" is an executable; don't
-#			feed it to Python.
+#       -X              The scons "script" is an executable; don't
+#                       feed it to Python.
 #
 #       -x scons        The scons script to use for tests.
 #
+#       --xml           Print test results to an output file (specified
+#                       by the -o option) in an SCons-specific XML format.
+#                       This is (will be) used for reporting results back
+#                       to a central SCons test monitoring infrastructure.
+#
 # (Note:  There used to be a -v option that specified the SCons
 # version to be tested, when we were installing in a version-specific
 # library directory.  If we ever resurrect that as the default, then
@@ -60,6 +77,7 @@ import getopt
 import glob
 import os
 import os.path
+import popen2
 import re
 import stat
 import string
@@ -67,12 +85,14 @@ import sys
 
 all = 0
 debug = ''
+format = None
 tests = []
-printcmd = 1
+printcommand = 1
 package = None
+print_passed_summary = None
 scons = None
 scons_exec = None
-output = None
+outputfile = None
 testlistfile = None
 version = ''
 
@@ -94,10 +114,11 @@ helpstr = """\
 Usage: runtest.py [OPTIONS] [TEST ...]
 Options:
   -a, --all                   Run all tests.
+  --aegis                     Print results in Aegis format.
   -d, --debug                 Run test scripts under the Python debugger.
   -f FILE, --file FILE        Run tests in specified FILE.
   -h, --help                  Print this message and exit.
-  -o FILE, --output FILE      Print test results to FILE (Aegis format).
+  -o FILE, --output FILE      Print test results to FILE.
   -P Python                   Use the specified Python interpreter.
   -p PACKAGE, --package PACKAGE
                               Test against the specified PACKAGE:
@@ -109,16 +130,20 @@ Options:
                                 src-zip       .zip source package
                                 tar-gz        .tar.gz distribution
                                 zip           .zip distribution
+  --passed                    Summarize which tests passed.
   -q, --quiet                 Don't print the test being executed.
   -v version                  Specify the SCons version.
   -X                          Test script is executable, don't feed to Python.
   -x SCRIPT, --exec SCRIPT    Test SCRIPT.
+  --xml                       Print results in SCons XML format.
 """
 
 opts, args = getopt.getopt(sys.argv[1:], "adf:ho:P:p:qv:Xx:",
-                            ['all', 'debug', 'file=', 'help', 'output=',
-                             'package=', 'python=', 'quiet',
-                             'version=', 'exec='])
+                            ['all', 'aegis',
+                             'debug', 'file=', 'help', 'output=',
+                             'package=', 'passed', 'python=', 'quiet',
+                             'version=', 'exec=',
+                             'xml'])
 
 for o, a in opts:
     if o == '-a' or o == '--all':
@@ -133,21 +158,25 @@ for o, a in opts:
         print helpstr
         sys.exit(0)
     elif o == '-o' or o == '--output':
-        if not os.path.isabs(a):
+        if a != '-' and not os.path.isabs(a):
             a = os.path.join(cwd, a)
-        output = a
-    elif o == '-P' or o == '--python':
-        python = a
+        outputfile = a
     elif o == '-p' or o == '--package':
         package = a
+    elif o == '--passed':
+        print_passed_summary = 1
+    elif o == '-P' or o == '--python':
+        python = a
     elif o == '-q' or o == '--quiet':
-        printcmd = 0
+        printcommand = 0
     elif o == '-v' or o == '--version':
         version = a
     elif o == '-X':
         scons_exec = 1
     elif o == '-x' or o == '--exec':
         scons = a
+    elif o in ['--aegis', '--xml']:
+        format = o
 
 def whereis(file):
     for dir in string.split(os.environ['PATH'], os.pathsep):
@@ -175,7 +204,7 @@ else:
 
 sp.append(cwd)
 
-class Test:
+class Base:
     def __init__(self, path, spe=None):
         self.path = path
         self.abspath = os.path.abspath(path)
@@ -187,6 +216,67 @@ class Test:
                     break
         self.status = None
 
+class SystemExecutor(Base):
+    def execute(self):
+        s = os.system(self.command)
+        if s >= 256:
+            s = s / 256
+        self.status = s
+        if s < 0 or s > 2:
+            sys.stdout.write("Unexpected exit status %d\n" % s)
+
+try:
+    popen2.Popen3
+except AttributeError:
+    class PopenExecutor(Base):
+        def execute(self):
+            (tochild, fromchild, childerr) = os.popen3(self.command)
+            tochild.close()
+            self.stdout = fromchild.read()
+            self.stderr = childerr.read()
+            fromchild.close()
+            self.status = childerr.close()
+            if not self.status:
+                self.status = 0
+else:
+    class PopenExecutor(Base):
+        def execute(self):
+            p = popen2.Popen3(self.command, 1)
+            p.tochild.close()
+            self.stdout = p.fromchild.read()
+            self.stderr = p.childerr.read()
+            self.status = p.wait()
+
+class Aegis(SystemExecutor):
+    def header(self, f):
+        f.write('test_result = [\n')
+    def write(self, f):
+        f.write('    { file_name = "%s";\n' % self.path)
+        f.write('      exit_status = %d; },\n' % self.status)
+    def footer(self, f):
+        f.write('];\n')
+
+class XML(PopenExecutor):
+    def header(self, f):
+        f.write('  <results>\n')
+    def write(self, f):
+        f.write('    <test>\n')
+        f.write('      <file_name>%s</file_name>\n' % self.path)
+        f.write('      <command_line>%s</command_line>\n' % self.command)
+        f.write('      <exit_status>%s</exit_status>\n' % self.status)
+        f.write('      <stdout>%s</stdout>\n' % self.stdout)
+        f.write('      <stderr>%s</stderr>\n' % self.stderr)
+        f.write('    </test>\n')
+    def footer(self, f):
+        f.write('  </results>\n')
+
+format_class = {
+    None        : SystemExecutor,
+    '--aegis'   : Aegis,
+    '--xml'     : XML,
+}
+Test = format_class[format]
+
 if args:
     if spe:
         for a in args:
@@ -370,20 +460,23 @@ def escape(s):
     return s
 
 for t in tests:
-    cmd = string.join(map(escape, [python, debug, t.abspath]), " ")
-    if printcmd:
-        sys.stdout.write(cmd + "\n")
-    s = os.system(cmd)
-    if s >= 256:
-        s = s / 256
-    t.status = s
-    if s < 0 or s > 2:
-        sys.stdout.write("Unexpected exit status %d\n" % s)
+    t.command = string.join(map(escape, [python, debug, t.abspath]), " ")
+    if printcommand:
+        sys.stdout.write(t.command + "\n")
+    t.execute()
 
+passed = filter(lambda t: t.status == 0, tests)
 fail = filter(lambda t: t.status == 1, tests)
 no_result = filter(lambda t: t.status == 2, tests)
 
 if len(tests) != 1:
+    if passed and print_passed_summary:
+        if len(passed) == 1:
+            sys.stdout.write("\nPassed the following test:\n")
+        else:
+            sys.stdout.write("\nPassed the following %d tests:\n" % len(passed))
+        paths = map(lambda x: x.path, passed)
+        sys.stdout.write("\t" + string.join(paths, "\n\t") + "\n")
     if fail:
         if len(fail) == 1:
             sys.stdout.write("\nFailed the following test:\n")
@@ -399,19 +492,25 @@ if len(tests) != 1:
         paths = map(lambda x: x.path, no_result)
         sys.stdout.write("\t" + string.join(paths, "\n\t") + "\n")
 
-if output:
-    f = open(output, 'w')
-    f.write("test_result = [\n")
+if outputfile:
+    if outputfile == '-':
+        f = sys.stdout
+    else:
+        f = open(outputfile, 'w')
+    tests[0].header(f)
+    #f.write("test_result = [\n")
     for t in tests:
-        f.write('    { file_name = "%s";\n' % t.path)
-        f.write('      exit_status = %d; },\n' % t.status)
-    f.write("];\n")
-    f.close()
+        t.write(f)
+    tests[0].footer(f)
+    #f.write("];\n")
+    if outputfile != '-':
+        f.close()
+
+if format == '--aegis':
     sys.exit(0)
+elif len(fail):
+    sys.exit(1)
+elif len(no_result):
+    sys.exit(2)
 else:
-    if len(fail):
-        sys.exit(1)
-    elif len(no_result):
-        sys.exit(2)
-    else:
-        sys.exit(0)
+    sys.exit(0)