Capture initial infrastructure for working performance tests.

10 files changed, 225 insertions, 136 deletions

diff --git a/QMTest/TestSCons.py b/QMTest/TestSCons.py
index a2bf8fc..1fed39a 100644
--- a/QMTest/TestSCons.py
+++ b/QMTest/TestSCons.py
@@ -18,6 +18,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 import os
 import re
+import shutil
 import string
 import sys
 import time
@@ -942,6 +943,129 @@ print py_ver
         else:
             alt_cpp_suffix = '.C'
         return alt_cpp_suffix
+
+
+class TimeSCons(TestSCons):
+    """Class for timing SCons."""
+    def __init__(self, *args, **kw):
+        """
+        In addition to normal TestSCons.TestSCons intialization,
+        this enables verbose mode (which causes the command lines to
+        be displayed in the output) and copies the contents of the
+        directory containing the executing script to the temporary
+        working directory.
+        """
+        if not kw.has_key('verbose'):
+            kw['verbose'] = True
+        TestSCons.__init__(self, *args, **kw)
+
+        # TODO(sgk):    better way to get the script dir than sys.argv[0]
+        test_dir = os.path.dirname(sys.argv[0])
+        test_name = os.path.basename(test_dir)
+
+        if not os.path.isabs(test_dir):
+            test_dir = os.path.join(self.orig_cwd, test_dir)
+        self.copy_timing_configuration(test_dir, self.workpath())
+
+    def main(self, *args, **kw):
+        """
+        The main entry point for standard execution of timings.
+
+        This method run SCons three times:
+
+          Once with the --help option, to have it exit after just reading
+          the configuration.
+
+          Once as a full build of all targets.
+
+          Once again as a (presumably) null or up-to-date build of
+          all targets.
+
+        The elapsed time to execute each build is printed after
+        it has finished.
+        """
+        self.help(*args, **kw)
+        self.full(*args, **kw)
+        self.null(*args, **kw)
+
+    def help(self, *args, **kw):
+        """
+        Runs scons with the --help option.
+
+        This serves as a way to isolate just the amount of time spent
+        reading up the configuration, since --help exits before any
+        "real work" is done.
+        """
+        kw['options'] = kw.get('options', '') + ' --help'
+        self.run_build(*args, **kw)
+        sys.stdout.write(self.stdout())
+        print "RESULT", self.elapsed_time()
+
+    def full(self, *args, **kw):
+        """
+        Runs a full build of SCons.
+        """
+        self.run_build(*args, **kw)
+        sys.stdout.write(self.stdout())
+        print "RESULT", self.elapsed_time()
+
+    def null(self, *args, **kw):
+        """
+        Runs an up-to-date null build of SCons.
+        """
+        # TODO(sgk):  allow the caller to specify the target (argument)
+        # that must be up-to-date.
+        self.up_to_date(arguments='.', **kw)
+        sys.stdout.write(self.stdout())
+        print "RESULT", self.elapsed_time()
+
+    def elapsed_time(self):
+        """
+        Returns the elapsed time of the most recent command execution.
+        """
+        return self.endTime - self.startTime
+
+    def run_build(self, *args, **kw):
+        """
+        Runs a single build command, capturing output in the specified file.
+
+        Because this class is about timing SCons, we record the start
+        and end times of the elapsed execution, and also add the
+        --debug=memory and --debug=time options to have SCons report
+        its own memory and timing statistics.
+        """
+        kw['options'] = kw.get('options', '') + ' --debug=memory --debug=time'
+        self.startTime = time.time()
+        try:
+            result = TestSCons.run(self, *args, **kw)
+        finally:
+            self.endTime = time.time()
+        return result
+
+    def copy_timing_configuration(self, source_dir, dest_dir):
+        """
+        Copies the timing configuration from the specified source_dir (the
+        directory in which the controlling script lives) to the specified
+        dest_dir (a temporary working directory).
+
+        This ignores all files and directories that begin with the string
+        'TimeSCons-', and all '.svn' subdirectories.
+        """
+        for root, dirs, files in os.walk(source_dir):
+            if '.svn' in dirs:
+                dirs.remove('.svn')
+            dirs = [ d for d in dirs if not d.startswith('TimeSCons-') ]
+            files = [ f for f in files if not f.startswith('TimeSCons-') ]
+            for dirname in dirs:
+                source = os.path.join(root, dirname)
+                destination = source.replace(source_dir, dest_dir)
+                os.mkdir(destination)
+                if sys.platform != 'win32':
+                    shutil.copystat(source, destination)
+            for filename in files:
+                source = os.path.join(root, filename)
+                destination = source.replace(source_dir, dest_dir)
+                shutil.copy2(source, destination)
     
 
 # In some environments, $AR will generate a warning message to stderr
diff --git a/timings/CPPPATH/SConstruct b/timings/CPPPATH/SConstruct
index 728db9c..d49e3d6 100644
--- a/timings/CPPPATH/SConstruct
+++ b/timings/CPPPATH/SConstruct
@@ -21,45 +21,10 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 
-"""
-This configuration is for testing the timing of searching long lists of
-CPPPATH directories.
+dir_count = int(ARGUMENTS['DIR_COUNT'])
 
-We create 100 on-disk directories, with a single .h file in the last
-directory in the list.  We set CPPPATH to a list of Dir Nodes for the
-created directories.  The .c file we create #includes the .h file to be
-found in the last directory in the list.
-"""
-
-import os
-import os.path
-
-dir_cnt = 100
-
-dir_list = map(lambda t: 'inc_%03d' % t, xrange(dir_cnt))
-
-for dir in dir_list:
-    if not os.path.isdir(dir):
-        os.mkdir(dir)
-
-foo_h = 'inc_099/foo.h'
-
-if not os.path.isfile(foo_h):
-    open(foo_h, 'w').write('#define     FOO     1\n')
-
-contents = """\
-#include "foo.h"
-void
-foo(void)
-{
-    ;
-}
-"""
-
-if not os.path.isfile('foo.c'):
-    open('foo.c', 'w').write(contents)
-
-inc_list = map(lambda d: Dir(d), dir_list)
+inc_list = [ Dir('inc_%04d' % t) for t in xrange(dir_count) ]
+inc_list.append(Dir('include'))
 
 env = Environment(CPPPATH = inc_list)
 
diff --git a/timings/CPPPATH/st.conf b/timings/CPPPATH/TimeSCons-run.py
index 6507ea2..d88042e 100644
--- a/timings/CPPPATH/st.conf
+++ b/timings/CPPPATH/TimeSCons-run.py
@@ -19,26 +19,26 @@
 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
 
 """
-scons-time.py configuration file for the "CPPPATH" timing test.
+This configuration times searching long lists of CPPPATH directories.
+
+We create 5000 on-disk directories.  A single checked-in .h file exists
+in the 'include' directory.  The SConstruct sets CPPPATH to a list of Dir
+Nodes for the created directories, followed by 'include'.  A checked-in .c
+file #includes the .h file to be found in the last directory in the list.
 """
 
-archive_list = [ 'SConstruct' ]
-subdir = '.'
+import TestSCons
+
+test = TestSCons.TimeSCons()
+
+dir_count = 5000
 
-import sys
-sys.path.insert(0, '..')
-import SCons_Bars
+for d in xrange(dir_count):
+    test.subdir('inc_%04d' % d)
 
-revs = [
-    1224,   # Don't create a Node for every file we try to find during scan.
-    1349,   # More efficient checking for on-disk file entries.
-    1407,   # Use a Dir scanner instead of a hard-coded method.
-    1433,   # Remove unnecessary creation of RCS and SCCS Node.Dir nodes.
-    1703,   # Lobotomize Memoizer.
-    2380,   # The Big Signature Refactoring hits branches/core.
-]
+test.main(options='DIR_COUNT=%s' % dir_count)
 
-vertical_bars = SCons_Bars.Release_Bars.gnuplot(labels=True) + \
-                SCons_Bars.Revision_Bars.gnuplot(labels=False, revs=revs)
+test.pass_test()
diff --git a/timings/CPPPATH/foo.c b/timings/CPPPATH/foo.c
new file mode 100644
index 0000000..3cbb036
--- /dev/null
+++ b/timings/CPPPATH/foo.c
@@ -0,0 +1,6 @@
+#include "foo.h"
+void
+foo(void)
+{
+    ;
+}
diff --git a/timings/CPPPATH/include/foo.h b/timings/CPPPATH/include/foo.h
new file mode 100644
index 0000000..b1e1311
--- /dev/null
+++ b/timings/CPPPATH/include/foo.h
@@ -0,0 +1 @@
+#define     FOO     1
diff --git a/timings/JTimer/SConstruct b/timings/JTimer/SConstruct
index e1e38d2..95764a6 100644
--- a/timings/JTimer/SConstruct
+++ b/timings/JTimer/SConstruct
@@ -21,33 +21,18 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 
-"""
-This configuration is for timing how we evaluate long chains of
-dependencies, specifically when -j is used.
-
-We set up a chain of 100 targets that get built from a Python function
-action with no source files (equivalent to "echo junk > $TARGET").
-Each target explicitly depends on the next target in turn, so the
-Taskmaster will do a deep walk of the dependency graph.
-
-This test case was contributed by Kevin Massey.  Prior to revision 1468,
-we had a serious O(N^2) problem in the Taskmaster when handling long
-dependency chains like this.  That was fixed by adding reference counts
-to the Taskmaster so it could be smarter about not re-evaluating Nodes.
-"""
-
-target_cnt = 100
+target_count = int(ARGUMENTS['TARGET_COUNT'])
 
 env = Environment()
 
 def write_file( env, target, source ):
-    path_target = env.File( target ).path
+    path_target = env.File( target )[0].path
     outfile = open( path_target, 'w' )
     outfile.write( 'junk' )
     outfile.close()
 
 list = []
-for i in range( target_cnt ):
+for i in range( target_count ):
     target = 'target_%03d' % i
     env.Command( target, [], write_file )
     env.Depends( target, list )
diff --git a/timings/hundred/st.conf b/timings/JTimer/TimeSCons-run.py
index adfb09e..36b016f 100644
--- a/timings/hundred/st.conf
+++ b/timings/JTimer/TimeSCons-run.py
@@ -22,26 +22,22 @@
 #
 
 """
-scons-time.py configuration file for the "hundred" timing test.
-"""
+This configuration is for timing how we evaluate long chains of
+dependencies, specifically when -j is used.
+
+We set up a chain of 500 targets that get built from a Python function
+action with no source files (equivalent to "echo junk > $TARGET").
+Each target explicitly depends on the next target in turn, so the
+Taskmaster will do a deep walk of the dependency graph.
 
-archive_list = [ 'SConstruct' ]
-subdir = '.'
+This test case was contributed by Kevin Massey.  Prior to revision 1468,
+we had a serious O(N^2) problem in the Taskmaster when handling long
+dependency chains like this.  That was fixed by adding reference counts
+to the Taskmaster so it could be smarter about not re-evaluating Nodes.
+"""
 
-import sys
-sys.path.insert(0, '..')
-import SCons_Bars
+import TestSCons
 
-revs = [
-    1220,   # Use WeakValueDicts in the Memoizer to reduce memory use.
-    1307,   # Move signature Node tranlation of rel_paths into the class.
-    1435,   # Fix Debug.caller() directory separators.
-    1477,   # Delay disambiguation of Node.FS.Entry into File/Dir.
-    1655,   # Reduce unnecessary calls to Node.FS.disambiguate().
-    1703,   # Lobotomize Memoizer.
-    1727,   # Cache Executor methods, reduce calls when scanning.
-    2380,   # The Big Signature Refactoring hits branches/core.
-]
+target_count = 500
 
-vertical_bars = SCons_Bars.Release_Bars.gnuplot(labels=True) + \
-                SCons_Bars.Revision_Bars.gnuplot(labels=False, revs=revs)
+TestSCons.TimeSCons().main(options='TARGET_COUNT=%d' % target_count)
diff --git a/timings/README.txt b/timings/README.txt
new file mode 100644
index 0000000..840c7f8
--- /dev/null
+++ b/timings/README.txt
@@ -0,0 +1,32 @@
+This directory contains timing configurations for SCons.
+
+Each configuration exists in a subdirectory.  The controlling script
+is named TimeSCons-run.py for the configuration.  The TimeSCons-run.py
+scripts use TestSCons.TimeSCons, a subclass of TestSCons.TestSCons (both
+defined in ../QMTest/TestScons.py), to manage execution of the timing
+runs.
+
+Unlike the TestSCons.TestSCons base class, the TestSCons.TimeSCons
+subclass copies the contents of its containing directory to the temporary
+working directory.  (It avoids copying the .svn directory, and any files
+or directories that start with the string "TimeSCons-".)  This allows
+the timing configuration files to be checked in directly to our source
+code management system, instead of requiring that they be created from
+in-line data inside the script.
+
+The simplest-possible TimeSCons-run.py script would look like:
+
+    import TestSCons
+    TestSCons.TimeSCons().main()
+
+The above script would end up executing a SConstruct file configuration
+in a temporary directory.  The main() method is the standard interface
+for a timing run.  See its docstring for precisely what it does.
+
+Although the TestSCons.TimeSCons subclass copies its directory contents to
+a temporary working directory for the timing run, because it is a subclass
+of TestSCons.TestSCons, it *can* also create files or directories from
+in-line data.  This is typically done when it's necessary to create
+hundreds of identical input files or directories before running the
+timing test, to avoid cluttering our SCM system with hundreds of otherwise
+meaningless files.
diff --git a/timings/hundred/SConstruct b/timings/hundred/SConstruct
index 2332d73..648c26a 100644
--- a/timings/hundred/SConstruct
+++ b/timings/hundred/SConstruct
@@ -21,32 +21,15 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 
-"""
-This configuration is for timing how we handle the NxM interaction when
-we build a lot of targets from a lot of source files.
+target_count = int(ARGUMENTS['TARGET_COUNT'])
 
-We create a list of 100 target files that will each be built by copying
-a file from a corresponding list of 100 source files.  The source
-files themselves are each built by a Python function action that's the
-equivalent of "echo contents > $TARGET".
-"""
+def copy_files( env, target, source ):
+    for t, s in zip(target, source):
+        open(str(t),  'wb').write(open(str(s), 'rb').read())
 
-target_cnt = 100
+source_list = map(lambda t: 'source_%04d' % t, xrange(target_count))
+target_list = map(lambda t: 'target_%04d' % t, xrange(target_count))
 
 env = Environment()
 
-def create_file( env, target, source ):
-    t = str(target[0])
-    open( t, 'w' ).write('contents\n')
-
-source_list = map(lambda t: 'source_%03d' % t, xrange(target_cnt))
-target_list = map(lambda t: 'target_%03d' % t, xrange(target_cnt))
-
-for source in source_list:
-    env.Command( source, [], create_file )
-
-def copy_files( env, target, source ):
-    for t, s in zip(target, source):
-        open(str(t),  'w').write(open(str(s), 'r').read())
-
 env.Command( target_list, source_list, copy_files )
diff --git a/timings/JTimer/st.conf b/timings/hundred/TimeSCons-run.py
index cf1ecbf..d21db8a 100644
--- a/timings/JTimer/st.conf
+++ b/timings/hundred/TimeSCons-run.py
@@ -22,27 +22,24 @@
 #
 
 """
-scons-time.py configuration file for the "JTimer" timing test.
+This configuration is for timing how we handle the NxM interaction when
+we build a lot of targets from a lot of source files.
+
+We create a list of 500 target files that will each be built by copying
+a file from a corresponding list of 500 source files.  The source
+files themselves are each built by a Python function action that's the
+equivalent of "echo contents > $TARGET".
 """
 
-archive_list = [ 'SConstruct' ]
-subdir = '.'
-targets = '-j2'
-
-import sys
-sys.path.insert(0, '..')
-import SCons_Bars
-
-revs = [
-    1261,   # Fix -j re-scanning built files for implicit deps.
-    1307,   # Move signature Node tranlation of rel_paths into the class.
-    1407,   # Use a Dir scanner instead of a hard-coded method.
-    1435,   # Don't prep .sconsign dependencies until needed.
-    1468,   # Use waiting-Node reference counts to speed up Taskmaster.
-    1703,   # Lobotomize Memoizer.
-    1706,   # Fix _doLookup value-cache misspellings.
-    2380,   # The Big Signature Refactoring hits branches/core.
-]
-
-vertical_bars = SCons_Bars.Release_Bars.gnuplot(labels=True) + \
-                SCons_Bars.Revision_Bars.gnuplot(labels=False, revs=revs)
+import TestSCons
+
+test = TestSCons.TimeSCons()
+
+target_count = 500
+
+for t in xrange(target_count):
+    open('source_%04d' % t, 'wb' ).write('contents\n')
+
+test.main(options='TARGET_COUNT=%s' % target_count)
+
+test.pass_test()