Add a script for calibrating settings for timing configurations.

Update the timings scripts with calibrated settings that run
a full build between 9.5 and 10.0 seconds on the buildbot slave.

3 files changed, 58 insertions, 15 deletions

diff --git a/timings/CPPPATH/TimeSCons-run.py b/timings/CPPPATH/TimeSCons-run.py
index dcee7c5..97d9e05 100644
--- a/timings/CPPPATH/TimeSCons-run.py
+++ b/timings/CPPPATH/TimeSCons-run.py
@@ -24,15 +24,28 @@
 """
 This configuration times searching long lists of CPPPATH directories.
 
-We create 5000 on-disk directories.  A single checked-in .h file exists
-in the 'include' directory.  The SConstruct sets CPPPATH to a list of Dir
-Nodes for the created directories, followed by 'include'.  A checked-in .c
-file #includes the .h file to be found in the last directory in the list.
+We create $DIR_COUNT on-disk directories.  A single checked-in .h file
+exists in the 'include' directory.  The SConstruct sets CPPPATH to a
+list of Dir Nodes for the created directories, followed by 'include'.
+A checked-in .c file #includes the .h file to be found in the last
+directory in the list.
 """
 
 import TestSCons
 
-test = TestSCons.TimeSCons(variables={'DIR_COUNT':5000})
+# Full-build time of just under 10 seconds on ubuntu-timings slave,
+# as determined by bin/calibrate.py on 9 December 2009:
+#
+# run   1:   2.235:  DIR_COUNT=50
+# run   2:   3.976:  DIR_COUNT=223
+# run   3:   7.353:  DIR_COUNT=560
+# run   4:   9.569:  DIR_COUNT=761
+# run   5:   9.353:  DIR_COUNT=761
+# run   6:   9.972:  DIR_COUNT=813
+# run   7:   9.930:  DIR_COUNT=813
+# run   8:   9.983:  DIR_COUNT=813
+
+test = TestSCons.TimeSCons(variables={'DIR_COUNT':813})
 
 for d in xrange(test.variables['DIR_COUNT']):
     test.subdir('inc_%04d' % d)
diff --git a/timings/JTimer/TimeSCons-run.py b/timings/JTimer/TimeSCons-run.py
index 05ffbfb..7fe1bf4 100644
--- a/timings/JTimer/TimeSCons-run.py
+++ b/timings/JTimer/TimeSCons-run.py
@@ -25,10 +25,10 @@
 This configuration is for timing how we evaluate long chains of
 dependencies, specifically when -j is used.
 
-We set up a chain of 500 targets that get built from a Python function
-action with no source files (equivalent to "echo junk > $TARGET").
-Each target explicitly depends on the next target in turn, so the
-Taskmaster will do a deep walk of the dependency graph.
+We set up a chain of $TARGET_COUNT targets that get built from a
+Python function action with no source files (equivalent to "echo junk >
+$TARGET").  Each target explicitly depends on the next target in turn,
+so the Taskmaster will do a deep walk of the dependency graph.
 
 This test case was contributed by Kevin Massey.  Prior to revision 1468,
 we had a serious O(N^2) problem in the Taskmaster when handling long
@@ -38,7 +38,26 @@ to the Taskmaster so it could be smarter about not re-evaluating Nodes.
 
 import TestSCons
 
-test = TestSCons.TimeSCons(variables={'TARGET_COUNT':500})
+# Full-build time of just under 10 seconds on ubuntu-timings slave,
+# as determined by bin/calibrate.py on 9 December 2009:
+#
+# run   1:   3.211:  TARGET_COUNT=50
+# run   2:  11.920:  TARGET_COUNT=155
+# run   3:   9.182:  TARGET_COUNT=130
+# run   4:  10.185:  TARGET_COUNT=141
+# run   5:   9.945:  TARGET_COUNT=138
+# run   6:  10.035:  TARGET_COUNT=138
+# run   7:   9.898:  TARGET_COUNT=137
+# run   8:   9.840:  TARGET_COUNT=137
+# run   9:  10.054:  TARGET_COUNT=137
+# run  10:   9.747:  TARGET_COUNT=136
+# run  11:   9.778:  TARGET_COUNT=136
+# run  12:   9.743:  TARGET_COUNT=136
+#
+# The fact that this varies so much suggests that it's pretty
+# non-deterministic, which makes sense for a test involving -j.
+
+test = TestSCons.TimeSCons(variables={'TARGET_COUNT':136})
 
 test.main()
 
diff --git a/timings/hundred/TimeSCons-run.py b/timings/hundred/TimeSCons-run.py
index 915c131..c90c26b 100644
--- a/timings/hundred/TimeSCons-run.py
+++ b/timings/hundred/TimeSCons-run.py
@@ -25,15 +25,26 @@
 This configuration is for timing how we handle the NxM interaction when
 we build a lot of targets from a lot of source files.
 
-We create a list of 500 target files that will each be built by copying
-a file from a corresponding list of 500 source files.  The source
-files themselves are each built by a Python function action that's the
-equivalent of "echo contents > $TARGET".
+We create a list of $TARGET_COUNT target files that will each be built by
+copying a file from a corresponding list of $TARGET_COUNT source files.
+The source files themselves are each built by a Python function action
+that's the equivalent of "echo contents > $TARGET".
 """
 
 import TestSCons
 
-test = TestSCons.TimeSCons(variables={'TARGET_COUNT':500})
+# Full-build time of just under 10 seconds on ubuntu-timings slave,
+# as determined by bin/calibrate.py on 9 December 2009:
+#
+# run   1:   3.124:  TARGET_COUNT=50
+# run   2:  11.936:  TARGET_COUNT=160
+# run   3:   9.175:  TARGET_COUNT=134
+# run   4:  10.489:  TARGET_COUNT=146
+# run   5:   9.798:  TARGET_COUNT=139
+# run   6:   9.695:  TARGET_COUNT=139
+# run   7:   9.670:  TARGET_COUNT=139
+
+test = TestSCons.TimeSCons(variables={'TARGET_COUNT':139})
 
 for t in xrange(test.variables['TARGET_COUNT']):
     open('source_%04d' % t, 'wb' ).write('contents\n')