Merge pull request #3339 from mwichmann/ci-multi

[WIP] Run all CI builds with -j 2

9 files changed, 58 insertions, 48 deletions

diff --git a/.appveyor.yml b/.appveyor.yml
index cbf7233..d2c63cb 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -217,7 +217,7 @@ build_script:
   # Windows run the tests
   # NOTE: running powershell from cmd on purpose because it formats the output
   # correctly
-  - cmd: powershell -Command "& { if($env:COVERAGE -eq 1) { coverage run -p --rcfile=$($env:COVERAGE_PROCESS_START) runtest.py --exclude-list exclude_list.txt -f build_tests.txt } else { C:\\%WINPYTHON%\\python.exe runtest.py -j 2 --exclude-list exclude_list.txt -f build_tests.txt }; if($LastExitCode -eq 2 -Or $LastExitCode -eq 0) { $host.SetShouldExit(0 )} else {$host.SetShouldExit(1)}}"
+  - cmd: powershell -Command "& { if($env:COVERAGE -eq 1) { coverage run -p --rcfile=$($env:COVERAGE_PROCESS_START) runtest.py -j 2 --exclude-list exclude_list.txt -f build_tests.txt } else { C:\\%WINPYTHON%\\python.exe runtest.py -j 2 --exclude-list exclude_list.txt -f build_tests.txt }; if($LastExitCode -eq 2 -Or $LastExitCode -eq 0) { $host.SetShouldExit(0 )} else {$host.SetShouldExit(1)}}"
 
   # linux run the tests
   # unset JAVA_TOOL_OPTIONS because newer java prints this to stderr
diff --git a/.travis.yml b/.travis.yml
index a8cb940..b0d64e9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,13 +27,38 @@ stages:
   - Coverage
   - Test
 
+# Note: Travis does not provide a way to specify the order of
+# jobs within a Stage, which are "run in parallel", but with
+# limitations: from observation four or five are kicked
+# off, then additional jobs as initial ones complete.
+# We want the slowest jobs in the first batch since the
+# faster ones are less than half the time of the slowest,
+# we should be able to finish the Test task in the time of the
+# slowest job rather than (a fast job + the slowest job).
+# Putting the pypy jobs first may help with this, though it's
+# apparently not guaranteed.
+
 jobs:
   include:
     - &test_job
       stage: Test
-      script: python runtest.py -a -t || if [[ $? == 2 ]]; then true; else false; fi
+      script: python runtest.py -a -t -j 2 || if [[ $? == 2 ]]; then true; else false; fi
       before_script: skip
       after_success: skip
+      python: pypy3
+      env: 
+        - PYVER=pypy3
+        - PYTHON=pypy3
+      sudo: required
+
+    - <<: *test_job
+      python: pypy
+      env: 
+        - PYVER=pypy
+        - PYTHON=pypy
+      sudo: required
+
+    - <<: *test_job
       python: 2.7
       env: 
         - PYVER=27
@@ -62,24 +87,6 @@ jobs:
       sudo: required
       dist: xenial  # required for Python 3.7 (travis-ci/travis-ci#9069)
 
-    - &slow_test_job
-      stage: Test
-      script: python runtest.py -a -j 4 -t || if [[ $? == 2 ]]; then true; else false; fi
-      before_script: skip
-      after_success: skip
-      python: pypy
-      env: 
-        - PYVER=pypy
-        - PYTHON=pypy
-      sudo: required
-
-    - <<: *slow_test_job
-      python: pypy3
-      env: 
-        - PYVER=pypy3
-        - PYTHON=pypy3
-      sudo: required
-
 
     - &coverage_jobs
       stage: Coverage
@@ -128,7 +135,7 @@ jobs:
         - if (( ${BUILD_JOB_NUM} == ${TOTAL_BUILD_JOBS} )); then end=$(wc -l < all_tests); fi
         - if (( ${start} == 0 )); then start=1; fi
         - sed -n ${start},${end}p all_tests > build_tests
-        - coverage run -p --rcfile=$PWD/.coveragerc runtest.py -f build_tests || if [[ $? == 2 ]]; then true; else false; fi
+        - coverage run -p --rcfile=$PWD/.coveragerc runtest.py -f build_tests -j 2 || if [[ $? == 2 ]]; then true; else false; fi
         
       after_script:
         - coverage combine
diff --git a/test/Interactive/configure.py b/test/Interactive/configure.py
index 906a7e9..a7f0735 100644
--- a/test/Interactive/configure.py
+++ b/test/Interactive/configure.py
@@ -38,10 +38,9 @@ test = TestSCons.TestSCons()
 
 test.write('mycc.py', r"""
 import sys
-outfile = open(sys.argv[1], 'wb')
-infile = open(sys.argv[2], 'rb')
-for l in [l for l in infile.readlines() if l[:7] != '/*c++*/']:
-    outfile.write(l)
+with open(sys.argv[1], 'wb') as ofp, open(sys.argv[2], 'rb') as ifp:
+    for l in [l for l in ifp.readlines() if l[:7] != '/*c++*/']:
+        ofp.write(l)
 sys.exit(0)
 """)
 
diff --git a/test/Interactive/option-j.py b/test/Interactive/option-j.py
index 279fc51..356a067 100644
--- a/test/Interactive/option-j.py
+++ b/test/Interactive/option-j.py
@@ -39,10 +39,10 @@ from SCons.Script import *
 def cat(target, source, env):
     t = str(target[0])
     os.mkdir(t + '.started')
-    fp = open(t, 'wb')
-    for s in source:
-        fp.write(open(str(s), 'rb').read())
-    fp.close()
+    with open(t, 'wb') as ofp:
+        for s in source:
+            with open(str(s), 'rb') as ifp:
+                ofp.write(ifp.read())
     os.mkdir(t + '.finished')
 
 def must_be_finished(target, source, env, dir):
@@ -63,10 +63,10 @@ def must_wait_for_f2_b_out(target, source, env):
     f2_b_started = 'f2-b.out.started'
     while not os.path.exists(f2_b_started):
         time.sleep(1)
-    fp = open(t, 'wb')
-    for s in source:
-        fp.write(open(str(s), 'rb').read())
-    fp.close()
+    with open(t, 'wb') as ofp:
+        for s in source:
+            with open(str(s), 'rb') as ifp:
+                ofp.write(ifp.read())
     os.mkdir(t + '.finished')
 
 def _f2_a_out_must_not_be_finished(target, source, env):
diff --git a/test/Parallel/duplicate-children.py b/test/Parallel/duplicate-children.py
index 8b8f4cd..fab1e58 100644
--- a/test/Parallel/duplicate-children.py
+++ b/test/Parallel/duplicate-children.py
@@ -37,10 +37,10 @@ test = TestSCons.TestSCons()
 
 test.write('cat.py', """\
 import sys
-fp = open(sys.argv[1], 'wb')
-for fname in sys.argv[2:]:
-    fp.write(open(fname, 'rb').read())
-fp.close()
+with open(sys.argv[1], 'wb') as ofp:
+    for fname in sys.argv[2:]:
+        with open(fname, 'rb') as ifp:
+            ofp.write(ifp.read())
 """)
 
 test.write('sleep.py', """\
diff --git a/test/Parallel/duplicate-target.py b/test/Parallel/duplicate-target.py
index efe20d9..e947edc 100644
--- a/test/Parallel/duplicate-target.py
+++ b/test/Parallel/duplicate-target.py
@@ -49,14 +49,15 @@ test.write(['work', 'mycopy.py'], """\
 import sys
 import time
 time.sleep(int(sys.argv[1]))
-open(sys.argv[2], 'wb').write(open(sys.argv[3], 'rb').read())
+with open(sys.argv[2], 'wb') as ofp, open(sys.argv[3], 'rb') as ifp:
+    ofp.write(ifp.read())
 """)
 
 test.write(['work', 'mytar.py'], """\
 import sys
 import os.path
 
-fp = open(sys.argv[1], 'wb')
+ofp = open(sys.argv[1], 'wb')
 
 def visit(dirname):
     names = os.listdir(dirname)
@@ -66,10 +67,12 @@ def visit(dirname):
         if os.path.isdir(p):
             visit(p)
         elif os.path.isfile(p):
-            fp.write(open(p, 'rb').read())
+            with open(p, 'rb') as ifp:
+                ofp.write(ifp.read())
 
 for s in sys.argv[2:]:
     visit(s)
+ofp.close()
 """)
 
 test.write(['work', 'SConstruct'], """\
diff --git a/test/Parallel/failed-build.py b/test/Parallel/failed-build.py
index 2360679..bfdc01d 100644
--- a/test/Parallel/failed-build.py
+++ b/test/Parallel/failed-build.py
@@ -78,7 +78,8 @@ import os
 import sys
 import time
 os.mkdir('mycopy.started')
-open(sys.argv[1], 'wb').write(open(sys.argv[2], 'rb').read())
+with open(sys.argv[1], 'wb') as ofp, open(sys.argv[2], 'rb') as ifp:
+    ofp.write(ifp.read())
 for i in [1, 2, 3, 4, 5]:
     time.sleep(2)
     if os.path.exists('myfail.exiting'):
diff --git a/test/Parallel/multiple-parents.py b/test/Parallel/multiple-parents.py
index 7fbcb00..5a52f28 100644
--- a/test/Parallel/multiple-parents.py
+++ b/test/Parallel/multiple-parents.py
@@ -147,8 +147,8 @@ Default(all)
 re_error = """\
 (scons: \\*\\*\\* \\[failed\\d+] Error 2\\n)|\
 (scons: \\*\\*\\* \\[missing\\d+] Source `MissingSrc' not found, needed by target `missing\\d+'\\.(  Stop\\.)?\\n)|\
-(scons: \\*\\*\\* \\[\\w+] Build interrupted\.\\n)|\
-(scons: Build interrupted\.\\n)\
+(scons: \\*\\*\\* \\[\\w+] Build interrupted\\.\\n)|\
+(scons: Build interrupted\\.\\n)\
 """
 
 re_errors = "(" + re_error + ")+"
diff --git a/test/Parallel/ref_count.py b/test/Parallel/ref_count.py
index 7ce5910..8a31bf3 100644
--- a/test/Parallel/ref_count.py
+++ b/test/Parallel/ref_count.py
@@ -74,12 +74,12 @@ while args:
         time.sleep(int(args.pop(0)))
 contents = ''
 for ifile in args:
-    contents = contents + open(ifile, 'r').read()
+    with open(ifile, 'r') as ifp:
+        contents = contents + ifp.read()
 for ofile in outputs:
-    ofp = open(ofile, 'w')
-    ofp.write('%s:  building from %s\\n' % (ofile, " ".join(args)))
-    ofp.write(contents)
-    ofp.close()
+    with open(ofile, 'w') as ofp:
+        ofp.write('%s:  building from %s\\n' % (ofile, " ".join(args)))
+        ofp.write(contents)
 """)
 
 #