updated Parallel/failed-build test to fix race condition

11 files changed, 267 insertions, 133 deletions

diff --git a/test/Parallel/failed-build.py b/test/Parallel/failed-build.py
deleted file mode 100644
index 54fd2fc..0000000
--- a/test/Parallel/failed-build.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python
-#
-# __COPYRIGHT__
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
-# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#
-
-"""
-Verify that a failed build action with -j works as expected.
-"""
-
-__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
-
-import TestSCons
-
-python = TestSCons.python
-
-try:
-    import threading
-except ImportError:
-    # if threads are not supported, then
-    # there is nothing to test
-    TestCmd.no_result()
-    sys.exit()
-
-
-test = TestSCons.TestSCons()
-
-# We want to verify that -j 2 starts precisely two jobs, the first of
-# which fails and the second of which succeeds, and then stops processing
-# due to the first build failure - the second build job does not just
-# continue processing tasks.  To try to control the timing, the two
-# created build scripts use a pair of marker directories.
-#
-# The failure script waits until it sees the 'mycopy.started' directory
-# that indicates the successful script has, in fact, gotten started.
-# If we don't wait, then SCons could detect our script failure early
-# (typically if a high system load happens to delay SCons' ability to
-# start the next script) and then not start the successful script at all.
-#
-# The successful script waits until it sees the 'myfail.exiting' directory
-# that indicates the failure script has finished (with everything except
-# the final sys.exit(), that is).  If we don't wait for that, then SCons
-# could detect our successful exit first (typically if a high system
-# load happens to delay the failure script) and start another job before
-# it sees the failure from the first script.
-#
-# Both scripts are set to bail if they had to wait too long for what
-# they expected to see.
-
-test.write('myfail.py', """\
-import os
-import sys
-import time
-WAIT = 10
-count = 0
-while not os.path.exists('mycopy.started') and count < WAIT:
-    time.sleep(1)
-    count += 1
-if count >= WAIT:
-    sys.exit(99)
-os.mkdir('myfail.exiting')
-sys.exit(1)
-""")
-
-test.write('mycopy.py', """\
-import os
-import sys
-import time
-os.mkdir('mycopy.started')
-with open(sys.argv[1], 'wb') as ofp, open(sys.argv[2], 'rb') as ifp:
-    ofp.write(ifp.read())
-WAIT = 10
-count = 0
-while not os.path.exists('myfail.exiting') and count < WAIT:
-    time.sleep(1)
-    count += 1
-if count >= WAIT:
-    sys.exit(99)
-os.rmdir('mycopy.started')
-sys.exit(0)
-""")
-
-test.write('SConstruct', """
-MyCopy = Builder(action=[[r'%(python)s', 'mycopy.py', '$TARGET', '$SOURCE']])
-Fail = Builder(action=[[r'%(python)s', 'myfail.py', '$TARGETS', '$SOURCE']])
-env = Environment(BUILDERS={'MyCopy' : MyCopy, 'Fail' : Fail})
-env.Fail(target='f3', source='f3.in')
-env.MyCopy(target='f4', source='f4.in')
-env.MyCopy(target='f5', source='f5.in')
-env.MyCopy(target='f6', source='f6.in')
-""" % locals())
-
-test.write('f3.in', "f3.in\n")
-test.write('f4.in', "f4.in\n")
-test.write('f5.in', "f5.in\n")
-test.write('f6.in', "f6.in\n")
-
-test.run(arguments='-j 2 .',
-         status=2,
-         stderr="scons: *** [f3] Error 1\n")
-
-test.must_not_exist(test.workpath('f3'))
-test.must_match(test.workpath('f4'), 'f4.in\n')
-test.must_not_exist(test.workpath('f5'))
-test.must_not_exist(test.workpath('f6'))
-
-
-
-test.pass_test()
-
-# Local Variables:
-# tab-width:4
-# indent-tabs-mode:nil
-# End:
-# vim: set expandtab tabstop=4 shiftwidth=4:
diff --git a/test/Parallel/failed-build/failed-build.py b/test/Parallel/failed-build/failed-build.py
new file mode 100644
index 0000000..39d3e56
--- /dev/null
+++ b/test/Parallel/failed-build/failed-build.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+#
+# Copyright The SCons Foundation
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+"""
+Verify that a failed build action with -j works as expected.
+"""
+
+import os
+
+import TestSCons
+
+python = TestSCons.python
+test = TestSCons.TestSCons()
+
+try:
+    import psutil
+except ImportError as e:
+    test.skip_test("Failed to import psutil required for test, skipping.")
+
+test.dir_fixture('fixture')
+
+# We want to verify that -j 2 starts precisely two jobs, the first of
+# which fails and the second of which succeeds, and then stops processing
+# due to the first build failure - the second build job does not just
+# continue processing tasks.  To try to control the timing, the two
+# task scripts will be managed by a server script which regulates the 
+# state of the test.
+#
+# The failure script waits until the server responds that the 
+# copy script has, in fact, gotten started. If we don't wait, then SCons 
+# could detect our script failure early (typically if a high system load 
+# happens to delay SCons' ability to start the next script) and then not 
+# start the successful script at all.
+#
+# The successful script waits until the server responds that the 
+# failure script has finished (the server checks that the task pid does not
+# exist). If we don't wait for that, then SCons could detect our successful 
+# exit first (typically if a high system load happens to delay the failure 
+# script) and start another job before it sees the failure from the first 
+# script.
+#
+# Both scripts are set to bail if they had to wait too long for what
+# they expected to see.
+
+test.run(arguments='-j 2 .',
+        status=2,
+        stderr="scons: *** [f3] Error 1\n")
+
+test.must_not_exist(test.workpath('f3'))
+test.must_match(test.workpath('f4'), 'f4.in')
+test.must_not_exist(test.workpath('f5'))
+test.must_not_exist(test.workpath('f6'))
+
+test.pass_test()
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4:
diff --git a/test/Parallel/failed-build/fixture/SConstruct b/test/Parallel/failed-build/fixture/SConstruct
new file mode 100644
index 0000000..21109fe
--- /dev/null
+++ b/test/Parallel/failed-build/fixture/SConstruct
@@ -0,0 +1,24 @@
+import sys
+import os
+import threading
+
+import random
+PORT = random.randint(10000, 60000)
+
+sys.path.append(os.getcwd())
+from teststate import server_thread
+
+# this thread will setup a sever for the different tasks to talk to
+# and act as a manager of IPC and the different tasks progressing
+# the test through different states
+x = threading.Thread(target=server_thread, args=(PORT,))
+x.daemon = True
+x.start()
+
+MyCopy = Builder(action=[[sys.executable, 'mycopy.py', '$TARGET', '$SOURCE', str(PORT)]])
+Fail = Builder(action=[[sys.executable, 'myfail.py', '$TARGETS', '$SOURCE', str(PORT)]])
+env = Environment(BUILDERS={'MyCopy' : MyCopy, 'Fail' : Fail})
+env.Fail(target='f3', source='f3.in')
+env.MyCopy(target='f4', source='f4.in')
+env.MyCopy(target='f5', source='f5.in')
+env.MyCopy(target='f6', source='f6.in')
\ No newline at end of file
diff --git a/test/Parallel/failed-build/fixture/f3.in b/test/Parallel/failed-build/fixture/f3.in
new file mode 100644
index 0000000..605d4f1
--- /dev/null
+++ b/test/Parallel/failed-build/fixture/f3.in
@@ -0,0 +1 @@
+f3.in
\ No newline at end of file
diff --git a/test/Parallel/failed-build/fixture/f4.in b/test/Parallel/failed-build/fixture/f4.in
new file mode 100644
index 0000000..d0834be
--- /dev/null
+++ b/test/Parallel/failed-build/fixture/f4.in
@@ -0,0 +1 @@
+f4.in
\ No newline at end of file
diff --git a/test/Parallel/failed-build/fixture/f5.in b/test/Parallel/failed-build/fixture/f5.in
new file mode 100644
index 0000000..d0834be
--- /dev/null
+++ b/test/Parallel/failed-build/fixture/f5.in
@@ -0,0 +1 @@
+f4.in
\ No newline at end of file
diff --git a/test/Parallel/failed-build/fixture/f6.in b/test/Parallel/failed-build/fixture/f6.in
new file mode 100644
index 0000000..12964b9
--- /dev/null
+++ b/test/Parallel/failed-build/fixture/f6.in
@@ -0,0 +1 @@
+f6.in
\ No newline at end of file
diff --git a/test/Parallel/failed-build/fixture/mycopy.py b/test/Parallel/failed-build/fixture/mycopy.py
new file mode 100644
index 0000000..97c8e88
--- /dev/null
+++ b/test/Parallel/failed-build/fixture/mycopy.py
@@ -0,0 +1,35 @@
+import os
+import sys
+import time
+import http.client
+
+sys.path.append(os.getcwd())
+from teststate import Response
+
+conn = http.client.HTTPConnection("127.0.0.1",  port=int(sys.argv[3]))
+conn.request("GET", "/?set_mycopy_started=1")
+conn.getresponse().read()
+
+with open(sys.argv[1], 'wb') as ofp, open(sys.argv[2], 'rb') as ifp:
+    ofp.write(ifp.read())
+    
+WAIT = 10
+count = 0
+
+def check_test_state():
+
+    conn.request("GET", "/?get_myfail_done=1")
+    response = conn.getresponse()
+    response.read()
+    status = response.status
+    return status == Response.OK.value
+
+while not check_test_state() and count < WAIT:
+    time.sleep(0.1)
+    count += 0.1
+
+if count >= WAIT:
+    sys.exit(99)
+
+conn.close()
+sys.exit(0)
\ No newline at end of file
diff --git a/test/Parallel/failed-build/fixture/myfail.py b/test/Parallel/failed-build/fixture/myfail.py
new file mode 100644
index 0000000..bd54d44
--- /dev/null
+++ b/test/Parallel/failed-build/fixture/myfail.py
@@ -0,0 +1,31 @@
+import os
+import sys
+import time
+import http.client
+
+sys.path.append(os.getcwd())
+from teststate import Response
+
+WAIT = 10
+count = 0
+
+conn = http.client.HTTPConnection("127.0.0.1", port=int(sys.argv[3]))
+
+def check_test_state():
+    conn.request("GET", "/?get_mycopy_started=1")
+    response = conn.getresponse()
+    response.read()
+    status = response.status
+    return status == Response.OK.value
+    
+while not check_test_state() and count < WAIT:
+    time.sleep(0.1)
+    count += 0.1
+
+if count >= WAIT:
+    sys.exit(99)
+
+conn.request("GET", "/?set_myfail_done=1&pid=" + str(os.getpid()))
+conn.close()
+
+sys.exit(1)
\ No newline at end of file
diff --git a/test/Parallel/failed-build/fixture/sconstest.skip b/test/Parallel/failed-build/fixture/sconstest.skip
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/Parallel/failed-build/fixture/sconstest.skip
diff --git a/test/Parallel/failed-build/fixture/teststate.py b/test/Parallel/failed-build/fixture/teststate.py
new file mode 100644
index 0000000..e549f2c
--- /dev/null
+++ b/test/Parallel/failed-build/fixture/teststate.py
@@ -0,0 +1,92 @@
+import http.server
+import socketserver
+import time
+from urllib.parse import urlparse, parse_qs
+from threading import Lock
+from enum import Enum
+import psutil
+
+class TestState(Enum):
+    start_state = 0
+    mycopy_started = 1
+    myfail_done = 2
+
+class Response(Enum):
+    OK = 200
+    WAIT = 201
+    DONE = 202
+
+def server_thread(PORT):
+
+    Handler = http.server.SimpleHTTPRequestHandler
+
+    class S( http.server.BaseHTTPRequestHandler):
+
+        current_state = TestState.start_state
+        mutex = Lock()
+        pid_killed_tries = 20
+
+        def do_GET(self):
+            gets = parse_qs(urlparse(self.path).query)
+
+            # the two tasks will communicate with the server with basic get
+            # requests, either updating or getting the state of the test to
+            # know if they continue. The server is regluating the state and making
+            # the right criteria is in place from both tasks before moving the 
+            # test state forward.
+
+            if gets.get('set_mycopy_started'):
+                S.mutex.acquire()
+                if S.current_state == TestState.start_state:
+                    S.current_state = TestState.mycopy_started
+                    response = Response.OK
+                else:
+                    response = Response.WAIT
+                S.mutex.release()
+                
+            elif gets.get('get_mycopy_started'):
+                S.mutex.acquire()
+                if S.current_state == TestState.mycopy_started:
+                    response = Response.OK
+                else:
+                    response = Response.WAIT
+                S.mutex.release()
+
+            elif gets.get('set_myfail_done'):
+                S.mutex.acquire()
+                if S.current_state == TestState.mycopy_started:
+                    count = 0
+                    pid = int(gets.get('pid')[0])
+                    while psutil.pid_exists(pid) and count < self.pid_killed_tries:
+                        time.sleep(0.5)
+                        count += 1
+                    if not psutil.pid_exists(pid):
+                        S.current_state = TestState.myfail_done
+                        response = Response.DONE
+                    else:
+                        response = Response.WAIT
+                else:
+                    response = Response.WAIT
+                S.mutex.release()
+
+            elif gets.get('get_myfail_done'):
+                S.mutex.acquire()
+                if S.current_state == TestState.myfail_done:
+                    response = Response.OK
+                else:
+                    response = Response.WAIT
+                S.mutex.release()
+
+            else:
+                response = Response.WAIT
+            self.send_response(response.value)
+            self.send_header('Content-type', 'text/html')
+            self.end_headers()
+            if response != Response.DONE:
+                self.wfile.write("".encode('utf-8'))
+
+        def log_message(self, format, *args):
+            return
+
+    httpd = socketserver.TCPServer(("", PORT), S)
+    httpd.serve_forever()
\ No newline at end of file