From 796d4f35a68e72d11c0cf3f3bae6c20332aee5fe Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 20 Feb 2018 11:31:30 -0500 Subject: changed the way JobTests.py checks for tests running in parallel to fix github issue 3102 --- src/engine/SCons/JobTests.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/engine/SCons/JobTests.py b/src/engine/SCons/JobTests.py index 6ae8e92..36dceb3 100644 --- a/src/engine/SCons/JobTests.py +++ b/src/engine/SCons/JobTests.py @@ -87,6 +87,7 @@ class Task(object): self.taskmaster = taskmaster self.was_executed = 0 self.was_prepared = 0 + def prepare(self): self.was_prepared = 1 @@ -104,9 +105,19 @@ class Task(object): self.taskmaster.guard.acquire() self.taskmaster.begin_list.append(self.i) self.taskmaster.guard.release() - + + # while task is executing, represent this in the parallel_list + # and then turn it off + self.taskmaster.parallel_list[self.i] = 1 self._do_something() - + self.taskmaster.parallel_list[self.i] = 0 + + # check if task was executing while another was also executing + for j in range(1, self.taskmaster.num_tasks): + if(self.taskmaster.parallel_list[j+1] == 1): + self.taskmaster.found_parallel = True + break + self.was_executed = 1 self.taskmaster.guard.acquire() @@ -115,7 +126,7 @@ class Task(object): def executed(self): self.taskmaster.num_executed = self.taskmaster.num_executed + 1 - + self.taskmaster.test_case.failUnless(self.was_prepared, "the task wasn't prepared") self.taskmaster.test_case.failUnless(self.was_executed, @@ -132,13 +143,16 @@ class Task(object): def postprocess(self): self.taskmaster.num_postprocessed = self.taskmaster.num_postprocessed + 1 + def exception_set(self): + pass + class RandomTask(Task): def _do_something(self): # do something that will take some random amount of time: - for i in range(random.randrange(0, num_sines, 1)): + for i in range(random.randrange(0, 1000 + num_sines, 1)): x = math.sin(i) - time.sleep(0.01) - + time.sleep(0.1) + class ExceptionTask(object): """A dummy task class for testing purposes.""" @@ -190,7 +204,10 @@ class Taskmaster(object): self.num_executed = 0 self.num_failed = 0 self.num_postprocessed = 0 + self.parallel_list = [0] * (n+1) + self.found_parallel = False self.Task = Task + # 'guard' guards 'task_begin_list' and 'task_end_list' try: import threading @@ -222,12 +239,7 @@ class Taskmaster(object): def tasks_were_serial(self): "analyze the task order to see if they were serial" - serial = 1 # assume the tasks were serial - for i in range(num_tasks): - serial = serial and (self.begin_list[i] - == self.end_list[i] - == (i + 1)) - return serial + return not self.found_parallel def exception_set(self): pass @@ -562,4 +574,4 @@ if __name__ == "__main__": # tab-width:4 # indent-tabs-mode:nil # End: -# vim: set expandtab tabstop=4 shiftwidth=4: +# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file -- cgit v0.12 From 8f45abe8808744685ea5d6384ae6368b32d9f19d Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 20 Feb 2018 12:06:24 -0500 Subject: removed JobTests.py workaround from travis --- .travis.yml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0e02b41..9d8b96d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,12 +14,7 @@ jobs: include: - &test_job stage: Test - script: - # WORKAROUND: attempt to retry JobTests.py if it fails and then continue if it passes, if it fails ten times - # then it is a real failure not related to intermittent travis failures - - n=0; while [[ $n -lt 10 ]]; do python runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi - - echo "src/engine/SCons/JobTests.py" > exclude_jobtest - - python runtest.py -a --exclude-list exclude_jobtest || if [[ $? == 2 ]]; then true; else false; fi + script: python runtest.py -a || if [[ $? == 2 ]]; then true; else false; fi before_script: skip after_success: skip python: 2.7 @@ -71,14 +66,8 @@ jobs: - echo "parallel = True" >> .coveragerc - printf "omit =\n\t*Tests.py\n\tsrc/test_*\n\tsrc/setup.py\n\n" >> .coveragerc - echo "[path] = $PWD" >> .coveragerc - # Not including this workaround in the coverage report, because it will result - # in constantly changing coverage reports depending on the number of times - # the JobTests.py had to run to pass - # TODO: figure out how to cover JobTests.py - # - n=0; while [[ $n -lt 10 ]]; do coverage run --rcfile=$PWD/.coveragerc runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi - # exclude JobTest.py becuase we already ran that - - echo "src/engine/SCons/JobTests.py" > exclude_jobtest - - python runtest.py -l -a --exclude-list exclude_jobtest > all_tests + # get a list of all the tests to split them up + - python runtest.py -l -a > all_tests - let "start = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * (${BUILD_JOB_NUM} - 1)"; true; - let "end = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * ${BUILD_JOB_NUM}" - if (( ${BUILD_JOB_NUM} == ${TOTAL_BUILD_JOBS} )); then end=$(wc -l < all_tests); fi -- cgit v0.12 From f620c01ef13e0ecc3a7f2086c7c4ecef9e560725 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 20 Feb 2018 12:17:46 -0500 Subject: whitespace clean up, no functional changes --- src/engine/SCons/JobTests.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/engine/SCons/JobTests.py b/src/engine/SCons/JobTests.py index 36dceb3..cc5e3c7 100644 --- a/src/engine/SCons/JobTests.py +++ b/src/engine/SCons/JobTests.py @@ -87,7 +87,6 @@ class Task(object): self.taskmaster = taskmaster self.was_executed = 0 self.was_prepared = 0 - def prepare(self): self.was_prepared = 1 @@ -105,19 +104,19 @@ class Task(object): self.taskmaster.guard.acquire() self.taskmaster.begin_list.append(self.i) self.taskmaster.guard.release() - + # while task is executing, represent this in the parallel_list - # and then turn it off + # and then turn it off self.taskmaster.parallel_list[self.i] = 1 self._do_something() self.taskmaster.parallel_list[self.i] = 0 - + # check if task was executing while another was also executing for j in range(1, self.taskmaster.num_tasks): if(self.taskmaster.parallel_list[j+1] == 1): self.taskmaster.found_parallel = True break - + self.was_executed = 1 self.taskmaster.guard.acquire() @@ -126,7 +125,7 @@ class Task(object): def executed(self): self.taskmaster.num_executed = self.taskmaster.num_executed + 1 - + self.taskmaster.test_case.failUnless(self.was_prepared, "the task wasn't prepared") self.taskmaster.test_case.failUnless(self.was_executed, @@ -152,7 +151,7 @@ class RandomTask(Task): for i in range(random.randrange(0, 1000 + num_sines, 1)): x = math.sin(i) time.sleep(0.1) - + class ExceptionTask(object): """A dummy task class for testing purposes.""" @@ -574,4 +573,4 @@ if __name__ == "__main__": # tab-width:4 # indent-tabs-mode:nil # End: -# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From fd6d431634dc461cca47e11d4a89cc1737220fe2 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 20 Feb 2018 15:23:20 -0500 Subject: reduced number of calculations and sleep time to improve test speed --- src/engine/SCons/JobTests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/JobTests.py b/src/engine/SCons/JobTests.py index cc5e3c7..39918db 100644 --- a/src/engine/SCons/JobTests.py +++ b/src/engine/SCons/JobTests.py @@ -52,7 +52,7 @@ def get_cpu_nums(): return 1 # Default # a large number -num_sines = 10000 +num_sines = 500 # how many parallel jobs to perform for the test num_jobs = get_cpu_nums()*2 @@ -148,9 +148,9 @@ class Task(object): class RandomTask(Task): def _do_something(self): # do something that will take some random amount of time: - for i in range(random.randrange(0, 1000 + num_sines, 1)): + for i in range(random.randrange(0, 100 + num_sines, 1)): x = math.sin(i) - time.sleep(0.1) + time.sleep(0.01) class ExceptionTask(object): """A dummy task class for testing purposes.""" @@ -282,7 +282,7 @@ class ParallelTestCase(unittest.TestCase): class SleepTask(Task): def _do_something(self): - time.sleep(0.1) + time.sleep(0.01) global SaveThreadPool SaveThreadPool = SCons.Job.ThreadPool @@ -292,7 +292,7 @@ class ParallelTestCase(unittest.TestCase): ThreadPoolCallList.append('put(%s)' % task.i) return SaveThreadPool.put(self, task) def get(self): - time.sleep(0.5) + time.sleep(0.05) result = SaveThreadPool.get(self) ThreadPoolCallList.append('get(%s)' % result[0].i) return result -- cgit v0.12