diff options
| author | William Deegan <bill@baddogconsulting.com> | 2018-02-21 00:31:09 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2018-02-21 00:31:09 (GMT) |
| commit | 867f762f6c1e23524cd1b0262b8e93e822b23d0c (patch) | |
| tree | a311e16e2d205d8ea5ad8ca29139e98f6f7028df | |
| parent | a1a453e5046dbb1151eba2ca605d08b5d5820e81 (diff) | |
| parent | fd6d431634dc461cca47e11d4a89cc1737220fe2 (diff) | |
| download | SCons-867f762f6c1e23524cd1b0262b8e93e822b23d0c.zip SCons-867f762f6c1e23524cd1b0262b8e93e822b23d0c.tar.gz SCons-867f762f6c1e23524cd1b0262b8e93e822b23d0c.tar.bz2 | |
Merge pull request #3104 from dmoody256/JobsTestsParallelFix
JobTests.py parallel issue fix
| -rw-r--r-- | .travis.yml | 17 | ||||
| -rw-r--r-- | src/engine/SCons/JobTests.py | 31 |
2 files changed, 24 insertions, 24 deletions
diff --git a/.travis.yml b/.travis.yml index 0e02b41..9d8b96d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,12 +14,7 @@ jobs: include: - &test_job stage: Test - script: - # WORKAROUND: attempt to retry JobTests.py if it fails and then continue if it passes, if it fails ten times - # then it is a real failure not related to intermittent travis failures - - n=0; while [[ $n -lt 10 ]]; do python runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi - - echo "src/engine/SCons/JobTests.py" > exclude_jobtest - - python runtest.py -a --exclude-list exclude_jobtest || if [[ $? == 2 ]]; then true; else false; fi + script: python runtest.py -a || if [[ $? == 2 ]]; then true; else false; fi before_script: skip after_success: skip python: 2.7 @@ -71,14 +66,8 @@ jobs: - echo "parallel = True" >> .coveragerc - printf "omit =\n\t*Tests.py\n\tsrc/test_*\n\tsrc/setup.py\n\n" >> .coveragerc - echo "[path] = $PWD" >> .coveragerc - # Not including this workaround in the coverage report, because it will result - # in constantly changing coverage reports depending on the number of times - # the JobTests.py had to run to pass - # TODO: figure out how to cover JobTests.py - # - n=0; while [[ $n -lt 10 ]]; do coverage run --rcfile=$PWD/.coveragerc runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi - # exclude JobTest.py becuase we already ran that - - echo "src/engine/SCons/JobTests.py" > exclude_jobtest - - python runtest.py -l -a --exclude-list exclude_jobtest > all_tests + # get a list of all the tests to split them up + - python runtest.py -l -a > all_tests - let "start = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * (${BUILD_JOB_NUM} - 1)"; true; - let "end = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * ${BUILD_JOB_NUM}" - if (( ${BUILD_JOB_NUM} == ${TOTAL_BUILD_JOBS} )); then end=$(wc -l < all_tests); fi diff --git a/src/engine/SCons/JobTests.py b/src/engine/SCons/JobTests.py index 6ae8e92..39918db 100644 --- a/src/engine/SCons/JobTests.py +++ b/src/engine/SCons/JobTests.py @@ -52,7 +52,7 @@ def get_cpu_nums(): return 1 # Default # a large number -num_sines = 10000 +num_sines = 500 # how many parallel jobs to perform for the test num_jobs = get_cpu_nums()*2 @@ -105,7 +105,17 @@ class Task(object): self.taskmaster.begin_list.append(self.i) self.taskmaster.guard.release() + # while task is executing, represent this in the parallel_list + # and then turn it off + self.taskmaster.parallel_list[self.i] = 1 self._do_something() + self.taskmaster.parallel_list[self.i] = 0 + + # check if task was executing while another was also executing + for j in range(1, self.taskmaster.num_tasks): + if(self.taskmaster.parallel_list[j+1] == 1): + self.taskmaster.found_parallel = True + break self.was_executed = 1 @@ -132,10 +142,13 @@ class Task(object): def postprocess(self): self.taskmaster.num_postprocessed = self.taskmaster.num_postprocessed + 1 + def exception_set(self): + pass + class RandomTask(Task): def _do_something(self): # do something that will take some random amount of time: - for i in range(random.randrange(0, num_sines, 1)): + for i in range(random.randrange(0, 100 + num_sines, 1)): x = math.sin(i) time.sleep(0.01) @@ -190,7 +203,10 @@ class Taskmaster(object): self.num_executed = 0 self.num_failed = 0 self.num_postprocessed = 0 + self.parallel_list = [0] * (n+1) + self.found_parallel = False self.Task = Task + # 'guard' guards 'task_begin_list' and 'task_end_list' try: import threading @@ -222,12 +238,7 @@ class Taskmaster(object): def tasks_were_serial(self): "analyze the task order to see if they were serial" - serial = 1 # assume the tasks were serial - for i in range(num_tasks): - serial = serial and (self.begin_list[i] - == self.end_list[i] - == (i + 1)) - return serial + return not self.found_parallel def exception_set(self): pass @@ -271,7 +282,7 @@ class ParallelTestCase(unittest.TestCase): class SleepTask(Task): def _do_something(self): - time.sleep(0.1) + time.sleep(0.01) global SaveThreadPool SaveThreadPool = SCons.Job.ThreadPool @@ -281,7 +292,7 @@ class ParallelTestCase(unittest.TestCase): ThreadPoolCallList.append('put(%s)' % task.i) return SaveThreadPool.put(self, task) def get(self): - time.sleep(0.5) + time.sleep(0.05) result = SaveThreadPool.get(self) ThreadPoolCallList.append('get(%s)' % result[0].i) return result |