From 1a6de1b9ec85b8856f9b0b977493d3bac1ebfeb2 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 30 Dec 2017 14:21:05 -0500 Subject: coverage reports have been missing the coverage data located in the tmp test dirs, so this change makes sure those data files get included in the report. --- .travis.yml | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7dad533..5703419 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,16 +51,30 @@ jobs: before_script: - sudo pip install coverage - sudo pip install coveralls - - echo "import coverage" | sudo tee --append /usr/lib/python2.7/sitecustomize.py - - echo "coverage.process_startup()" | sudo tee --append /usr/lib/python2.7/sitecustomize.py + # set this ensure user sites are available + - export PYTHONNOUSERSITE= + # we need to preserve the test directories because they have the coverage data + - export PRESERVE=1 + # attempt to get a location where we can store the usercustomize.py fiel + - python -m site + - export PYSITEDIR=$(python -m site --user-site) + - sudo mkdir -p $PYSITEDIR + - sudo touch ${PYSITEDIR}/usercustomize.py + # write the usercustomize.py file so all python processes use coverage and know where the config file is + - echo "import os" | sudo tee --append ${PYSITEDIR}/usercustomize.py + - echo "os.environ['COVERAGE_PROCESS_START'] = '$PWD/.coveragerc'" | sudo tee --append ${PYSITEDIR}/usercustomize.py + - echo "import coverage" | sudo tee --append ${PYSITEDIR}/usercustomize.py + - echo "coverage.process_startup()" | sudo tee --append ${PYSITEDIR}/usercustomize.py script: - export TOTAL_BUILD_JOBS=8 + # write the coverage config file - export COVERAGE_PROCESS_START=$PWD/.coveragerc - echo "[run]" >> .coveragerc - echo "source = $PWD/src" >> .coveragerc - echo "parallel = True" >> .coveragerc - - printf "omit =\n\t*Tests.py\n\tsrc/test_*\n\tsrc/setup.py\n" >> .coveragerc + - printf "omit =\n\t*Tests.py\n\tsrc/test_*\n\tsrc/setup.py\n\n" >> .coveragerc + - echo "[path] = $PWD" >> .coveragerc # Not including this workaround in the coverage report, because it will result # in constantly changing coverage reports depending on the number of times # the JobTests.py had to run to pass @@ -78,10 +92,21 @@ jobs: - if (( ${BUILD_JOB_NUM} == ${TOTAL_BUILD_JOBS} )); then end=$(wc -l < all_tests); fi - if (( ${start} == 0 )); then start=1; fi - sed -n ${start},${end}p all_tests > build_tests - - coverage run --rcfile=$PWD/.coveragerc runtest.py -f build_tests || if [[ $? == 2 ]]; then true; else false; fi + - coverage run -p --rcfile=$PWD/.coveragerc runtest.py -f build_tests || if [[ $? == 2 ]]; then true; else false; fi after_success: - - coverage combine + # get all the coverage data files from the tests and configure them into + # a command line arg list of directories + - find /tmp -name '.coverage.*' -print > coverage_reports.txt + # get just the directories + - sed -i 's/\/\.coverage\..*$//g' coverage_reports.txt + # remove duplicates + - awk '!seen[$0]++' coverage_reports.txt > coverage_reports.txt.tmp + - mv coverage_reports.txt.tmp coverage_reports.txt + # switch newlines for spaces + - sed -i -e ':a' -e 'N' -e '$!ba' -e 's/\n/ /g' coverage_reports.txt + # use all the tmp dir's as locations to collect coverage data + - coverage combine $PWD `cat coverage_reports.txt` - coverage report - coveralls --rcfile=$PWD/.coveragerc @@ -101,4 +126,4 @@ jobs: env: BUILD_JOB_NUM=7 - <<: *coverage_jobs env: BUILD_JOB_NUM=8 - + -- cgit v0.12 From 9f693ab6601ac60bfdc2e7785a1de44bf0bf0e1b Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 30 Dec 2017 17:22:46 -0500 Subject: several tests will not work with coverage, so set them to skip --- .travis.yml | 4 ---- test/Repository/M4.py | 4 ++++ test/exitfns.py | 6 ++++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5703419..22fb9d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,10 +82,6 @@ jobs: # - n=0; while [[ $n -lt 10 ]]; do coverage run --rcfile=$PWD/.coveragerc runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi # exclude JobTest.py becuase we already ran that - echo "src/engine/SCons/JobTests.py" > exclude_jobtest - # also exclude this test since it overides the exit function which doesnt work with coverage - # more info here: https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html# - # TODO: figure out how to cover test/exitfns.py - - echo "test/exitfns.py" >> exclude_jobtest - python runtest.py -l -a --exclude-list exclude_jobtest > all_tests - let "start = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * (${BUILD_JOB_NUM} - 1)"; true; - let "end = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * ${BUILD_JOB_NUM}" diff --git a/test/Repository/M4.py b/test/Repository/M4.py index fe1eb7b..29647e9 100644 --- a/test/Repository/M4.py +++ b/test/Repository/M4.py @@ -36,6 +36,10 @@ _python_ = TestSCons._python_ test = TestSCons.TestSCons() +# TODO: figure out how to write the coverage data to the locked folder or maybe somewhere else +if 'COVERAGE_PROCESS_START' in os.environ: + test.skip_test("this test locks the folder for writing meaning coverage data can not be written; skipping test.") + test.subdir('work', 'repository', ['repository', 'src']) test.write('mym4.py', """ diff --git a/test/exitfns.py b/test/exitfns.py index d1bda3a..dcd8762 100644 --- a/test/exitfns.py +++ b/test/exitfns.py @@ -28,6 +28,12 @@ import TestSCons test = TestSCons.TestSCons() +# also exclude these tests since it overides the exit function which doesnt work with coverage +# # more info here: https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html# +# TODO: figure out how to cover tests which use exit functions +if 'COVERAGE_PROCESS_START' in os.environ: + test.skip_test("This test replaces the exit function which is needed by coverage to write test data; skipping test.") + sconstruct = """ from SCons.exitfuncs import * -- cgit v0.12 From 573c0510aa57e5e1ba228226a2fdbbb1f633d3f8 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 30 Dec 2017 17:23:55 -0500 Subject: forget to add import os for checking the environment for coverage --- test/exitfns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/exitfns.py b/test/exitfns.py index dcd8762..c0da861 100644 --- a/test/exitfns.py +++ b/test/exitfns.py @@ -25,7 +25,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import TestSCons - +import os test = TestSCons.TestSCons() # also exclude these tests since it overides the exit function which doesnt work with coverage -- cgit v0.12 From a90dd653a51ac7be36122a456b3667fe45d30154 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sun, 31 Dec 2017 16:53:51 -0500 Subject: switched to using COVERAGE_FILE for simplicity --- .travis.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 22fb9d7..2b6049a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,13 +53,12 @@ jobs: - sudo pip install coveralls # set this ensure user sites are available - export PYTHONNOUSERSITE= - # we need to preserve the test directories because they have the coverage data - - export PRESERVE=1 - # attempt to get a location where we can store the usercustomize.py fiel + # attempt to get a location where we can store the usercustomize.py file - python -m site - export PYSITEDIR=$(python -m site --user-site) - sudo mkdir -p $PYSITEDIR - sudo touch ${PYSITEDIR}/usercustomize.py + - export COVERAGE_FILE=$PWD/.coverage_file # write the usercustomize.py file so all python processes use coverage and know where the config file is - echo "import os" | sudo tee --append ${PYSITEDIR}/usercustomize.py - echo "os.environ['COVERAGE_PROCESS_START'] = '$PWD/.coveragerc'" | sudo tee --append ${PYSITEDIR}/usercustomize.py @@ -91,18 +90,7 @@ jobs: - coverage run -p --rcfile=$PWD/.coveragerc runtest.py -f build_tests || if [[ $? == 2 ]]; then true; else false; fi after_success: - # get all the coverage data files from the tests and configure them into - # a command line arg list of directories - - find /tmp -name '.coverage.*' -print > coverage_reports.txt - # get just the directories - - sed -i 's/\/\.coverage\..*$//g' coverage_reports.txt - # remove duplicates - - awk '!seen[$0]++' coverage_reports.txt > coverage_reports.txt.tmp - - mv coverage_reports.txt.tmp coverage_reports.txt - # switch newlines for spaces - - sed -i -e ':a' -e 'N' -e '$!ba' -e 's/\n/ /g' coverage_reports.txt - # use all the tmp dir's as locations to collect coverage data - - coverage combine $PWD `cat coverage_reports.txt` + - coverage combine - coverage report - coveralls --rcfile=$PWD/.coveragerc -- cgit v0.12 From 74080a2f9ce5595a7fb9d88433cec9c83c99969c Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 16 Jan 2018 23:31:47 -0500 Subject: moved coverage check for tests to common location. --- QMTest/TestSCons.py | 5 +++++ test/Repository/M4.py | 2 +- test/exitfns.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/QMTest/TestSCons.py b/QMTest/TestSCons.py index f0d25cb..ef907d8 100644 --- a/QMTest/TestSCons.py +++ b/QMTest/TestSCons.py @@ -1036,6 +1036,11 @@ SConscript( sconscript ) # to use cygwin compilers on cmd.exe -> uncomment following line #Configure_lib = 'm' + def coverage_run(self): + """ Check if the the tests are being run under coverage. + """ + return 'COVERAGE_PROCESS_START' in os.environ or 'COVERAGE_FILE' in os.environ + def skip_if_not_msvc(self, check_platform=True): """ Check whether we are on a Windows platform and skip the test if not. This check can be omitted by setting diff --git a/test/Repository/M4.py b/test/Repository/M4.py index 29647e9..8fa91ab 100644 --- a/test/Repository/M4.py +++ b/test/Repository/M4.py @@ -37,7 +37,7 @@ _python_ = TestSCons._python_ test = TestSCons.TestSCons() # TODO: figure out how to write the coverage data to the locked folder or maybe somewhere else -if 'COVERAGE_PROCESS_START' in os.environ: +if test.coverage_run(): test.skip_test("this test locks the folder for writing meaning coverage data can not be written; skipping test.") test.subdir('work', 'repository', ['repository', 'src']) diff --git a/test/exitfns.py b/test/exitfns.py index c0da861..fca44b8 100644 --- a/test/exitfns.py +++ b/test/exitfns.py @@ -31,7 +31,7 @@ test = TestSCons.TestSCons() # also exclude these tests since it overides the exit function which doesnt work with coverage # # more info here: https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html# # TODO: figure out how to cover tests which use exit functions -if 'COVERAGE_PROCESS_START' in os.environ: +if test.coverage_run(): test.skip_test("This test replaces the exit function which is needed by coverage to write test data; skipping test.") sconstruct = """ -- cgit v0.12