From 2022adb40ed36476e741b4fcfcac1ccb85fb1060 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 9 Dec 2017 11:23:25 -0500 Subject: testing coverage package --- .travis.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e0ea70e..014af2f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,15 @@ before_install: - wget https://github.com/swig/swig/archive/rel-3.0.12.tar.gz - tar xzf rel-3.0.12.tar.gz - cd swig-rel-3.0.12 && ./autogen.sh && ./configure --prefix=/usr && make && sudo make install && cd .. + - sudo pip install coverage + - sudo echo "import coverage" >> /usr/lib/python2.7/sitecustomize.py + - sudo echo "coverage.process_startup()" >> /usr/lib/python2.7/sitecustomize.py script: - - python runtest.py -a || if [[ $? == 2 ]]; then exit 0; else exit 1; fi + - export COVERAGE_PROCESS_START=$PWD/.coveragerc + - echo "[run]" >> .coveragerc + - echo "source = $PWD/src" >> .coveragerc + - echo "parallel = True" >> .coveragerc + - coverage run --rcfile=$PWD/.coverage runtest.py -a || if [[ $? == 2 ]]; then exit 0; else exit 1; fi + - coverage combine + - coverage report -- cgit v0.12 From c6cdc1f988c5163a2345a9ea858dc3e9d647a673 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 9 Dec 2017 11:36:44 -0500 Subject: can't use >> with sudo, switching to tee --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 014af2f..df932f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,8 @@ before_install: - tar xzf rel-3.0.12.tar.gz - cd swig-rel-3.0.12 && ./autogen.sh && ./configure --prefix=/usr && make && sudo make install && cd .. - sudo pip install coverage - - sudo echo "import coverage" >> /usr/lib/python2.7/sitecustomize.py - - sudo echo "coverage.process_startup()" >> /usr/lib/python2.7/sitecustomize.py + - echo "import coverage" | sudo tee --append /usr/lib/python2.7/sitecustomize.py + - echo "coverage.process_startup()" | sudo tee --append /usr/lib/python2.7/sitecustomize.py script: - export COVERAGE_PROCESS_START=$PWD/.coveragerc -- cgit v0.12 From a94f96ffa5568ea465dd8fc8ac41b98780be6161 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 9 Dec 2017 12:53:00 -0500 Subject: updated travis script to not exit, but instead just set the exit code, also setup travis to use code coverage correctly for parallel tests, and finally had to remove the .coveralls, because it is overiding the repo token for my repo, and this should not be in public repos --- .coveralls.yml | 3 --- .travis.yml | 6 +++++- 2 files changed, 5 insertions(+), 4 deletions(-) delete mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml deleted file mode 100644 index 7a1d4a3..0000000 --- a/.coveralls.yml +++ /dev/null @@ -1,3 +0,0 @@ -service_name: travis-pro -repo_token: 1mUyRvK28QaLiKElPYgQGVlrUHxFhS037 - diff --git a/.travis.yml b/.travis.yml index df932f4..9a23b1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ before_install: - tar xzf rel-3.0.12.tar.gz - cd swig-rel-3.0.12 && ./autogen.sh && ./configure --prefix=/usr && make && sudo make install && cd .. - sudo pip install coverage + - sudo pip install coveralls - echo "import coverage" | sudo tee --append /usr/lib/python2.7/sitecustomize.py - echo "coverage.process_startup()" | sudo tee --append /usr/lib/python2.7/sitecustomize.py @@ -20,6 +21,9 @@ script: - echo "[run]" >> .coveragerc - echo "source = $PWD/src" >> .coveragerc - echo "parallel = True" >> .coveragerc - - coverage run --rcfile=$PWD/.coverage runtest.py -a || if [[ $? == 2 ]]; then exit 0; else exit 1; fi + - coverage run --rcfile=$PWD/.coveragerc runtest.py -a || if [[ $? == 2 ]]; then true; else false; fi + +after_success: - coverage combine - coverage report + - coveralls --rcfile=$PWD/.coveragerc -- cgit v0.12 From 4fcdce1a9ba1a2701df19b553f36e3461b0c8c08 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 9 Dec 2017 14:03:05 -0500 Subject: adding in JobTests.py workaround with the coverage testing, also updated runtest.py with the exclude option from Github PR 27. --- .travis.yml | 6 +++++- runtest.py | 26 ++++++++++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9a23b1b..1192edd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,11 @@ script: - echo "[run]" >> .coveragerc - echo "source = $PWD/src" >> .coveragerc - echo "parallel = True" >> .coveragerc - - coverage run --rcfile=$PWD/.coveragerc runtest.py -a || if [[ $? == 2 ]]; then true; else false; fi +# WORKAROUND: attempt to retry JobTests.py if it fails and then continue if it passes, if it fails ten times +# then it is a real failure not related to intermittent travis failures + - n=0; while [[ $n -lt 10 ]]; do coverage run --rcfile=$PWD/.coveragerc runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi +# exclude JobTest.py becuase we already ran that + - coverage run --rcfile=$PWD/.coveragerc runtest.py -a --exclude-list exclude.test || if [[ $? == 2 ]]; then true; else false; fi after_success: - coverage combine diff --git a/runtest.py b/runtest.py index 487100b..88ea45c 100755 --- a/runtest.py +++ b/runtest.py @@ -72,6 +72,10 @@ # This is (will be) used for reporting results back # to a central SCons test monitoring infrastructure. # +# --exclude-list file +# list of tests to exclude in the current selection of test +# mostly meant to easily exclude tests from -a option +# # (Note: There used to be a -v option that specified the SCons # version to be tested, when we were installing in a version-specific # library directory. If we ever resurrect that as the default, then @@ -126,6 +130,7 @@ suppress_stdout = False suppress_stderr = False allow_pipe_files = True quit_on_failure = False +excludelistfile = None usagestr = """\ Usage: runtest.py [OPTIONS] [TEST ...] @@ -173,6 +178,8 @@ Options: -X Test script is executable, don't feed to Python. -x --exec SCRIPT Test SCRIPT. --xml file Save results to file in SCons XML format. + --exclude-list FILE List of tests to exclude in the current selection of test, + mostly meant to easily exclude tests from the -a option Environment Variables: @@ -224,7 +231,7 @@ opts, args = getopt.getopt(args, "b:def:hj:klnP:p:qsv:Xx:t", 'quit-on-failure', 'short-progress', 'time', 'version=', 'exec=', - 'verbose=']) + 'verbose=', 'exclude-list=']) for o, a in opts: if o in ['-b', '--baseline']: @@ -284,7 +291,8 @@ for o, a in opts: scons_exec = 1 elif o in ['-x', '--exec']: scons = a - + elif o in ['--exclude-list']: + excludelistfile = a # --- setup stdout/stderr --- @@ -642,7 +650,7 @@ if old_pythonpath: # ---[ test discovery ]------------------------------------ tests = [] - +excludetests = [] unittests = [] endtests = [] @@ -734,7 +742,7 @@ else: tests.extend(unittests) tests.extend(endtests) tests.sort() - + if not tests: sys.stderr.write(usagestr + """ runtest.py: No tests were found. @@ -743,9 +751,15 @@ runtest.py: No tests were found. """) sys.exit(1) +if excludelistfile: + excludetests = open(excludelistfile, 'r').readlines() + excludetests = [x for x in excludetests if x[0] != '#'] + excludetests = [x[:-1] for x in excludetests] + excludetests = [x.strip() for x in excludetests] + excludetests = [x for x in excludetests if len(x) > 0] # ---[ test processing ]----------------------------------- - +tests = [t for t in tests if t not in excludetests] tests = [Test(t) for t in tests] if list_only: @@ -927,4 +941,4 @@ else: # tab-width:4 # indent-tabs-mode:nil # End: -# vim: set expandtab tabstop=4 shiftwidth=4: +# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file -- cgit v0.12 From 2ea302bf92097955818388649f70557f964b8007 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 9 Dec 2017 14:05:21 -0500 Subject: needed to write the JobTest.py to the exclude list --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1192edd..a6b2102 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,7 @@ script: # then it is a real failure not related to intermittent travis failures - n=0; while [[ $n -lt 10 ]]; do coverage run --rcfile=$PWD/.coveragerc runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi # exclude JobTest.py becuase we already ran that + - echo "src/engine/SCons/JobTests.py" > exclude.test - coverage run --rcfile=$PWD/.coveragerc runtest.py -a --exclude-list exclude.test || if [[ $? == 2 ]]; then true; else false; fi after_success: -- cgit v0.12 From b16418686e04ff2ab8295969e4233f55f376f747 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 9 Dec 2017 18:18:04 -0500 Subject: had to split the build up because it was exceed travis max time limit per job, so this runs all tests split into 8 separate groups --- .travis.yml | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6b2102..b1c44e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,17 @@ dist: trusty + +matrix: + include: + - env: BUILD_JOB_NUM=1 + - env: BUILD_JOB_NUM=2 + - env: BUILD_JOB_NUM=3 + - env: BUILD_JOB_NUM=4 + - env: BUILD_JOB_NUM=5 + - env: BUILD_JOB_NUM=6 + - env: BUILD_JOB_NUM=7 + - env: BUILD_JOB_NUM=8 + + before_install: - sudo apt-get -y install clang gdc docbook-xml xsltproc libxml2-dev libxslt-dev python-pip python-dev fop docbook-xsl-doc-pdf texlive-full biber texmaker build-essential libpcre3-dev autoconf automake libtool bison subversion git - sudo pip install lxml @@ -17,6 +30,7 @@ before_install: - echo "coverage.process_startup()" | sudo tee --append /usr/lib/python2.7/sitecustomize.py script: + - export TOTAL_BUILD_JOBS=8 - export COVERAGE_PROCESS_START=$PWD/.coveragerc - echo "[run]" >> .coveragerc - echo "source = $PWD/src" >> .coveragerc @@ -25,10 +39,20 @@ script: # then it is a real failure not related to intermittent travis failures - n=0; while [[ $n -lt 10 ]]; do coverage run --rcfile=$PWD/.coveragerc runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi # exclude JobTest.py becuase we already ran that - - echo "src/engine/SCons/JobTests.py" > exclude.test - - coverage run --rcfile=$PWD/.coveragerc runtest.py -a --exclude-list exclude.test || if [[ $? == 2 ]]; then true; else false; fi + - echo "src/engine/SCons/JobTests.py" > exclude_jobtest +# also exclude this test since it overides the exit function which doesnt work with coverage +# more info here: https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html# +# TODO: figure out how to cover test/exitfns.py + - echo "test/exitfns.py" >> exclude_jobtest + - python runtest.py -l -a --exclude-list exclude_jobtest > all_tests + - let "start = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * (${BUILD_JOB_NUM} - 1)"; true; + - let "end = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * ${BUILD_JOB_NUM}" + - if (( ${BUILD_JOB_NUM} == ${TOTAL_BUILD_JOBS} )); then end=$(wc -l < all_tests); fi + - if (( ${start} == 0 )); then start=1; fi + - sed -n ${start},${end}p all_tests > build_tests + - coverage run --rcfile=$PWD/.coveragerc runtest.py -f build_tests || if [[ $? == 2 ]]; then true; else false; fi after_success: - coverage combine - coverage report - - coveralls --rcfile=$PWD/.coveragerc + - coveralls --rcfile=$PWD/.coveragerc \ No newline at end of file -- cgit v0.12 From a19c00f5061b620b27f570851566cd93c8b2e0db Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sun, 10 Dec 2017 01:11:45 -0500 Subject: rewrote the travis script to first run a plain test, then do the coverage tests. --- .travis.yml | 108 ++++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 65 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index b1c44e3..a593e1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,6 @@ dist: trusty -matrix: - include: - - env: BUILD_JOB_NUM=1 - - env: BUILD_JOB_NUM=2 - - env: BUILD_JOB_NUM=3 - - env: BUILD_JOB_NUM=4 - - env: BUILD_JOB_NUM=5 - - env: BUILD_JOB_NUM=6 - - env: BUILD_JOB_NUM=7 - - env: BUILD_JOB_NUM=8 - - -before_install: +install: - sudo apt-get -y install clang gdc docbook-xml xsltproc libxml2-dev libxslt-dev python-pip python-dev fop docbook-xsl-doc-pdf texlive-full biber texmaker build-essential libpcre3-dev autoconf automake libtool bison subversion git - sudo pip install lxml - sudo wget http://master.dl.sourceforge.net/project/d-apt/files/d-apt.list -O /etc/apt/sources.list.d/d-apt.list @@ -24,35 +12,69 @@ before_install: - wget https://github.com/swig/swig/archive/rel-3.0.12.tar.gz - tar xzf rel-3.0.12.tar.gz - cd swig-rel-3.0.12 && ./autogen.sh && ./configure --prefix=/usr && make && sudo make install && cd .. - - sudo pip install coverage - - sudo pip install coveralls - - echo "import coverage" | sudo tee --append /usr/lib/python2.7/sitecustomize.py - - echo "coverage.process_startup()" | sudo tee --append /usr/lib/python2.7/sitecustomize.py -script: - - export TOTAL_BUILD_JOBS=8 - - export COVERAGE_PROCESS_START=$PWD/.coveragerc - - echo "[run]" >> .coveragerc - - echo "source = $PWD/src" >> .coveragerc - - echo "parallel = True" >> .coveragerc -# WORKAROUND: attempt to retry JobTests.py if it fails and then continue if it passes, if it fails ten times -# then it is a real failure not related to intermittent travis failures - - n=0; while [[ $n -lt 10 ]]; do coverage run --rcfile=$PWD/.coveragerc runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi -# exclude JobTest.py becuase we already ran that - - echo "src/engine/SCons/JobTests.py" > exclude_jobtest -# also exclude this test since it overides the exit function which doesnt work with coverage -# more info here: https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html# -# TODO: figure out how to cover test/exitfns.py - - echo "test/exitfns.py" >> exclude_jobtest - - python runtest.py -l -a --exclude-list exclude_jobtest > all_tests - - let "start = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * (${BUILD_JOB_NUM} - 1)"; true; - - let "end = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * ${BUILD_JOB_NUM}" - - if (( ${BUILD_JOB_NUM} == ${TOTAL_BUILD_JOBS} )); then end=$(wc -l < all_tests); fi - - if (( ${start} == 0 )); then start=1; fi - - sed -n ${start},${end}p all_tests > build_tests - - coverage run --rcfile=$PWD/.coveragerc runtest.py -f build_tests || if [[ $? == 2 ]]; then true; else false; fi +jobs: + include: + - &test_job + stage: Test + script: + - n=0; while [[ $n -lt 10 ]]; do python runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi + - echo "src/engine/SCons/JobTests.py" > exclude_jobtest + - python runtest.py -a --exclude-list exclude_jobtest || if [[ $? == 2 ]]; then true; else false; fi + before_script: skip + after_success: skip + + - &coverage_jobs + stage: Coverage + + before_script: + - sudo pip install coverage + - sudo pip install coveralls + - echo "import coverage" | sudo tee --append /usr/lib/python2.7/sitecustomize.py + - echo "coverage.process_startup()" | sudo tee --append /usr/lib/python2.7/sitecustomize.py + + script: + - export TOTAL_BUILD_JOBS=8 + - export COVERAGE_PROCESS_START=$PWD/.coveragerc + - echo "[run]" >> .coveragerc + - echo "source = $PWD/src" >> .coveragerc + - echo "parallel = True" >> .coveragerc + # WORKAROUND: attempt to retry JobTests.py if it fails and then continue if it passes, if it fails ten times + # then it is a real failure not related to intermittent travis failures + - n=0; while [[ $n -lt 10 ]]; do coverage run --rcfile=$PWD/.coveragerc runtest.py src/engine/SCons/JobTests.py && break; n=$((n+1)); done; if [ "$n" -gt "9" ]; then false; fi + # exclude JobTest.py becuase we already ran that + - echo "src/engine/SCons/JobTests.py" > exclude_jobtest + # also exclude this test since it overides the exit function which doesnt work with coverage + # more info here: https://coverage.readthedocs.io/en/coverage-4.4.2/subprocess.html# + # TODO: figure out how to cover test/exitfns.py + - echo "test/exitfns.py" >> exclude_jobtest + - python runtest.py -l -a --exclude-list exclude_jobtest > all_tests + - let "start = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * (${BUILD_JOB_NUM} - 1)"; true; + - let "end = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * ${BUILD_JOB_NUM}" + - if (( ${BUILD_JOB_NUM} == ${TOTAL_BUILD_JOBS} )); then end=$(wc -l < all_tests); fi + - if (( ${start} == 0 )); then start=1; fi + - sed -n ${start},${end}p all_tests > build_tests + - coverage run --rcfile=$PWD/.coveragerc runtest.py -f build_tests || if [[ $? == 2 ]]; then true; else false; fi + + after_success: + - coverage combine + - coverage report + - coveralls --rcfile=$PWD/.coveragerc + + env: BUILD_JOB_NUM=1 -after_success: - - coverage combine - - coverage report - - coveralls --rcfile=$PWD/.coveragerc \ No newline at end of file + - <<: *coverage_jobs + env: BUILD_JOB_NUM=2 + - <<: *coverage_jobs + env: BUILD_JOB_NUM=3 + - <<: *coverage_jobs + env: BUILD_JOB_NUM=4 + - <<: *coverage_jobs + env: BUILD_JOB_NUM=5 + - <<: *coverage_jobs + env: BUILD_JOB_NUM=6 + - <<: *coverage_jobs + env: BUILD_JOB_NUM=7 + - <<: *coverage_jobs + env: BUILD_JOB_NUM=8 + \ No newline at end of file -- cgit v0.12