test coverage

2 files changed, 24 insertions, 125 deletions

diff --git a/.appveyor.yml b/.appveyor.yml
index 40f13de..f6b56d7 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -14,10 +14,9 @@ cache:
   - C:\ProgramData\chocolatey\lib -> appveyor.yml
 
 install:
-  ### WINDOWS ###
   # add python and python user-base to path for pip installs
   - cmd: "C:\\%WINPYTHON%\\python.exe --version"
-  - cmd: for /F "tokens=*" %%g in ('C:\\%WINPYTHON%\\python.exe -m site --user-site') do (set PYSITEDIR=%%g)
+  - cmd: for /F "tokens=*" %%g in ('C:\\%WINPYTHON%\\python.exe -c "import sys; print(sys.path[-1])"') do (set PYSITEDIR=%%g)
   # use mingw 32 bit until #3291 is resolved
   - cmd: "set PATH=C:\\%WINPYTHON%;C:\\%WINPYTHON%\\Scripts;C:\\ProgramData\\chocolatey\\bin;C:\\MinGW\\bin;C:\\MinGW\\msys\\1.0\\bin;C:\\cygwin\\bin;C:\\msys64\\usr\\bin;C:\\msys64\\mingw64\\bin;%PATH%"
   - cmd: "C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pip setuptools wheel "
@@ -28,77 +27,24 @@ install:
   - cmd: set SCONS_CACHE_MSVC_CONFIG=true
   - cmd: set
 
-  ### LINUX ###
-  - sh: sudo add-apt-repository -y ppa:deadsnakes/ppa
-  # allow the CI to continue even if some pkg in the pkglist may not be up to date
-  - sh: sudo apt-get update || true
-  - sh: sudo apt-get -y install python$PYTHON
-  - sh: wget https://bootstrap.pypa.io/get-pip.py
-  - sh: sudo -H python$PYTHON get-pip.py
-  - sh: which python$PYTHON
-  - sh: python$PYTHON --version
-  - sh: export PYSITEDIR=$(python$PYTHON -m site --user-site)
-  - sh: python$PYTHON -m pip install --user -U --progress-bar off pip setuptools wheel
-  - sh: python$PYTHON -m pip install --user -U --progress-bar off coverage codecov
-  - sh: STATIC_DEPS=true python$PYTHON -m pip install --user -U --progress-bar off lxml
-  - sh: ./.travis/install.sh
-  - sh: printenv
-
 # build matrix will be number of images multiplied by each '-' below,
 # less any exclusions.
 # split builds into sets of four jobs due to appveyor per-job time limit
 environment:
   matrix:
     - WINPYTHON: "Python35"
-      PYTHON: "3.5"
-      PYVER: 35
-      BUILD_JOB_NUM: 1
       COVERAGE: 0
 
     - WINPYTHON: "Python36"
-      PYTHON: "3.6"
-      PYVER: 36
-      BUILD_JOB_NUM: 1
       COVERAGE: 1
 
     - WINPYTHON: "Python37"
-      PYTHON: "3.7"
-      PYVER: 37
-      BUILD_JOB_NUM: 1
       COVERAGE: 0
 
     - WINPYTHON: "Python38"
-      PYTHON: "3.8"
-      PYVER: 38
-      BUILD_JOB_NUM: 1
       COVERAGE: 0
 
-    # - WINPYTHON: "Python35"
-    #   PYTHON: "3.5"
-    #   PYVER: 35
-    #   BUILD_JOB_NUM: 2
-    #   COVERAGE: 0
-
-    # - WINPYTHON: "Python36"
-    #   PYTHON: "3.6"
-    #   PYVER: 36
-    #   BUILD_JOB_NUM: 2
-    #   COVERAGE: 1
-
-    # - WINPYTHON: "Python37"
-    #   PYTHON: "3.7"
-    #   PYVER: 37
-    #   BUILD_JOB_NUM: 2
-    #   COVERAGE: 0
-
-    # - WINPYTHON: "Python38"
-    #   PYTHON: "3.8"
-    #   PYVER: 38
-    #   BUILD_JOB_NUM: 2
-    #   COVERAGE: 0
-
-
-# remove sets of build jobs based on critia below
+# remove sets of build jobs based on criteria below
 # to fine tune the number and platforms tested
 matrix:
   exclude:
@@ -124,13 +70,7 @@ matrix:
     - image: Visual Studio 2019
       WINPYTHON: "Python36"
 
-    # skip on Ubuntu
-    - image: Ubuntu
-      WINPYTHON: "Python35"
-    - image: Ubuntu
-      WINPYTHON: "Python36"
-
-# remove some binaries we dont to be found
+# remove some binaries we don't want to be found
 before_build:
     - ps: |
         if ($isWindows) {
@@ -144,79 +84,36 @@ before_build:
 build: off
 
 build_script:
-  # get all tests into a list
-  - cmd: "C:\\%WINPYTHON%\\python.exe runtest.py -l -a > all_tests.txt"
-  - sh: python$PYTHON runtest.py -l -a > all_tests.txt
 
   # setup coverage by creating the coverage config file, and adding coverage
-  # to the usercustomize so that all python processes start with coverage
+  # to the sitecustomize so that all python processes start with coverage
   - ps: |
       if ($env:COVERAGE -eq 1) {
         $env:COVERAGE_PROCESS_START = "$($env:APPVEYOR_BUILD_FOLDER)/.coveragerc";
-        $env:PYTHONNOUSERSITE = "";
-        New-Item -ItemType Directory -Force -Path "$($env:PYSITEDIR)";
         $env:COVERAGE_FILE = "$($env:APPVEYOR_BUILD_FOLDER)/.coverage";
-        $usercustomizeText = "import os`r`nos.environ['COVERAGE_PROCESS_START'] = '$($env:COVERAGE_PROCESS_START)'`r`nimport coverage`r`ncoverage.process_startup()";
-        $usercustomizeText|Set-Content "$($env:PYSITEDIR)/usercustomize.py";
-        if ($isWindows) {
-          $coveragercFile = "[run]`r`nsource = $($env:APPVEYOR_BUILD_FOLDER)/SCons`r`nparallel = True`r`ndisable_warnings = trace-changed`r`nomit =`r`n`t*Tests.py`r`n`tsrc\test_*`r`n`tsrc\setup.py`r`n`r`n[path]`r`nsource = $($env:APPVEYOR_BUILD_FOLDER)`r`n[report]`r`nomit =`r`n`t*Tests.py`r`n`tsrc\test_*`r`n`tsrc\setup.py`r`n`r`n"
-        }
-        else
-        {
-          $coveragercFile = "[run]`nsource = $($env:APPVEYOR_BUILD_FOLDER)/SCons`nparallel = True`ndisable_warnings = trace-changed`nomit =`n`t*Tests.py`n`tsrc/test_*`n`tsrc/setup.py`n`n[path]`nsource = $($env:APPVEYOR_BUILD_FOLDER)`n[report]`nomit =`n`t*Tests.py`n`tsrc/test_*`n`tsrc/setup.py`n`n"
-        }
+        New-Item -ItemType Directory -Force -Path "$($env:PYSITEDIR)";
+        $sitecustomizeText = "import os`r`nos.environ['COVERAGE_PROCESS_START'] = '$($env:COVERAGE_PROCESS_START)'`r`nos.environ['COVERAGE_FILE'] = '$($env:COVERAGE_FILE)'`r`nimport coverage`r`ncoverage.process_startup()";
+        $sitecustomizeText|Set-Content "$($env:PYSITEDIR)/sitecustomize.py";
+        Get-Content -Path "$($env:PYSITEDIR)/sitecustomize.py";
+        $coveragercFile = "[run]`r`nsource = $($env:APPVEYOR_BUILD_FOLDER)/SCons`r`nparallel = True`r`ndisable_warnings = trace-changed`r`nomit =`r`n`t*Tests.py`r`n`t*\src\*`r`n`t*\test\*`r`n`t*\testing\*`r`n`t*\template\*`r`n`t*\scripts\*`r`n`t*\scons-time.py`r`n`t*\bootstrap.py`r`n`t*\runtest.py`r`n`t*\setup.py`r`n`r`n[path]`r`nsource = $($env:APPVEYOR_BUILD_FOLDER)`r`n[report]`r`nomit =`r`n`t*Tests.py`r`n`t*\src\*`r`n`t*\test\*`r`n`t*\testing\*`r`n`t*\template\*`r`n`t*\scripts\*`r`n`t*\scons-time.py`r`n`t*\bootstrap.py`r`n`t*\runtest.py`r`n`t*\setup.py`r`n`r`n"
         $coveragercFile|Set-Content "$($env:COVERAGE_PROCESS_START)";
+        Get-Content -Path "$($env:COVERAGE_PROCESS_START)";
       }
 
-  # exclude VS 10.0 because it hangs the testing until this is resolved:
-  # https://help.appveyor.com/discussions/problems/19283-visual-studio-2010-trial-license-has-expired
-  - ps: |
-      New-Item -Name exclude_list.txt -ItemType File
-      $workaround_image = "Visual Studio 2015"
-      if ($env:APPVEYOR_BUILD_WORKER_IMAGE -eq $workaround_image) {
-        Add-Content -Path 'exclude_list.txt' -Value 'test\MSVS\vs-10.0-exec.py'
-      }
 
-  # setup portion of tests for this build job (1-4)
-  - ps: |
-      $TOTAL_BUILD_JOBS = 1;
-      $Lines = (Get-Content all_tests.txt | Measure-Object -line).Lines;
-      $start = ($Lines / $TOTAL_BUILD_JOBS) * ($Env:BUILD_JOB_NUM - 1);
-      $end = ($Lines / $TOTAL_BUILD_JOBS) * $Env:BUILD_JOB_NUM;
-      if ( $Env:BUILD_JOB_NUM -eq $TOTAL_BUILD_JOBS){ $end = $Lines };
-      if ( $start -eq 0 ){ $start = 1 };
-      get-content all_tests.txt | select -first ($end - $start) -skip ($start - 1) | Out-File -Encoding ASCII build_tests.txt;
-
-  # Windows run the tests
-  # NOTE: running powershell from cmd on purpose because it formats the output
-  # correctly
-  - cmd: powershell -Command "& { if($env:COVERAGE -eq 1) { coverage run -p --rcfile=$($env:COVERAGE_PROCESS_START) runtest.py -j 2 -t --exclude-list exclude_list.txt -f build_tests.txt } else { C:\\%WINPYTHON%\\python.exe runtest.py -j 2 -t --exclude-list exclude_list.txt -f build_tests.txt }; if($LastExitCode -eq 2 -Or $LastExitCode -eq 0) { $host.SetShouldExit(0 )} else {$host.SetShouldExit(1)}}"
-
-  # linux run the tests
-  # unset JAVA_TOOL_OPTIONS because newer java prints this to stderr
-  - sh: |
-      unset JAVA_TOOL_OPTIONS
-      if [ "$COVERAGE" -eq "1" ]; then
-        coverage run -p --rcfile="$COVERAGE_PROCESS_START" runtest.py --exclude-list exclude_list.txt -f build_tests.txt || if [[ $? == 2 ]]; then true; else false; fi;
-      else
-        python$PYTHON runtest.py -j 2 --exclude-list exclude_list.txt -f build_tests.txt || if [[ $? == 2 ]]; then true; else false; fi;
-      fi
-
-# run converage even if there was a test failure
+  # NOTE: running powershell from cmd is intended because 
+  # it formats the output correctly
+  - cmd: powershell -Command "& { if($env:COVERAGE -eq 1) { coverage run -p --rcfile=$($env:COVERAGE_PROCESS_START) runtest.py -j 2 -t -a } else { C:\\%WINPYTHON%\\python.exe runtest.py -j 2 -t -a }; if($LastExitCode -eq 2 -Or $LastExitCode -eq 0) { $host.SetShouldExit(0 )} else {$host.SetShouldExit(1)}}"
+
+# run coverage even if there was a test failure
 on_finish:
   - ps: |
       if ($env:COVERAGE -eq 1) {
         & coverage combine
         & coverage report
-        & coverage xml -o coverage_xml.xml
+        & coverage -i xml -o coverage_xml.xml
       }
   # running codecov in powershell causes an error so running in platform
   # shells
   - cmd: if %COVERAGE% equ 1 codecov -X gcov --file coverage_xml.xml
-  - sh: if [ $COVERAGE -eq 1 ]; then codecov -X gcov --file coverage_xml.xml; fi
-    # not using coveralls, so leaving it commented out in case we switch back
-    #- cmd: "C:\\%WINPYTHON%\\python.exe -m pip install --user -U coveralls"
-    #- sh: python$PYTHON -m pip install --user -U coveralls
-    #- ps: coveralls --rcfile="$($env:COVERAGE_PROCESS_START)"
-  # - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
-
+    
diff --git a/.travis.yml b/.travis.yml
index c31b1fc..25da402 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -72,23 +72,25 @@ jobs:
         # setup sitecustomize so we can make all subprocess start coverage
         - export PYSITEDIR=$(python -c "import sys; print(sys.path[-1])")
         - export COVERAGE_PROCESS_START="$PWD/.coveragerc"
+        - export COVERAGE_FILE="$PWD/.coverage"
         - mkdir -p "$PYSITEDIR"
         - echo "$PYSITEDIR"
         - touch "${PYSITEDIR}/sitecustomize.py"
         - echo "import os" |  tee --append "${PYSITEDIR}/sitecustomize.py"
         - echo "os.environ['COVERAGE_PROCESS_START'] = '$PWD/.coveragerc'" | tee --append "${PYSITEDIR}/sitecustomize.py"
+        - echo "os.environ['COVERAGE_FILE'] = '$PWD/.coverage'" | tee --append "${PYSITEDIR}/sitecustomize.py"
         - echo "import coverage" | tee --append "${PYSITEDIR}/sitecustomize.py"
         - echo "coverage.process_startup()" | tee --append "${PYSITEDIR}/sitecustomize.py"
         - cat "${PYSITEDIR}/sitecustomize.py"
         # write the coverage config file
         - echo "[run]" >> "$PWD/.coveragerc"
-        - echo "source = $PWD/SCons" >> "$PWD/.coveragerc"
+        - echo "source = $PWD" >> "$PWD/.coveragerc"
         - echo "parallel = True" >> "$PWD/.coveragerc"
-        - echo "omit = *Tests.py" >> "$PWD/.coveragerc"
+        - printf "omit =\n\t*Tests.py\n\t*/src/*\n\t*/test/*\n\t*/testing/*\n\t*/template/*\n\t*/scripts/*\n\t*/scons-time.py\n\t*/bootstrap.py\n\t*/runtest.py\n\t*/setup.py\n\n" >> "$PWD/.coveragerc"
         - echo "[path]" >> "$PWD/.coveragerc"
         - echo "source = $PWD" >> "$PWD/.coveragerc"
         - echo "[report]" >> "$PWD/.coveragerc"
-        - echo "omit = *Tests.py" >> "$PWD/.coveragerc"
+        - printf "omit =\n\t*Tests.py\n\t*/src/*\n\t*/test/*\n\t*/testing/*\n\t*/template/*\n\t*/scripts/*\n\t*/scons-time.py\n\t*/bootstrap.py\n\t*/runtest.py\n\t*/setup.py\n\n" >> "$PWD/.coveragerc"
         - cat "$PWD/.coveragerc"
 
       script:
@@ -97,5 +99,5 @@ jobs:
       after_script:
         - coverage combine
         - coverage report
-        - coverage xml -o coverage_xml.xml
-        - codecov -X gcov --file coverage_xml.xml
+        - coverage xml -i -o coverage_xml.xml
+        - codecov -X gcov --file coverage_xml.xml
\ No newline at end of file