Merge remote-tracking branch 'origin/master' into change_appendenvpath_default

4 files changed, 319 insertions, 65 deletions

diff --git a/.appveyor.yml b/.appveyor.yml
index 7f45e17..b8f0913 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -1,8 +1,12 @@
 #version: '3.0.1.{build}'
 
-image: Visual Studio 2017
-shallow_clone: true
-
+image:
+  # linux builds disabled for now 
+  # - Ubuntu
+  - Visual Studio 2015
+  - Visual Studio 2017
+  
+  
 cache:
   - downloads -> appveyor.yml
   - '%LOCALAPPDATA%\pip\Cache'
@@ -10,69 +14,176 @@ cache:
   - C:\ProgramData\chocolatey\lib -> appveyor.yml
 
 install:
-  - "set PATH=%PYTHON%;%PYTHON%\\Scripts;C:\\cygwin64\\bin;C:\\msys64;%PATH%"
-  - python --version
-  - pip install pypiwin32
-  - set STATIC_DEPS=true & pip install lxml
-  - choco install --allow-empty-checksums dmd
-  - choco install --allow-empty-checksums ldc
-  - choco install --allow-empty-checksums swig
-  - choco install --allow-empty-checksums vswhere
-  - choco install --allow-empty-checksums xsltproc
+  ### WINDOWS ###
+  # add python and python user-base to path for pip installs
+  - cmd: "C:\\%WINPYTHON%\\python.exe --version"
+  - cmd: for /F "tokens=*" %%g in ('C:\\%WINPYTHON%\\python.exe -m site --user-site') do (set PYSITEDIR=%%g)
+  - cmd: "set PATH=C:\\%WINPYTHON%;C:\\%WINPYTHON%\\Scripts;C:\\cygwin64\\bin;C:\\msys64;C:\\ProgramData\\chocolatey\\bin;%PATH%"
+  - cmd: "C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pip setuptools wheel "
+  - cmd: "C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pypiwin32 coverage codecov"
+  - cmd: set STATIC_DEPS=true & C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off lxml
+  # install 3rd party tools to test with
+  - cmd: choco install --allow-empty-checksums dmd ldc swig vswhere xsltproc
+  - cmd: set
+
+  ### LINUX ###
+  - sh: sudo add-apt-repository -y ppa:deadsnakes/ppa
+  # allow the CI to continue even if some pkg in the pkglist may not be up to date
+  - sh: sudo apt-get update || true
+  - sh: sudo apt-get -y install python$PYTHON
+  - sh: wget https://bootstrap.pypa.io/get-pip.py
+  - sh: sudo -H python$PYTHON get-pip.py
+  - sh: which python$PYTHON
+  - sh: python$PYTHON --version
+  - sh: export PYSITEDIR=$(python$PYTHON -m site --user-site)
+  - sh: python$PYTHON -m pip install --user -U --progress-bar off pip setuptools wheel 
+  - sh: python$PYTHON -m pip install --user -U --progress-bar off coverage codecov
+  - sh: STATIC_DEPS=true python$PYTHON -m pip install --user -U --progress-bar off lxml
+  - sh: ./.travis/install.sh
+  - sh: printenv
 
 
 environment:
   matrix:
-    - PYTHON: "C:\\Python27"
+    - WINPYTHON: "Python27"
+      PYTHON: "2.7"
+      PYVER: 27
       BUILD_JOB_NUM: 1
-    - PYTHON: "C:\\Python27"
+      COVERAGE: 1
+    - WINPYTHON: "Python27"
+      PYTHON: "2.7"
+      PYVER: 27
       BUILD_JOB_NUM: 2
-    - PYTHON: "C:\\Python27"
+      COVERAGE: 1
+    - WINPYTHON: "Python27"
+      PYTHON: "2.7"
+      PYVER: 27
       BUILD_JOB_NUM: 3
-    - PYTHON: "C:\\Python27"
+      COVERAGE: 1
+    - WINPYTHON: "Python27"
+      PYTHON: "2.7"
+      PYVER: 27
       BUILD_JOB_NUM: 4
+      COVERAGE: 1
 
-    - PYTHON: "C:\\Python35"
+    - WINPYTHON: "Python36"
+      PYTHON: "3.6"
+      PYVER: 36
       BUILD_JOB_NUM: 1
-    - PYTHON: "C:\\Python35"
+      COVERAGE: 1
+    - WINPYTHON: "Python36"
+      PYTHON: "3.6"
+      PYVER: 36
       BUILD_JOB_NUM: 2
-    - PYTHON: "C:\\Python35"
+      COVERAGE: 1
+    - WINPYTHON: "Python36"
+      PYTHON: "3.6"
+      PYVER: 36
       BUILD_JOB_NUM: 3
-    - PYTHON: "C:\\Python35"
+      COVERAGE: 1
+    - WINPYTHON: "Python36"
+      PYTHON: "3.6"
+      PYVER: 36
       BUILD_JOB_NUM: 4
+      COVERAGE: 1
 
-    - PYTHON: "C:\\Python36"
+    - WINPYTHON: "Python35"
+      PYTHON: "3.5"
+      PYVER: 35
       BUILD_JOB_NUM: 1
-    - PYTHON: "C:\\Python36"
+      COVERAGE: 0
+    - WINPYTHON: "Python35"
+      PYTHON: "3.5"
+      PYVER: 35
       BUILD_JOB_NUM: 2
-    - PYTHON: "C:\\Python36"
+      COVERAGE: 0
+    - WINPYTHON: "Python35"
+      PYTHON: "3.5"
+      PYVER: 35
       BUILD_JOB_NUM: 3
-    - PYTHON: "C:\\Python36"
+      COVERAGE: 0
+    - WINPYTHON: "Python35"
+      PYTHON: "3.5"
+      PYVER: 35
       BUILD_JOB_NUM: 4
+      COVERAGE: 0
 
-    - PYTHON: "C:\\Python37"
+    - WINPYTHON: "Python37"
+      PYTHON: "3.7"
+      PYVER: 37
       BUILD_JOB_NUM: 1
-    - PYTHON: "C:\\Python37"
+      COVERAGE: 0
+    - WINPYTHON: "Python37"
+      PYTHON: "3.7"
+      PYVER: 37
       BUILD_JOB_NUM: 2
-    - PYTHON: "C:\\Python37"
+      COVERAGE: 0
+    - WINPYTHON: "Python37"
+      PYTHON: "3.7"
+      PYVER: 37
       BUILD_JOB_NUM: 3
-    - PYTHON: "C:\\Python37"
+      COVERAGE: 0
+    - WINPYTHON: "Python37"
+      PYTHON: "3.7"
+      PYVER: 37
       BUILD_JOB_NUM: 4
+      COVERAGE: 0
+
+matrix:	
+  exclude:	
+    # test python 3.6 on Visual Studio 2015 image
+    - image: Visual Studio 2015	
+      WINPYTHON: "Python37"
+    - image: Visual Studio 2015	
+      WINPYTHON: "Python35"
 
+    # test python 3.7 on Ubuntu
+    - image: Ubuntu	
+      WINPYTHON: "Python27"	
+    - image: Ubuntu		
+      WINPYTHON: "Python35"
+    - image: Ubuntu		
+      WINPYTHON: "Python36"
 
 before_build:
-    - ps: dir "C:\Program Files\Git\usr\bin\x*.exe"
     - ps: |
-        if (Test-Path "C:\Program Files\Git\usr\bin\xsltproc.EXE" ) {
-          Remove-Item  "C:\Program Files\Git\usr\bin\xsltproc.EXE" -ErrorAction Ignore
+        if ($isWindows) 
+        {
+          dir "C:\Program Files\Git\usr\bin\x*.exe"
+          if (Test-Path "C:\Program Files\Git\usr\bin\xsltproc.EXE" ) {
+            Remove-Item  "C:\Program Files\Git\usr\bin\xsltproc.EXE" -ErrorAction Ignore
+          }
+          dir "C:\Program Files\Git\usr\bin\x*.exe"
         }
-    - ps: dir "C:\Program Files\Git\usr\bin\x*.exe"
-
+  
 build: off
 
 build_script:
-  - cmd: python runtest.py -l -a > all_tests.txt
-  - ps: >-
+  # get tests all tests
+  - cmd: "C:\\%WINPYTHON%\\python.exe runtest.py -l -a > all_tests.txt"
+  - sh: python$PYTHON runtest.py -l -a > all_tests.txt
+
+  # setup coverage is set in environment
+  - ps: |
+      if ($env:COVERAGE -eq 1){ 
+        $env:COVERAGE_PROCESS_START = "$($env:APPVEYOR_BUILD_FOLDER)/.coveragerc";
+        $env:PYTHONNOUSERSITE = "";
+        New-Item -ItemType Directory -Force -Path "$($env:PYSITEDIR)";
+        $env:COVERAGE_FILE = "$($env:APPVEYOR_BUILD_FOLDER)/.coverage";
+        $usercustomizeText = "import os`r`nos.environ['COVERAGE_PROCESS_START'] = '$($env:COVERAGE_PROCESS_START)'`r`nimport coverage`r`ncoverage.process_startup()";
+        $usercustomizeText|Set-Content "$($env:PYSITEDIR)/usercustomize.py";
+        if ($isWindows) 
+        {
+          $coveragercFile = "[run]`r`nsource = $($env:APPVEYOR_BUILD_FOLDER)/src`r`nparallel = True`r`ndisable_warnings = trace-changed`r`nomit =`r`n`t*Tests.py`r`n`tsrc\test_*`r`n`tsrc\setup.py`r`n`r`n[path]`r`nsource = $($env:APPVEYOR_BUILD_FOLDER)`r`n[report]`r`nomit =`r`n`t*Tests.py`r`n`tsrc\test_*`r`n`tsrc\setup.py`r`n`r`n"
+        }
+        else
+        {
+          $coveragercFile = "[run]`nsource = $($env:APPVEYOR_BUILD_FOLDER)/src`nparallel = True`ndisable_warnings = trace-changed`nomit =`n`t*Tests.py`n`tsrc/test_*`n`tsrc/setup.py`n`n[path]`nsource = $($env:APPVEYOR_BUILD_FOLDER)`n[report]`nomit =`n`t*Tests.py`n`tsrc/test_*`n`tsrc/setup.py`n`n"
+        }
+        $coveragercFile|Set-Content "$($env:COVERAGE_PROCESS_START)";
+      }
+  # setup portion of tests for this build job (1-4)
+  - ps: |
       $TOTAL_BUILD_JOBS = 4;
       $Lines = (Get-Content all_tests.txt | Measure-Object -line).Lines;
       $start = ($Lines / $TOTAL_BUILD_JOBS) * ($Env:BUILD_JOB_NUM - 1);
@@ -80,4 +191,43 @@ build_script:
       if ( $Env:BUILD_JOB_NUM -eq $TOTAL_BUILD_JOBS){ $end = $Lines };
       if ( $start -eq 0 ){ $start = 1 };
       get-content all_tests.txt | select -first ($end - $start) -skip ($start - 1) | Out-File -Encoding ASCII build_tests.txt;
-  - cmd: powershell -Command "& { python runtest.py -j 2 -f build_tests.txt; if($LastExitCode -eq 2 -Or $LastExitCode -eq 0) { $host.SetShouldExit(0 )} else {$host.SetShouldExit(1)}}"
\ No newline at end of file
+
+  # exclude VS 10.0 because it hangs the testing until this is resolved: 
+  # https://help.appveyor.com/discussions/problems/19283-visual-studio-2010-trial-license-has-expired
+  - ps: |
+      New-Item -Name exclude_list.txt -ItemType File
+      $workaround_image = "Visual Studio 2015"
+      if ($env:APPVEYOR_BUILD_WORKER_IMAGE -eq $workaround_image){ 
+        Add-Content -Path 'exclude_list.txt' -Value 'test\MSVS\vs-10.0-exec.py'
+      }
+
+  # NOTE: running powershell from cmd on purpose because it formats the output
+  # correctly
+  - cmd: powershell -Command "& { if($env:COVERAGE -eq 1) { coverage run -p --rcfile=$($env:COVERAGE_PROCESS_START) runtest.py --exclude-list exclude_list.txt -f build_tests.txt } else { C:\\%WINPYTHON%\\python.exe runtest.py -j 2 --exclude-list exclude_list.txt -f build_tests.txt }; if($LastExitCode -eq 2 -Or $LastExitCode -eq 0) { $host.SetShouldExit(0 )} else {$host.SetShouldExit(1)}}"
+
+  # unset JAVA_TOOL_OPTIONS because newer java prints this to stderr
+  - sh: |
+      unset JAVA_TOOL_OPTIONS
+      if [ "$COVERAGE" -eq "1" ]; then 
+        coverage run -p --rcfile="$COVERAGE_PROCESS_START" runtest.py --exclude-list exclude_list.txt -f build_tests.txt || if [[ $? == 2 ]]; then true; else false; fi; 
+      else
+        python$PYTHON runtest.py -j 2 --exclude-list exclude_list.txt -f build_tests.txt || if [[ $? == 2 ]]; then true; else false; fi; 
+      fi
+
+  # run converage 
+on_finish:
+  - ps: |
+      if ($env:COVERAGE -eq 1)
+      { 
+        & coverage combine
+        & coverage report
+        & coverage xml -o coverage_xml.xml
+      }
+  # running in powershell causes an error so running in platform
+  # shells
+  - cmd: if %COVERAGE% equ 1 codecov -X gcov --file coverage_xml.xml
+  - sh: if [ $COVERAGE -eq 1 ]; then codecov -X gcov --file coverage_xml.xml; fi
+    # not using coveralls, so leaving it commented out in case we switch back
+    #- cmd: "C:\\%WINPYTHON%\\python.exe -m pip install --user -U coveralls"
+    #- sh: python$PYTHON -m pip install --user -U coveralls 
+    #- ps: coveralls --rcfile="$($env:COVERAGE_PROCESS_START)"
diff --git a/.codecov.yml b/.codecov.yml
new file mode 100644
index 0000000..375b10f
--- /dev/null
+++ b/.codecov.yml
@@ -0,0 +1,50 @@
+codecov:
+  notify:
+    # calculate coverge even when we fail
+    require_ci_to_pass: no
+
+ignore:
+  # ignore test files in the source
+  # this is redundant and should not be in the report anyways
+  # because the coveragerc file ignores them
+  - "*Test.py"
+  - "setup.py"
+  - "test_*"
+
+coverage:
+  precision: 2
+  round: down
+  range: "70...100"
+
+  notify:
+    irc:
+      default:
+        server: "chat.freenode.net#scons"
+        branches: master
+        threshold: null
+        message: "Coverage {{changed}} for {{owner}}/{{repo}}"  # customize the message
+        flags: null
+        paths: null
+
+  status:
+    project:
+      default:
+        # compare against the current coverage 
+        # that PR is attempt to merge to
+        # don't consider a drop in coverage success
+        target: auto
+        threshold: null
+        base: pr
+
+    patch:
+      default:
+        # considering only the lines changed 
+        # make sure all new lines in the PR are covered
+        # to consider a success
+        target: 100
+        threshold: null
+        base: pr
+
+    changes: no
+
+comment: off
diff --git a/.travis.yml b/.travis.yml
index e279ad9..fb48776 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,7 +17,7 @@ install:
 # pypy is not passing, but allow failures for coverage stage to be reached
 matrix:
   allow_failures:
-    - python: pypy
+    - stage: Test
 
 jobs:
   include:
@@ -27,46 +27,63 @@ jobs:
       before_script: skip
       after_success: skip
       python: 2.7
-      env: PYVER=27
+      env: 
+        - PYVER=27
+        - PYTHON=2.7
       sudo: required
-
+      
     - <<: *test_job
       python: 3.5
-      env: PYVER=35
+      env: 
+        - PYVER=35
+        - PYTHON=3.5
       sudo: required
 
     - <<: *test_job
       python: 3.6
-      env: PYVER=36
+      env: 
+        - PYVER=36
+        - PYTHON=3.6
       sudo: required
 
     - <<: *test_job
       python: 3.7
       env: 
         - PYVER=37
+        - PYTHON=3.7
       sudo: required
       dist: xenial  # required for Python 3.7 (travis-ci/travis-ci#9069)
 
     - <<: *test_job
       python: pypy
-      env: PYVER=pypy
+      env: 
+        - PYVER=pypy
+        - PYTHON=pypy
       sudo: required
 
 
     - &coverage_jobs
       stage: Coverage
-      
+      python: 2.7
       before_script:
-        - sudo pip install coverage
-        - sudo pip install coveralls
+        # install our own python so we can modify usercustomize.py
+        - deactivate
+        - sudo add-apt-repository -y ppa:deadsnakes/ppa
+        - sudo apt-get update || true
+        - sudo apt-get -y install python$PYTHON
+        - wget https://bootstrap.pypa.io/get-pip.py
+        - sudo -H python$PYTHON get-pip.py
+        - which python$PYTHON
+        - python$PYTHON --version
+        - python$PYTHON -m pip install --user -U coverage codecov
         # set this ensure user sites are available
         - export PYTHONNOUSERSITE=
         # attempt to get a location where we can store the usercustomize.py file
-        - python -m site
-        - export PYSITEDIR=$(python -m site --user-site)
+        - python$PYTHON -m site
+        - export PYSITEDIR=$(python$PYTHON -m site --user-site)
         - sudo mkdir -p $PYSITEDIR
         - sudo touch ${PYSITEDIR}/usercustomize.py
-        - export COVERAGE_FILE=$PWD/.coverage_file
+        - export COVERAGE_FILE=$PWD/.coverage
         # write the usercustomize.py file so all python processes use coverage and know where the config file is
         - echo "import os" | sudo tee --append ${PYSITEDIR}/usercustomize.py
         - echo "os.environ['COVERAGE_PROCESS_START'] = '$PWD/.coveragerc'" | sudo tee --append ${PYSITEDIR}/usercustomize.py
@@ -74,42 +91,79 @@ jobs:
         - echo "coverage.process_startup()" | sudo tee --append ${PYSITEDIR}/usercustomize.py
 
       script: 
-        - export TOTAL_BUILD_JOBS=8
+        - export TOTAL_BUILD_JOBS=4
         # write the coverage config file
         - export COVERAGE_PROCESS_START=$PWD/.coveragerc
         - echo "[run]" >> .coveragerc
         - echo "source = $PWD/src" >> .coveragerc
         - echo "parallel = True" >> .coveragerc
         - printf "omit =\n\t*Tests.py\n\tsrc/test_*\n\tsrc/setup.py\n\n" >> .coveragerc
-        - echo "[path] = $PWD" >> .coveragerc
+        - echo "[path]" >> .coveragerc
+        - echo "source = $PWD" >> .coveragerc
+        - echo "[report]" >> .coveragerc
+        - printf "omit =\n\t*Tests.py\n\tsrc/test_*\n\tsrc/setup.py\n\n" >> .coveragerc
         # get a list of all the tests to split them up
-        - python runtest.py -l -a > all_tests
+        - python$PYTHON runtest.py -l -a > all_tests
         - let "start = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * (${BUILD_JOB_NUM} - 1)"; true;
         - let "end = ($(wc -l < all_tests) / ${TOTAL_BUILD_JOBS}) * ${BUILD_JOB_NUM}"
         - if (( ${BUILD_JOB_NUM} == ${TOTAL_BUILD_JOBS} )); then end=$(wc -l < all_tests); fi
         - if (( ${start} == 0 )); then start=1; fi
         - sed -n ${start},${end}p all_tests > build_tests
         - coverage run -p --rcfile=$PWD/.coveragerc runtest.py -f build_tests || if [[ $? == 2 ]]; then true; else false; fi
-
-      after_success:
+        
+      after_script:
         - coverage combine
         - coverage report
-        - coveralls --rcfile=$PWD/.coveragerc
+        - coverage xml -o coverage_xml.xml
+        - codecov -X gcov --file coverage_xml.xml
+        # not using coveralls but leaving it commented to
+        # make it easy to re-enable
+        #- python$PYTHON -m pip install --user -U coveralls
+        #- coveralls --rcfile=$PWD/.coveragerc
 
-      env: BUILD_JOB_NUM=1
+      env: 
+        - PYVER=27
+        - PYTHON=2.7
+        - BUILD_JOB_NUM=1
 
     - <<: *coverage_jobs
-      env: BUILD_JOB_NUM=2
+      env: 
+        - PYVER=27
+        - PYTHON=2.7
+        - BUILD_JOB_NUM=2
     - <<: *coverage_jobs
-      env: BUILD_JOB_NUM=3
+      env: 
+        - PYVER=27
+        - PYTHON=2.7
+        - BUILD_JOB_NUM=3
     - <<: *coverage_jobs
-      env: BUILD_JOB_NUM=4
+      env: 
+        - PYVER=27
+        - PYTHON=2.7
+        - BUILD_JOB_NUM=4
+
     - <<: *coverage_jobs
-      env: BUILD_JOB_NUM=5
+      python: 3.6
+      env: 
+        - PYVER=36
+        - PYTHON=3.6
+        - BUILD_JOB_NUM=1
     - <<: *coverage_jobs
-      env: BUILD_JOB_NUM=6
+      python: 3.6
+      env: 
+        - PYVER=36
+        - PYTHON=3.6
+        - BUILD_JOB_NUM=2
     - <<: *coverage_jobs
-      env: BUILD_JOB_NUM=7
+      python: 3.6
+      env: 
+        - PYVER=36
+        - PYTHON=3.6
+        - BUILD_JOB_NUM=3
     - <<: *coverage_jobs
-      env: BUILD_JOB_NUM=8
+      python: 3.6
+      env: 
+        - PYVER=36
+        - PYTHON=3.6
+        - BUILD_JOB_NUM=4
 
diff --git a/README.rst b/README.rst
index 17bbffa..cffd4f3 100755
--- a/README.rst
+++ b/README.rst
@@ -21,9 +21,9 @@ SCons - a software construction tool
    :target: https://ci.appveyor.com/project/SCons/scons
    :alt: AppVeyor CI build Status
    
-.. image:: https://coveralls.io/repos/github/SCons/scons/badge.svg?branch=master
-   :target: https://coveralls.io/github/SCons/scons?branch=master
-   :alt: Coveralls.io Coverage Status
+.. image:: https://codecov.io/gh/SCons/scons/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/SCons/scons
+   :alt: CodeCov Coverage Status
 
 
 Welcome to the SCons development tree.  The real purpose of this tree is to