bpo-35683: Improve Azure Pipelines steps (GH-11493)

13 files changed, 148 insertions, 166 deletions

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index 78075bc..15a83dd 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -2,6 +2,11 @@ variables:
   manylinux: false
   coverage: false
 
+resources:
+  containers:
+  - container: manylinux1
+    image: pyca/cryptography-manylinux1:x86_64
+
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
@@ -54,10 +59,12 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.0g
+    openssl_version: 1.1.0j
 
   steps:
   - template: ./posix-steps.yml
+    parameters:
+      dependencies: apt
 
 
 - job: ManyLinux1_CI_Tests
@@ -75,13 +82,20 @@ jobs:
   pool:
     vmImage: ubuntu-16.04
 
+  container: manylinux1
+
   variables:
     testRunTitle: '$(build.sourceBranchName)-manylinux1'
     testRunPlatform: manylinux1
-    imageName: 'dockcross/manylinux-x64'
+    openssl_version: ''
 
   steps:
-  - template: ./docker-steps.yml
+  - template: ./posix-steps.yml
+    parameters:
+      dependencies: yum
+      sudo_dependencies: ''
+      xvfb: false
+      patchcheck: false
 
 
 - job: Ubuntu_Coverage_CI_Tests
@@ -102,11 +116,12 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.0g
+    openssl_version: 1.1.0j
 
   steps:
   - template: ./posix-steps.yml
     parameters:
+      dependencies: apt
       coverage: true
 
 
@@ -144,3 +159,4 @@ jobs:
   - template: ./windows-layout-steps.yml
     parameters:
       kind: appx
+      fulltest: true
diff --git a/.azure-pipelines/docker-steps.yml b/.azure-pipelines/docker-steps.yml
deleted file mode 100644
index ba4dfd7..0000000
--- a/.azure-pipelines/docker-steps.yml
+++ /dev/null
@@ -1,76 +0,0 @@
-steps:
-- checkout: self
-  clean: true
-  fetchDepth: 5
-
-- ${{ if ne(parameters.targetBranch, '') }}:
-  - script: |
-     git fetch -q origin ${{ parameters.targetbranch }}
-     if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)'
-     then
-       echo "Only docs were updated, stopping build process."
-       echo "##vso[task.setvariable variable=DocOnly]true"
-       exit
-     fi
-    displayName: Detect doc-only changes
-
-- task: docker@0
-  displayName: 'Configure CPython (debug)'
-  inputs:
-    action: 'Run an image'
-    imageName: $(imageName)
-    volumes: |
-      $(build.sourcesDirectory):/src
-      $(build.binariesDirectory):/build
-    workDir: '/src'
-    containerCommand: './configure --with-pydebug'
-    detached: false
-  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: docker@0
-  displayName: 'Build CPython'
-  inputs:
-    action: 'Run an image'
-    imageName: $(imageName)
-    volumes: |
-      $(build.sourcesDirectory):/src
-      $(build.binariesDirectory):/build
-    workDir: '/src'
-    containerCommand: 'make -s -j4'
-    detached: false
-  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: docker@0
-  displayName: 'Display build info'
-  inputs:
-    action: 'Run an image'
-    imageName: $(imageName)
-    volumes: |
-      $(build.sourcesDirectory):/src
-      $(build.binariesDirectory):/build
-    workDir: '/src'
-    containerCommand: 'make pythoninfo'
-    detached: false
-  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: docker@0
-  displayName: 'Tests'
-  inputs:
-    action: 'Run an image'
-    imageName: $(imageName)
-    volumes: |
-      $(build.sourcesDirectory):/src
-      $(build.binariesDirectory):/build
-    workDir: '/src'
-    containerCommand: 'make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=/build/test-results.xml"'
-    detached: false
-  condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
-
-- task: PublishTestResults@2
-  displayName: 'Publish Test Results'
-  inputs:
-    testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
-    mergeTestResults: true
-    testRunTitle: $(testRunTitle)
-    platform: $(testRunPlatform)
-  condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))
diff --git a/.azure-pipelines/posix-deps.sh b/.azure-pipelines/posix-deps-apt.sh
index a572107..4f48990 100755
--- a/.azure-pipelines/posix-deps.sh
+++ b/.azure-pipelines/posix-deps-apt.sh
@@ -1,6 +1,6 @@
-sudo apt-get update
+apt-get update
 
-sudo apt-get -yq install \
+apt-get -yq install \
     build-essential \
     zlib1g-dev \
     libbz2-dev \
diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml
index 6e2606f..2affb50 100644
--- a/.azure-pipelines/posix-steps.yml
+++ b/.azure-pipelines/posix-steps.yml
@@ -1,12 +1,16 @@
 parameters:
   coverage: false
+  sudo_dependencies: sudo
+  dependencies: apt
+  patchcheck: true
+  xvfb: true
 
 steps:
 - checkout: self
   clean: true
   fetchDepth: 5
 
-- script: ./.azure-pipelines/posix-deps.sh $(openssl_version)
+- script: ${{ parameters.sudo_dependencies }} ./.azure-pipelines/posix-deps-${{ parameters.dependencies }}.sh $(openssl_version)
   displayName: 'Install dependencies'
 
 - script: ./configure --with-pydebug
@@ -23,7 +27,7 @@ steps:
     displayName: 'Display build info'
 
   - script: |
-      xvfb-run ./venv/bin/python -m coverage run --pylib -m test \
+      $COMMAND -m coverage run --pylib -m test \
                 --fail-env-changed \
                 -uall,-cpu \
                 --junit-xml=$(build.binariesDirectory)/test-results.xml \
@@ -32,6 +36,11 @@ steps:
                 -x test_multiprocessing_spawn \
                 -x test_concurrent_futures
     displayName: 'Tests with coverage'
+    env:
+      ${{ if eq(parameters.xvfb, 'true') }}:
+        COMMAND: xvfb-run ./venv/bin/python
+      ${{ if ne(parameters.xvfb, 'true') }}:
+        COMMAND: ./venv/bin/python
 
   - script: ./venv/bin/python -m coverage xml
     displayName: 'Generate coverage.xml'
@@ -44,13 +53,18 @@ steps:
   - script: make pythoninfo
     displayName: 'Display build info'
 
-  - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
+  - script: $COMMAND buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
     displayName: 'Tests'
-
-
-- script: ./python Tools/scripts/patchcheck.py --travis true
-  displayName: 'Run patchcheck.py'
-  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
+    env:
+      ${{ if eq(parameters.xvfb, 'true') }}:
+        COMMAND: xvfb-run make
+      ${{ if ne(parameters.xvfb, 'true') }}:
+        COMMAND: make
+
+- ${{ if eq(parameters.patchcheck, 'true') }}:
+  - script: ./python Tools/scripts/patchcheck.py --travis true
+    displayName: 'Run patchcheck.py'
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
 
 
 - task: PublishTestResults@2
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
index 2d7fba9..0bd7921 100644
--- a/.azure-pipelines/pr.yml
+++ b/.azure-pipelines/pr.yml
@@ -1,3 +1,12 @@
+variables:
+  manylinux: false
+  coverage: false
+
+resources:
+  containers:
+  - container: manylinux1
+    image: pyca/cryptography-manylinux1:x86_64
+
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
@@ -50,12 +59,70 @@ jobs:
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.0g
+    openssl_version: 1.1.0j
 
   steps:
   - template: ./posix-steps.yml
     parameters:
-      targetBranch: $(System.PullRequest.TargetBranch)
+      dependencies: apt
+
+
+- job: ManyLinux1_PR_Tests
+  displayName: ManyLinux1 PR Tests
+  dependsOn: Prebuild
+  condition: |
+    and(
+        and(
+            succeeded(),
+            eq(variables['manylinux'], 'true')
+        ),
+        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+    )
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  container: manylinux1
+
+  variables:
+    testRunTitle: '$(system.pullRequest.TargetBranch)-manylinux1'
+    testRunPlatform: manylinux1
+    openssl_version: ''
+
+  steps:
+  - template: ./posix-steps.yml
+    parameters:
+      dependencies: yum
+      sudo_dependencies: ''
+      xvfb: false
+      patchcheck: false
+
+
+- job: Ubuntu_Coverage_PR_Tests
+  displayName: Ubuntu PR Tests (coverage)
+  dependsOn: Prebuild
+  condition: |
+    and(
+        and(
+            succeeded(),
+            eq(variables['coverage'], 'true')
+        ),
+        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+    )
+
+  pool:
+    vmImage: ubuntu-16.04
+
+  variables:
+    testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
+    testRunPlatform: linux-coverage
+    openssl_version: 1.1.0j
+
+  steps:
+  - template: ./posix-steps.yml
+    parameters:
+      dependencies: apt
+      coverage: true
 
 
 - job: Windows_PR_Tests
diff --git a/.azure-pipelines/windows-appx-test.yml b/.azure-pipelines/windows-appx-test.yml
deleted file mode 100644
index cad752b..0000000
--- a/.azure-pipelines/windows-appx-test.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-jobs:
-- job: Prebuild
-  displayName: Pre-build checks
-
-  pool:
-    vmImage: ubuntu-16.04
-
-  steps:
-  - template: ./prebuild-checks.yml
-
-
-- job: Windows_Appx_Tests
-  displayName: Windows Appx Tests
-  dependsOn: Prebuild
-  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
-
-  pool:
-    vmImage: vs2017-win2016
-
-  strategy:
-    matrix:
-      win64:
-        arch: amd64
-        buildOpt: '-p x64'
-        testRunTitle: '$(Build.SourceBranchName)-win64-appx'
-        testRunPlatform: win64
-    maxParallel: 2
-
-  steps:
-  - checkout: self
-    clean: true
-    fetchDepth: 5
-
-  - powershell: |
-      # Relocate build outputs outside of source directory to make cleaning faster
-      Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj'
-      # UNDONE: Do not build to a different directory because of broken tests
-      Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild'
-      Write-Host '##vso[task.setvariable variable=EXTERNALS_DIR]$(Build.BinariesDirectory)\externals'
-    displayName: Update build locations
-
-  - script: PCbuild\build.bat -e $(buildOpt)
-    displayName: 'Build CPython'
-    env:
-      IncludeUwp: true
-
-  - script: python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-$(arch)" --copy "$(Py_IntDir)\layout-$(arch)" --precompile --preset-appx --include-tests
-    displayName: 'Create APPX layout'
-
-  - script: .\python.exe -m test.pythoninfo
-    workingDirectory: $(Py_IntDir)\layout-$(arch)
-    displayName: 'Display build info'
-
-  - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir "$(Py_IntDir)\tmp-$(arch)"
-    workingDirectory: $(Py_IntDir)\layout-$(arch)
-    displayName: 'Tests'
-    env:
-      PREFIX: $(Py_IntDir)\layout-$(arch)
-
-  - task: PublishTestResults@2
-    displayName: 'Publish Test Results'
-    inputs:
-      testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml'
-      mergeTestResults: true
-      testRunTitle: $(testRunTitle)
-      platform: $(testRunPlatform)
-    condition: succeededOrFailed()
diff --git a/.azure-pipelines/windows-layout-steps.yml b/.azure-pipelines/windows-layout-steps.yml
index 62e5259..e15729f 100644
--- a/.azure-pipelines/windows-layout-steps.yml
+++ b/.azure-pipelines/windows-layout-steps.yml
@@ -1,11 +1,28 @@
 parameters:
   kind: nuget
   extraOpts: --precompile
+  fulltest: false
 
 steps:
-- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-${{ parameters['kind'] }}-$(arch)" --copy "$(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)" ${{ parameters['extraOpts'] }} --preset-${{ parameters['kind'] }} --include-tests
-  displayName: Create ${{ parameters['kind'] }} layout
+- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Build.BinariesDirectory)\layout-tmp-${{ parameters.kind }}-$(arch)" --copy "$(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)" ${{ parameters.extraOpts }} --preset-${{ parameters.kind }} --include-tests
+  displayName: Create ${{ parameters.kind }} layout
 
 - script: .\python.exe -m test.pythoninfo
-  workingDirectory: $(Py_OutDir)\layout-${{ parameters['kind'] }}-$(arch)
-  displayName: Show layout info (${{ parameters['kind'] }})
+  workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)
+  displayName: Show layout info (${{ parameters.kind }})
+
+- ${{ if eq(parameters.fulltest, 'true') }}:
+  - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml" --tempdir "$(Build.BinariesDirectory)\tmp-${{ parameters.kind }}-$(arch)"
+    workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)
+    displayName: ${{ parameters.kind }} Tests
+    env:
+      PREFIX: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)
+
+  - task: PublishTestResults@2
+    displayName: Publish ${{ parameters.kind }} Test Results
+    inputs:
+      testResultsFiles: $(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml
+      mergeTestResults: true
+      testRunTitle: ${{ parameters.kind }}-$(testRunTitle)
+      platform: $(testRunPlatform)
+    condition: succeededOrFailed()
diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml
index 3651ae0..794a23a 100644
--- a/.azure-pipelines/windows-steps.yml
+++ b/.azure-pipelines/windows-steps.yml
@@ -1,6 +1,6 @@
 steps:
 - checkout: self
-  clean: true
+  clean: false
   fetchDepth: 5
 
 - powershell: |
@@ -8,6 +8,7 @@ steps:
     Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj'
     # UNDONE: Do not build to a different directory because of broken tests
     Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild'
+    #Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.BinariesDirectory)\bin'
     Write-Host '##vso[task.setvariable variable=EXTERNALS_DIR]$(Build.BinariesDirectory)\externals'
   displayName: Update build locations
 
diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py
index 5839b5d..7c148d2 100644
--- a/Lib/idlelib/idle_test/test_help_about.py
+++ b/Lib/idlelib/idle_test/test_help_about.py
@@ -61,6 +61,8 @@ class LiveDialogTest(unittest.TestCase):
                 button.invoke()
                 get = dialog._current_textview.viewframe.textframe.text.get
                 lines = printer._Printer__lines
+                if len(lines) < 2:
+                    self.fail(name + ' full text was not found')
                 self.assertEqual(lines[0], get('1.0', '1.end'))
                 self.assertEqual(lines[1], get('2.0', '2.end'))
                 dialog._current_textview.destroy()
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 8d44caf..32ac440 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -1,5 +1,6 @@
 import datetime
 import faulthandler
+import json
 import locale
 import os
 import platform
@@ -565,6 +566,9 @@ class Regrtest:
 
         if self.ns.tempdir:
             TEMPDIR = self.ns.tempdir
+        elif self.ns.worker_args:
+            ns_dict, _ = json.loads(self.ns.worker_args)
+            TEMPDIR = ns_dict.get("tempdir") or TEMPDIR
 
         os.makedirs(TEMPDIR, exist_ok=True)
 
diff --git a/Lib/test/test_symbol.py b/Lib/test/test_symbol.py
index ed86aec..645d8f4 100644
--- a/Lib/test/test_symbol.py
+++ b/Lib/test/test_symbol.py
@@ -2,6 +2,7 @@ import unittest
 from test import support
 import os
 import sys
+import sysconfig
 import subprocess
 
 
@@ -38,8 +39,8 @@ class TestSymbolGeneration(unittest.TestCase):
             lines2 = fp.readlines()
         self.assertEqual(lines1, lines2)
 
-    @unittest.skipIf(not os.path.exists(GRAMMAR_FILE),
-                     'test only works from source build directory')
+    @unittest.skipUnless(sysconfig.is_python_build(),
+                         'test only works from source build directory')
     def test_real_grammar_and_symbol_file(self):
         output = support.TESTFN
         self.addCleanup(support.unlink, output)
diff --git a/Misc/NEWS.d/next/Build/2019-01-10-11-37-18.bpo-35683.pf5Oos.rst b/Misc/NEWS.d/next/Build/2019-01-10-11-37-18.bpo-35683.pf5Oos.rst
new file mode 100644
index 0000000..f396101
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-01-10-11-37-18.bpo-35683.pf5Oos.rst
@@ -0,0 +1 @@
+Improved Azure Pipelines build steps and now verifying layouts correctly
diff --git a/PC/layout/main.py b/PC/layout/main.py
index d372fe5..910085c 100644
--- a/PC/layout/main.py
+++ b/PC/layout/main.py
@@ -156,6 +156,8 @@ def get_layout(ns):
     for dest, src in rglob(ns.build, "vcruntime*.dll"):
         yield dest, src
 
+    yield "LICENSE.txt", ns.source / "LICENSE"
+
     for dest, src in rglob(ns.build, ("*.pyd", "*.dll")):
         if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS:
             continue