diff options
author | Steve Dower <steve.dower@python.org> | 2019-06-17 22:27:36 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-06-17 22:27:36 (GMT) |
commit | fe2ad927f457043cb9f3a1148ed737af05a2b1c4 (patch) | |
tree | 515340185113075f6c560c43e4b088cc6a069b64 | |
parent | 343ed0ffe0d5ddd4f17c31e14a656a04ac7dfc19 (diff) | |
download | cpython-fe2ad927f457043cb9f3a1148ed737af05a2b1c4.zip cpython-fe2ad927f457043cb9f3a1148ed737af05a2b1c4.tar.gz cpython-fe2ad927f457043cb9f3a1148ed737af05a2b1c4.tar.bz2 |
Improve release build performance using new artifacts tasks (GH-14175)
12 files changed, 97 insertions, 126 deletions
diff --git a/.azure-pipelines/windows-release/build-steps.yml b/.azure-pipelines/windows-release/build-steps.yml index 508d73b..d4563cd 100644 --- a/.azure-pipelines/windows-release/build-steps.yml +++ b/.azure-pipelines/windows-release/build-steps.yml @@ -53,19 +53,19 @@ steps: env: CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python -- task: PublishBuildArtifacts@1 +- task: PublishPipelineArtifact@0 displayName: 'Publish binaries' condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))) inputs: - PathtoPublish: '$(Build.BinariesDirectory)\bin\$(Arch)' - ArtifactName: bin_$(Name) + targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' + artifactName: bin_$(Name) -- task: PublishBuildArtifacts@1 +- task: PublishPipelineArtifact@0 displayName: 'Publish binaries for signing' condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])) inputs: - PathtoPublish: '$(Build.BinariesDirectory)\bin\$(Arch)' - ArtifactName: unsigned_bin_$(Name) + targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' + artifactName: unsigned_bin_$(Name) - task: CopyFiles@2 displayName: 'Layout Artifact: symbols' diff --git a/.azure-pipelines/windows-release/layout-command.yml b/.azure-pipelines/windows-release/layout-command.yml index 3ec9b69..2dcd6ed 100644 --- a/.azure-pipelines/windows-release/layout-command.yml +++ b/.azure-pipelines/windows-release/layout-command.yml @@ -2,19 +2,14 @@ steps: - powershell: > Write-Host ( '##vso[task.setvariable variable=LayoutCmd]& - "{0}" + "{0}\bin\python.exe" "{1}\PC\layout" -vv --source "{1}" - --build "{2}" - --temp "{3}" - --include-cat "{2}\python.cat" - --doc-build "{4}"' - -f ( - "$(PYTHON)", - "$(Build.SourcesDirectory)", - (Split-Path -Parent "$(PYTHON)"), - "$(Build.BinariesDirectory)\layout-temp", - "$(Build.BinariesDirectory)\doc" - )) + --build "{0}\bin" + --temp "{0}\layout-temp" + --include-cat "{0}\bin\python.cat" + --doc-build "{0}\doc"' + -f ("$(Build.BinariesDirectory)", "$(Build.SourcesDirectory)") + ) displayName: 'Set LayoutCmd' diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml index 1534082..c55fa53 100644 --- a/.azure-pipelines/windows-release/msi-steps.yml +++ b/.azure-pipelines/windows-release/msi-steps.yml @@ -1,11 +1,11 @@ steps: - template: ./checkout.yml - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: doc' inputs: artifactName: doc - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\doc - task: CopyFiles@2 displayName: 'Merge documentation files' @@ -15,63 +15,41 @@ steps: contents: | htmlhelp\*.chm - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: bin_win32' inputs: artifactName: bin_win32 - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\win32 - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: bin_win32_d' inputs: artifactName: bin_win32_d - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\win32 - - task: CopyFiles@2 - displayName: 'Merge win32 debug files' - inputs: - sourceFolder: $(Build.BinariesDirectory)\bin_win32_d - targetFolder: $(Build.BinariesDirectory)\bin_win32 - contents: | - **\*_d.* - - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: bin_amd64' inputs: artifactName: bin_amd64 - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\amd64 - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: bin_amd64_d' inputs: artifactName: bin_amd64_d - downloadPath: $(Build.BinariesDirectory) - - - task: CopyFiles@2 - displayName: 'Merge amd64 debug files' - inputs: - sourceFolder: $(Build.BinariesDirectory)\bin_amd64_d - targetFolder: $(Build.BinariesDirectory)\bin_amd64 - contents: | - **\*_d.* + targetPath: $(Build.BinariesDirectory)\amd64 - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: tcltk_lib_win32' inputs: artifactName: tcltk_lib_win32 - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\tcltk_lib_win32 - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: tcltk_lib_amd64' inputs: artifactName: tcltk_lib_amd64 - downloadPath: $(Build.BinariesDirectory) - - - script: | - ren bin_win32 win32 - ren bin_amd64 amd64 - displayName: 'Correct artifact directory names' - workingDirectory: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\tcltk_lib_amd64 - script: | call Tools\msi\get_externals.bat @@ -139,8 +117,8 @@ steps: *.cab *.exe - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@0 displayName: 'Publish MSI' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\msi' - ArtifactName: msi + targetPath: '$(Build.ArtifactStagingDirectory)\msi' + artifactName: msi diff --git a/.azure-pipelines/windows-release/stage-build.yml b/.azure-pipelines/windows-release/stage-build.yml index a5093a0..ce7b381 100644 --- a/.azure-pipelines/windows-release/stage-build.yml +++ b/.azure-pipelines/windows-release/stage-build.yml @@ -33,11 +33,11 @@ jobs: html\**\* htmlhelp\*.chm - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@0 displayName: 'Publish artifact: doc' inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory)\Doc - ArtifactName: doc + targetPath: $(Build.ArtifactStagingDirectory)\Doc + artifactName: doc - job: Build_Python displayName: Python build @@ -147,14 +147,14 @@ jobs: platform: x64 msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_amd64" - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@0 displayName: 'Publish artifact: tcltk_lib_win32' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\tcl_win32' - ArtifactName: tcltk_lib_win32 + targetPath: '$(Build.ArtifactStagingDirectory)\tcl_win32' + artifactName: tcltk_lib_win32 - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@0 displayName: 'Publish artifact: tcltk_lib_amd64' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\tcl_amd64' - ArtifactName: tcltk_lib_amd64 + targetPath: '$(Build.ArtifactStagingDirectory)\tcl_amd64' + artifactName: tcltk_lib_amd64 diff --git a/.azure-pipelines/windows-release/stage-layout-embed.yml b/.azure-pipelines/windows-release/stage-layout-embed.yml index e2689db..09857ff 100644 --- a/.azure-pipelines/windows-release/stage-layout-embed.yml +++ b/.azure-pipelines/windows-release/stage-layout-embed.yml @@ -13,21 +13,21 @@ jobs: matrix: win32: Name: win32 - Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + Python: $(Build.BinariesDirectory)\bin\python.exe PYTHONHOME: $(Build.SourcesDirectory) amd64: Name: amd64 - Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe + Python: $(Build.BinariesDirectory)\bin\python.exe PYTHONHOME: $(Build.SourcesDirectory) steps: - template: ./checkout.yml - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: bin_$(Name)' inputs: artifactName: bin_$(Name) - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\bin - template: ./layout-command.yml @@ -43,11 +43,11 @@ jobs: --preset-embed displayName: 'Generate embeddable layout' - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: layout_embed_$(Name)' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\layout' - ArtifactName: layout_embed_$(Name) + targetPath: '$(Build.ArtifactStagingDirectory)\layout' + artifactName: layout_embed_$(Name) - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: embed' diff --git a/.azure-pipelines/windows-release/stage-layout-full.yml b/.azure-pipelines/windows-release/stage-layout-full.yml index 3593cf0..8b412df 100644 --- a/.azure-pipelines/windows-release/stage-layout-full.yml +++ b/.azure-pipelines/windows-release/stage-layout-full.yml @@ -23,29 +23,29 @@ jobs: steps: - template: ./checkout.yml - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: bin_$(Name)' inputs: artifactName: bin_$(Name) - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\bin - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: bin_$(Name)_d' inputs: artifactName: bin_$(Name)_d - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\bin - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: doc' inputs: artifactName: doc - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\doc - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: tcltk_lib_$(Name)' inputs: artifactName: tcltk_lib_$(Name) - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\tcltk_lib - template: ./layout-command.yml @@ -53,10 +53,10 @@ jobs: $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default displayName: 'Generate full layout' env: - TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8 + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: layout_full_$(Name)' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\layout' - ArtifactName: layout_full_$(Name) + targetPath: '$(Build.ArtifactStagingDirectory)\layout' + artifactName: layout_full_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-msix.yml b/.azure-pipelines/windows-release/stage-layout-msix.yml index 1a1e0a2f..7d66e8f 100644 --- a/.azure-pipelines/windows-release/stage-layout-msix.yml +++ b/.azure-pipelines/windows-release/stage-layout-msix.yml @@ -22,23 +22,23 @@ jobs: steps: - template: ./checkout.yml - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: bin_$(Name)' inputs: artifactName: bin_$(Name) - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\bin - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: bin_$(Name)_d' inputs: artifactName: bin_$(Name)_d - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\bin - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: tcltk_lib_$(Name)' inputs: artifactName: tcltk_lib_$(Name) - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\tcltk_lib - template: ./layout-command.yml @@ -47,20 +47,20 @@ jobs: $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile displayName: 'Generate store APPX layout' env: - TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8 + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: layout_appxstore_$(Name)' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\appx-store' - ArtifactName: layout_appxstore_$(Name) + targetPath: '$(Build.ArtifactStagingDirectory)\appx-store' + artifactName: layout_appxstore_$(Name) - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: cert' condition: and(succeeded(), variables['SigningCertificate']) inputs: artifactName: cert - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\cert - powershell: | $info = (gc "$(Build.BinariesDirectory)\cert\certinfo.json" | ConvertFrom-JSON) @@ -75,12 +75,10 @@ jobs: $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests displayName: 'Generate sideloading APPX layout' env: - TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8 - APPX_DATA_PUBLISHER: $(APPX_DATA_PUBLISHER) - APPX_DATA_SHA256: $(APPX_DATA_SHA256) + TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8 - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: layout_appx_$(Name)' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\appx' - ArtifactName: layout_appx_$(Name) + targetPath: '$(Build.ArtifactStagingDirectory)\appx' + artifactName: layout_appx_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-nuget.yml b/.azure-pipelines/windows-release/stage-layout-nuget.yml index ca4213d..0151297 100644 --- a/.azure-pipelines/windows-release/stage-layout-nuget.yml +++ b/.azure-pipelines/windows-release/stage-layout-nuget.yml @@ -23,11 +23,11 @@ jobs: steps: - template: ./checkout.yml - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: bin_$(Name)' inputs: artifactName: bin_$(Name) - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\bin - template: ./layout-command.yml @@ -37,8 +37,8 @@ jobs: env: TCL_LIBRARY: $(Build.BinariesDirectory)\bin_$(Name)\tcl\tcl8 - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: layout_nuget_$(Name)' inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\nuget' - ArtifactName: layout_nuget_$(Name) + targetPath: '$(Build.ArtifactStagingDirectory)\nuget' + artifactName: layout_nuget_$(Name) diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml index 6f1846e..eebc63f 100644 --- a/.azure-pipelines/windows-release/stage-pack-msix.yml +++ b/.azure-pipelines/windows-release/stage-pack-msix.yml @@ -24,11 +24,11 @@ jobs: steps: - template: ./checkout.yml - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: layout_$(Artifact)_$(Name)' inputs: artifactName: layout_$(Artifact)_$(Name) - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\layout - task: DownloadBuildArtifacts@0 displayName: 'Download artifact: symbols' @@ -46,7 +46,7 @@ jobs: displayName: 'Extract version numbers' - powershell: | - ./Tools/msi/make_appx.ps1 -layout "$(Build.BinariesDirectory)\layout_$(Artifact)_$(Name)" -msix "$(Build.ArtifactStagingDirectory)\msix\$(Filename).msix" + ./Tools/msi/make_appx.ps1 -layout "$(Build.BinariesDirectory)\layout" -msix "$(Build.ArtifactStagingDirectory)\msix\$(Filename).msix" displayName: 'Build msix' - powershell: | diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml index 5aa394f..f59bbe9 100644 --- a/.azure-pipelines/windows-release/stage-pack-nuget.yml +++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml @@ -19,11 +19,11 @@ jobs: steps: - checkout: none - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: layout_nuget_$(Name)' inputs: artifactName: layout_nuget_$(Name) - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\layout - task: NugetToolInstaller@0 displayName: 'Install Nuget' @@ -31,7 +31,7 @@ jobs: versionSpec: '>=5.0' - powershell: | - nuget pack "$(Build.BinariesDirectory)\layout_nuget_$(Name)\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive + nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive displayName: 'Create nuget package' - task: PublishBuildArtifacts@1 diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml index 3d6ca94..d6984a0 100644 --- a/.azure-pipelines/windows-release/stage-sign.yml +++ b/.azure-pipelines/windows-release/stage-sign.yml @@ -24,17 +24,17 @@ jobs: Write-Host "##vso[build.addbuildtag]signed" displayName: 'Add build tags' - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: unsigned_bin_$(Name)' inputs: artifactName: unsigned_bin_$(Name) - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\bin - powershell: | $files = (gi *.exe, *.dll, *.pyd, *.cat -Exclude vcruntime*, libffi*, libcrypto*, libssl*) signtool sign /a /n "$(SigningCertificate)" /fd sha256 /d "$(SigningDescription)" $files displayName: 'Sign binaries' - workingDirectory: $(Build.BinariesDirectory)\unsigned_bin_$(Name) + workingDirectory: $(Build.BinariesDirectory)\bin - powershell: | $files = (gi *.exe, *.dll, *.pyd, *.cat -Exclude vcruntime*, libffi*, libcrypto*, libssl*) @@ -51,14 +51,14 @@ jobs: Write-Host "##vso[task.logissue type=error]Failed to timestamp files" } displayName: 'Timestamp binaries' - workingDirectory: $(Build.BinariesDirectory)\unsigned_bin_$(Name) + workingDirectory: $(Build.BinariesDirectory)\bin continueOnError: true - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@0 displayName: 'Publish artifact: bin_$(Name)' inputs: - PathtoPublish: '$(Build.BinariesDirectory)\unsigned_bin_$(Name)' - ArtifactName: bin_$(Name) + targetPath: '$(Build.BinariesDirectory)\bin' + artifactName: bin_$(Name) - job: Dump_CertInfo @@ -91,11 +91,11 @@ jobs: $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" displayName: "Extract certificate info" - - task: PublishBuildArtifacts@1 + - task: PublishPipelineArtifact@0 displayName: 'Publish artifact: cert' inputs: - PathtoPublish: '$(Build.BinariesDirectory)\cert' - ArtifactName: cert + targetPath: '$(Build.BinariesDirectory)\cert' + artifactName: cert - job: Mark_Unsigned diff --git a/.azure-pipelines/windows-release/stage-test-msi.yml b/.azure-pipelines/windows-release/stage-test-msi.yml index 1003929..27b0c96 100644 --- a/.azure-pipelines/windows-release/stage-test-msi.yml +++ b/.azure-pipelines/windows-release/stage-test-msi.yml @@ -30,11 +30,11 @@ jobs: steps: - checkout: none - - task: DownloadBuildArtifacts@0 + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: msi' inputs: artifactName: msi - downloadPath: $(Build.BinariesDirectory) + targetPath: $(Build.BinariesDirectory)\msi - powershell: | $p = (gci -r *.exe | ?{ $_.Name -match '$(ExeMatch)' } | select -First 1) |