bpo-33125: Add support for building and releasing Windows ARM64 packages (GH-16828)

Note that the support is not actually enabled yet, and so we won't be publishing these packages. However, for those who want to build it themselves (even by reusing the Azure Pipelines definition), it's now relatively easy to enable.

16 files changed, 98 insertions, 21 deletions

diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index fe10e45b67..4c2f115cd9 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -145,7 +145,7 @@ jobs:
         buildOpt: '-p x64'
         testRunTitle: '$(Build.SourceBranchName)-win64'
         testRunPlatform: win64
-    maxParallel: 2
+    maxParallel: 4
 
   steps:
   - template: ./windows-steps.yml
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
index 09209fc0c9..73d4f55b86 100644
--- a/.azure-pipelines/pr.yml
+++ b/.azure-pipelines/pr.yml
@@ -145,7 +145,10 @@ jobs:
         buildOpt: '-p x64'
         testRunTitle: '$(System.PullRequest.TargetBranch)-win64'
         testRunPlatform: win64
-    maxParallel: 2
+      winarm64:
+        arch: arm64
+        buildOpt: '-p arm64'
+    maxParallel: 4
 
   steps:
   - template: ./windows-steps.yml
diff --git a/.azure-pipelines/windows-release/build-steps.yml b/.azure-pipelines/windows-release/build-steps.yml
index d4563cd0d7..e2b6683f46 100644
--- a/.azure-pipelines/windows-release/build-steps.yml
+++ b/.azure-pipelines/windows-release/build-steps.yml
@@ -43,7 +43,7 @@ steps:
 
 - powershell: |
     $env:SigningCertificate = $null
-    .\python.bat PC\layout -vv -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default
+    python PC\layout -vv -b "$(Build.BinariesDirectory)\bin" -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default --arch $(Arch)
     makecat "${env:CAT}.cdf"
     del "${env:CAT}.cdf"
     if (-not (Test-Path "${env:CAT}.cat")) {
@@ -52,6 +52,7 @@ steps:
   displayName: 'Generate catalog'
   env:
     CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python
+    PYTHON_HEXVERSION: $(VersionHex)
 
 - task: PublishPipelineArtifact@0
   displayName: 'Publish binaries'
diff --git a/.azure-pipelines/windows-release/layout-command.yml b/.azure-pipelines/windows-release/layout-command.yml
index 2dcd6ed26c..406ccd859f 100644
--- a/.azure-pipelines/windows-release/layout-command.yml
+++ b/.azure-pipelines/windows-release/layout-command.yml
@@ -1,12 +1,20 @@
 steps:
+- task: DownloadPipelineArtifact@1
+  displayName: 'Download artifact: bin_$(HostArch)'
+  condition: and(succeeded(), variables['HostArch'])
+  inputs:
+    artifactName: bin_$(HostArch)
+    targetPath: $(Build.BinariesDirectory)\bin_$(HostArch)
+
 - powershell: >
     Write-Host (
     '##vso[task.setvariable variable=LayoutCmd]&
-    "{0}\bin\python.exe"
+    "$(Python)"
     "{1}\PC\layout"
     -vv
     --source "{1}"
     --build "{0}\bin"
+    --arch "$(Name)"
     --temp "{0}\layout-temp"
     --include-cat "{0}\bin\python.cat"
     --doc-build "{0}\doc"'
diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml
index f7bff162f8..a460eb1bac 100644
--- a/.azure-pipelines/windows-release/msi-steps.yml
+++ b/.azure-pipelines/windows-release/msi-steps.yml
@@ -54,6 +54,7 @@ steps:
   - powershell: |
       copy $(Build.BinariesDirectory)\amd64\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force
     displayName: 'Copy signed files into sources'
+    condition: and(succeeded(), variables['SigningCertificate'])
 
   - script: |
       call Tools\msi\get_externals.bat
diff --git a/.azure-pipelines/windows-release/stage-build.yml b/.azure-pipelines/windows-release/stage-build.yml
index c98576ef97..60d72b282d 100644
--- a/.azure-pipelines/windows-release/stage-build.yml
+++ b/.azure-pipelines/windows-release/stage-build.yml
@@ -16,14 +16,16 @@ jobs:
     env:
       BUILDDIR: $(Build.BinariesDirectory)\Doc
 
-  #- powershell: iwr "https://www.python.org/ftp/python/3.7.3/python373.chm" -OutFile "$(Build.BinariesDirectory)\python390a0.chm"
-  #  displayName: 'Cheat at building CHM docs'
-
   - script: Doc\make.bat htmlhelp
     displayName: 'Build CHM docs'
     env:
       BUILDDIR: $(Build.BinariesDirectory)\Doc
 
+  #- powershell: |
+  #    mkdir -Force "$(Build.BinariesDirectory)\Doc\htmlhelp"
+  #    iwr "https://www.python.org/ftp/python/3.8.0/python380.chm" -OutFile "$(Build.BinariesDirectory)\Doc\htmlhelp\python390a0.chm"
+  #  displayName: 'Cheat at building CHM docs'
+
   - task: CopyFiles@2
     displayName: 'Assemble artifact: Doc'
     inputs:
@@ -65,6 +67,16 @@ jobs:
         Arch: amd64
         Platform: x64
         Configuration: Debug
+      arm64:
+        Name: arm64
+        Arch: arm64
+        Platform: ARM64
+        Configuration: Release
+      arm64_d:
+        Name: arm64_d
+        Arch: arm64
+        Platform: ARM64
+        Configuration: Debug
 
   steps:
     - template: ./build-steps.yml
diff --git a/.azure-pipelines/windows-release/stage-layout-embed.yml b/.azure-pipelines/windows-release/stage-layout-embed.yml
index 09857ff676..3306e1cbc4 100644
--- a/.azure-pipelines/windows-release/stage-layout-embed.yml
+++ b/.azure-pipelines/windows-release/stage-layout-embed.yml
@@ -19,6 +19,11 @@ jobs:
         Name: amd64
         Python: $(Build.BinariesDirectory)\bin\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
+      arm64:
+        Name: arm64
+        HostArch: amd64
+        Python: $(Build.BinariesDirectory)\bin_amd64\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
 
   steps:
   - template: ./checkout.yml
diff --git a/.azure-pipelines/windows-release/stage-layout-full.yml b/.azure-pipelines/windows-release/stage-layout-full.yml
index 12c3472390..78bc1b3975 100644
--- a/.azure-pipelines/windows-release/stage-layout-full.yml
+++ b/.azure-pipelines/windows-release/stage-layout-full.yml
@@ -13,11 +13,18 @@ jobs:
     matrix:
       win32:
         Name: win32
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
+        TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8
       amd64:
         Name: amd64
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+        TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      arm64:
+        Name: arm64
+        HostArch: amd64
+        Python: $(Build.BinariesDirectory)\bin_amd64\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
 
   steps:
@@ -43,13 +50,15 @@ jobs:
 
   - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact: tcltk_lib_$(Name)'
+    condition: and(succeeded(), variables['TclLibrary'])
     inputs:
       artifactName: tcltk_lib_$(Name)
       targetPath: $(Build.BinariesDirectory)\tcltk_lib
 
   - powershell: |
-      copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force
+      copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force
     displayName: 'Copy signed files into sources'
+    condition: and(succeeded(), variables['SigningCertificate'])
 
   - template: ./layout-command.yml
 
@@ -57,7 +66,7 @@ jobs:
       $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default
     displayName: 'Generate full layout'
     env:
-      TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      TCL_LIBRARY: $(TclLibrary)
 
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Artifact: layout_full_$(Name)'
diff --git a/.azure-pipelines/windows-release/stage-layout-msix.yml b/.azure-pipelines/windows-release/stage-layout-msix.yml
index ba86392f3e..60a5c9ea54 100644
--- a/.azure-pipelines/windows-release/stage-layout-msix.yml
+++ b/.azure-pipelines/windows-release/stage-layout-msix.yml
@@ -12,11 +12,18 @@ jobs:
     matrix:
       #win32:
       #  Name: win32
-      #  Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+      #  Python: $(Build.BinariesDirectory)\bin\python.exe
       #  PYTHONHOME: $(Build.SourcesDirectory)
+      #  TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8
       amd64:
         Name: amd64
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+        TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      arm64:
+        Name: arm64
+        HostArch: amd64
+        Python: $(Build.BinariesDirectory)\bin_amd64\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
 
   steps:
@@ -36,13 +43,15 @@ jobs:
 
   - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact: tcltk_lib_$(Name)'
+    condition: and(succeeded(), variables['TclLibrary'])
     inputs:
       artifactName: tcltk_lib_$(Name)
       targetPath: $(Build.BinariesDirectory)\tcltk_lib
 
   - powershell: |
-      copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force
+      copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force
     displayName: 'Copy signed files into sources'
+    condition: and(succeeded(), variables['SigningCertificate'])
 
   - template: ./layout-command.yml
 
@@ -51,7 +60,7 @@ jobs:
       $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile
     displayName: 'Generate store APPX layout'
     env:
-      TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      TCL_LIBRARY: $(TclLibrary)
 
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Artifact: layout_appxstore_$(Name)'
@@ -79,7 +88,7 @@ jobs:
       $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests
     displayName: 'Generate sideloading APPX layout'
     env:
-      TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib\tcl8
+      TCL_LIBRARY: $(TclLibrary)
 
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Artifact: layout_appx_$(Name)'
diff --git a/.azure-pipelines/windows-release/stage-layout-nuget.yml b/.azure-pipelines/windows-release/stage-layout-nuget.yml
index 7954c4547f..7e20f89530 100644
--- a/.azure-pipelines/windows-release/stage-layout-nuget.yml
+++ b/.azure-pipelines/windows-release/stage-layout-nuget.yml
@@ -13,11 +13,16 @@ jobs:
     matrix:
       win32:
         Name: win32
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
       amd64:
         Name: amd64
-        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        Python: $(Build.BinariesDirectory)\bin\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+      arm64:
+        Name: arm64
+        HostArch: amd64
+        Python: $(Build.BinariesDirectory)\bin_amd64\python.exe
         PYTHONHOME: $(Build.SourcesDirectory)
 
   steps:
@@ -32,14 +37,13 @@ jobs:
   - powershell: |
       copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force
     displayName: 'Copy signed files into sources'
+    condition: and(succeeded(), variables['SigningCertificate'])
 
   - template: ./layout-command.yml
 
   - powershell: |
       $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\nuget" --preset-nuget
     displayName: 'Generate nuget layout'
-    env:
-      TCL_LIBRARY: $(Build.BinariesDirectory)\bin_$(Name)\tcl\tcl8
 
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Artifact: layout_nuget_$(Name)'
diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml
index eebc63fb88..f17ba9628e 100644
--- a/.azure-pipelines/windows-release/stage-pack-msix.yml
+++ b/.azure-pipelines/windows-release/stage-pack-msix.yml
@@ -20,6 +20,16 @@ jobs:
         Artifact: appxstore
         Suffix: -store
         Upload: true
+      arm64:
+        Name: arm64
+        Artifact: appx
+        Suffix:
+        ShouldSign: true
+      arm64_store:
+        Name: arm64
+        Artifact: appxstore
+        Suffix: -store
+        Upload: true
 
   steps:
   - template: ./checkout.yml
diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml
index f59bbe9b39..34619fc5fd 100644
--- a/.azure-pipelines/windows-release/stage-pack-nuget.yml
+++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml
@@ -15,6 +15,8 @@ jobs:
         Name: amd64
       win32:
         Name: win32
+      arm64:
+        Name: arm64
 
   steps:
   - checkout: none
diff --git a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml
index 570cdb3ec5..5aba048cd7 100644
--- a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml
+++ b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml
@@ -31,6 +31,11 @@ jobs:
       buildVersionToDownload: specific
       buildId: $(BuildToPublish)
 
+  - powershell: gci pythonarm*.nupkg | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del
+    displayName: 'Prevent publishing ARM/ARM64 packages'
+    workingDirectory: '$(Build.BinariesDirectory)\nuget'
+    condition: and(succeeded(), not(variables['PublishArmPackages']))
+
   - task: NuGetCommand@2
     displayName: Push packages
     condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate']))
diff --git a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml
index 2dd354a8c2..6c61e9ac3b 100644
--- a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml
+++ b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml
@@ -39,6 +39,10 @@ jobs:
       artifactName: embed
       downloadPath: $(Build.BinariesDirectory)
 
+  - powershell: gci *embed-arm*.zip | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del
+    displayName: 'Prevent publishing ARM/ARM64 packages'
+    workingDirectory: '$(Build.BinariesDirectory)\embed'
+    condition: and(succeeded(), not(variables['PublishArmPackages']))
 
   - task: DownloadPipelineArtifact@1
     displayName: 'Download artifact from $(BuildToPublish): Doc'
diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml
index 2307c6c9c8..a0adc05812 100644
--- a/.azure-pipelines/windows-release/stage-sign.yml
+++ b/.azure-pipelines/windows-release/stage-sign.yml
@@ -19,6 +19,8 @@ jobs:
         Name: win32
       amd64:
         Name: amd64
+      arm64:
+        Name: arm64
 
   steps:
   - template: ./checkout.yml
diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml
index 794a23a5d7..f502c40637 100644
--- a/.azure-pipelines/windows-steps.yml
+++ b/.azure-pipelines/windows-steps.yml
@@ -19,9 +19,11 @@ steps:
 
 - script: python.bat -m test.pythoninfo
   displayName: 'Display build info'
+  condition: and(succeeded(), variables['testRunPlatform'])
 
 - script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir="$(Build.BinariesDirectory)\test"
   displayName: 'Tests'
+  condition: and(succeeded(), variables['testRunPlatform'])
   env:
     PREFIX: $(Py_OutDir)\$(arch)
 
@@ -32,4 +34,4 @@ steps:
     mergeTestResults: true
     testRunTitle: $(testRunTitle)
     platform: $(testRunPlatform)
-  condition: succeededOrFailed()
+  condition: and(succeededOrFailed(), variables['testRunPlatform'])