Improved flexibility for PyManager package builds (python#239)

Author: zooba

windows-release/merge-and-upload.py

@@ -155,6 +155,7 @@ def purge(url):
     if not UPLOAD_HOST or NO_UPLOAD:
         print("Skipping purge of", url, "because UPLOAD_HOST is missing")
         return
+    print("Purging", url)
     with urlopen(Request(url, method="PURGE", headers={"Fastly-Soft-Purge": 1})) as r:
         r.read()
 
@@ -302,7 +303,15 @@ def install_sortkey(install):
         upload_ssh(MANIFEST_FILE, MANIFEST_PATH)
 
     print("Purging", len(UPLOADS), "uploaded files")
+    parents = set()
     for i, *_ in UPLOADS:
         purge(i["url"])
+        parents.add(i["url"].rpartition("/")[0] + "/")
+    for i in parents:
+        purge(i)
     if INDEX_URL:
         purge(INDEX_URL)
+        purge(INDEX_URL.rpartition("/")[0] + "/")
+    if MANIFEST_URL:
+        purge(MANIFEST_URL)
+        purge(MANIFEST_URL.rpartition("/")[0] + "/")

windows-release/msi-steps.yml

@@ -131,13 +131,15 @@ steps:
       "--cpython-source-dir=$(Build.SourcesDirectory)"
       $(gci -r "$(Build.ArtifactStagingDirectory)\msi\**\python-*.exe")
     workingDirectory: $(Build.BinariesDirectory)
+    condition: and(succeeded(), not(variables['SkipSBOM']))
     displayName: 'Create SBOMs for binaries'
     env:
       PYTHON: $(Build.BinariesDirectory)\win32\python.exe
       PYTHONHOME: $(Build.SourcesDirectory)
 
   - task: CopyFiles@2
     displayName: 'Layout Artifact: sbom'
+    condition: and(succeeded(), not(variables['SkipSBOM']))
     inputs:
       sourceFolder: $(Build.ArtifactStagingDirectory)\msi
       targetFolder: $(Build.ArtifactStagingDirectory)\sbom
@@ -152,6 +154,7 @@ steps:
 
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: sbom'
+    condition: and(succeeded(), not(variables['SkipSBOM']))
     inputs:
       PathtoPublish: '$(Build.ArtifactStagingDirectory)\sbom'
       ArtifactName: sbom

windows-release/stage-layout-embed.yml

@@ -72,6 +72,7 @@ jobs:
       "--cpython-source-dir=$(Build.SourcesDirectory)"
       "$(Build.ArtifactStagingDirectory)\embed\python-$(VersionText)-embed-$(Name).zip"
     workingDirectory: $(Build.BinariesDirectory)
+    condition: and(succeeded(), not(variables['SkipSBOM']))
     displayName: 'Create SBOMs for binaries'
 
   - task: CopyFiles@2

windows-release/stage-layout-pymanager.yml

@@ -215,6 +215,7 @@ jobs:
       "--cpython-source-dir=$(Build.SourcesDirectory)\cpython"
       $(gci "zip\*.zip")
     workingDirectory: $(Build.ArtifactStagingDirectory)
+    condition: and(succeeded(), not(variables['SkipSBOM']))
     displayName: 'Create SBOMs for package'
 
   - publish: '$(Build.ArtifactStagingDirectory)\layout'

windows-release/stage-publish-pymanager.yml

@@ -72,6 +72,18 @@ jobs:
     workingDirectory: $(Pipeline.Workspace)
     displayName: 'Download PuTTY binaries'
 
+  - powershell: |
+      if ($env:FILENAME) {
+        "##vso[task.setvariable variable=_PyManagerIndexFilename]${env:FILENAME}"
+        "Updating index named '${env:FILENAME}'"
+      } else {
+        "##vso[task.setvariable variable=_PyManagerIndexFilename]index-windows.json"
+        "Updating index named 'index-windows.json'"
+      }
+    env:
+      FILENAME: $(PyManagerIndexFilename)
+    displayName: 'Infer index filename'
+
   - powershell: |
       "Uploading following packages:"
       (dir "__install__.*.json").FullName
@@ -82,8 +94,8 @@ jobs:
     displayName: 'Produce uploadable ZIPs'
     env:
       NO_UPLOAD: 1
-      INDEX_URL: '$(PyDotOrgUrlPrefix)python/index-windows.json'
-      INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\index-windows.json'
+      INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)'
+      INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)'
       MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json'
 
   - powershell: |
@@ -95,8 +107,8 @@ jobs:
     condition: and(succeeded(), eq(variables['IsRealSigned'], 'true'))
     displayName: 'Upload ZIPs'
     env:
-      INDEX_URL: '$(PyDotOrgUrlPrefix)python/index-windows.json'
-      INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\index-windows.json'
+      INDEX_URL: '$(PyDotOrgUrlPrefix)python/$(_PyManagerIndexFilename)'
+      INDEX_FILE: '$(Build.ArtifactStagingDirectory)\index\$(_PyManagerIndexFilename)'
       MANIFEST_FILE: '$(Build.ArtifactStagingDirectory)\index\windows.json'
       UPLOAD_URL_PREFIX: $(PyDotOrgUrlPrefix)
       UPLOAD_PATH_PREFIX: $(PyDotOrgUploadPathPrefix)