Ensure unicity of S3 path

Author: evertlammerts

.github/workflows/cleanup_pypi.yml

@@ -60,4 +60,15 @@ jobs:
           uv run --no-sync -s scripts/pypi_cleanup.py ${{ inputs.dry-run && '--dry' || '' }} \
             --index-hostname "${{ vars.PYPI_HOST }}" \
             --username "${{ vars.PYPI_CLEANUP_USERNAME }}" \
-            --max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }}
+            --max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }} > cleanup_output 2>&1
+
+      - name: PyPI Cleanup Summary
+        run : |
+          echo "## PyPI Cleanup Summary" >> $GITHUB_STEP_SUMMARY
+          echo "* Dry run: ${{ inputs.dry-run }}" >> $GITHUB_STEP_SUMMARY
+          echo "* PyPI Host: ${{ vars.PYPI_HOST }}" >> $GITHUB_STEP_SUMMARY
+          echo "* CI Environment: " >> $GITHUB_STEP_SUMMARY
+          echo "* Output:" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          cat cleanup_output >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY

.github/workflows/on_external_dispatch.yml

@@ -80,7 +80,7 @@ jobs:
   upload_s3:
     name: Upload Artifacts to the S3 Staging Bucket
     runs-on: ubuntu-latest
-    needs: externally_triggered_build
+    needs: [commit_submodule, externally_triggered_build]
     outputs:
       version: ${{ steps.s3_upload.outputs.version }}
     if: ${{ github.repository_owner == 'duckdb' && inputs.publish-packages }}
@@ -102,15 +102,24 @@ jobs:
       - name: Upload Artifacts
         id: s3_upload
         run: |
-          version=$(basename artifacts/*.tar.gz | sed 's/duckdb-\(.*\).tar.gz/\1/g')
-          aws s3 cp artifacts s3://duckdb-staging/${{ github.repository }}/${version}/ --recursive
+          sha=${{ needs.commit_submodule.outputs.sha-after-commit }}
+          aws s3 cp artifacts s3://duckdb-staging/${{ github.repository }}/${sha:0:10}/ --recursive
           echo "version=${version}" >> $GITHUB_OUTPUT
 
+      - name: S3 Upload Summary
+        run : |
+          sha=${{ needs.commit_submodule.outputs.sha-after-commit }}
+          version=$(basename artifacts/*.tar.gz | sed 's/duckdb-\(.*\).tar.gz/\1/g')
+          echo "## S3 Upload Summary" >> $GITHUB_STEP_SUMMARY
+          echo "* Version: ${version}" >> $GITHUB_STEP_SUMMARY
+          echo "* SHA: ${sha:0:10}" >> $GITHUB_STEP_SUMMARY
+          echo "* S3 URL: s3://duckdb-staging/${{ github.repository }}/${sha:0:10}/" >> $GITHUB_STEP_SUMMARY
+
   publish_to_pypi:
     name: Upload Artifacts to PyPI
-    needs: upload_s3
+    needs: [ commit_submodule, upload_s3 ]
     if: ${{ force-version == '' }}
     uses: ./.github/workflows/upload_to_pypi.yml
     with:
-      version: ${{ needs.upload_s3.outputs.version }}
-      environment: pypi.production
+      sha: ${{ needs.commit_submodule.outputs.sha-after-commit }}
+      environment: pypi.production

.github/workflows/upload_to_pypi.yml

@@ -6,8 +6,8 @@ on:
         description: CI environment to run in (test.pypi or production.pypi)
         type: string
         required: true
-      version:
-        description: The version to upload (must be present in the S3 staging bucket)
+      sha:
+        description: The SHA of the commit that the packages were built from
         type: string
         required: true
   workflow_dispatch:
@@ -20,13 +20,13 @@ on:
         options:
           - test.pypi
           - production.pypi
-      version:
-        description: The version to upload (must be present in the S3 staging bucket)
+      sha:
+        description: The SHA of the commit that the packages were built from
         type: string
         required: true
 
 concurrency:
-  group: ${{ inputs.version }}
+  group: ${{ inputs.sha }}
   cancel-in-progress: true
 
 jobs:
@@ -53,7 +53,7 @@ jobs:
 
       - name: Download Artifacts From S3
         env:
-          S3_URL: 's3://duckdb-staging/${{ github.repository }}/${{ inputs.version }}/'
+          S3_URL: 's3://duckdb-staging/${{ github.repository }}/${{ inputs.sha }}/'
           AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
         run: |
@@ -66,6 +66,14 @@ jobs:
           repository-url: 'https://${{ vars.PYPI_HOST }}/legacy/'
           packages-dir: packages
 
+      - name: PyPI Upload Summary
+        run : |
+          version=$(basename packages/*.tar.gz | sed 's/duckdb-\(.*\).tar.gz/\1/g')
+          echo "## PyPI Upload Summary" >> $GITHUB_STEP_SUMMARY
+          echo "* Version: ${version}" >> $GITHUB_STEP_SUMMARY
+          echo "* PyPI Host: ${{ vars.PYPI_HOST }}" >> $GITHUB_STEP_SUMMARY
+          echo "* CI Environment: ${{ inputs.environment }}" >> $GITHUB_STEP_SUMMARY
+
   cleanup_nightlies:
     name: Remove Nightlies from PyPI
     needs: publish-pypi

scripts/pypi_cleanup.py

@@ -185,8 +185,8 @@ def run(self):
                     sorted_versions = sorted(versions, key=lambda x: int(x.split('dev')[-1]))
                     pkg_vers.extend(sorted_versions[:-self.max_dev_releases])
 
+            print("Following pkg_vers can be deleted: ", pkg_vers)
             if not self.do_it:
-                print("Following pkg_vers can be deleted: ", pkg_vers)
                 return
 
             if not pkg_vers:
@@ -231,9 +231,9 @@ def run(self):
 
             two_factor = False
             with s.post(
-                f"{self.url}/account/login/",
-                data={"csrf_token": csrf, "username": self.username, "password": self.password},
-                headers={"referer": f"{self.url}/account/login/"},
+                    f"{self.url}/account/login/",
+                    data={"csrf_token": csrf, "username": self.username, "password": self.password},
+                    headers={"referer": f"{self.url}/account/login/"},
             ) as r:
                 r.raise_for_status()
                 if r.url == f"{self.url}/account/login/":
@@ -255,9 +255,9 @@ def run(self):
                 for i in range(3):
                     auth_code = pyotp.TOTP(self.otp).now()
                     with s.post(
-                        two_factor_url,
-                        data={"csrf_token": csrf, "method": "totp", "totp_value": auth_code},
-                        headers={"referer": two_factor_url},
+                            two_factor_url,
+                            data={"csrf_token": csrf, "method": "totp", "totp_value": auth_code},
+                            headers={"referer": two_factor_url},
                     ) as r:
                         r.raise_for_status()
                         if r.url == two_factor_url:
@@ -286,12 +286,12 @@ def run(self):
                         referer = r.url
 
                     with s.post(
-                        form_url,
-                        data={
-                            "csrf_token": csrf,
-                            "confirm_delete_version": pkg_ver,
-                        },
-                        headers={"referer": referer},
+                            form_url,
+                            data={
+                                "csrf_token": csrf,
+                                "confirm_delete_version": pkg_ver,
+                            },
+                            headers={"referer": referer},
                     ) as r:
                         r.raise_for_status()