Made pypi upload simpler

evertlammerts · evertlammerts · commit d5ddd12f1016 · 2025-07-24T11:35:44.000+02:00
diff --git a/.github/workflows/cleanup_pypi.yml b/.github/workflows/cleanup_pypi.yml
@@ -1,16 +1,32 @@
 name: Cleanup PyPI
 on:
   workflow_call:
+    inputs:
+      environment:
+        description: CI environment to run in (test.pypi or production.pypi)
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       dry-run:
         description: List packages that would be deleted but don't delete them
         type: boolean
         default: false
+      environment:
+        description: CI environment to run in
+        type: choice
+        required: true
+        default: test.pypi
+        options:
+          - test.pypi
+          - production.pypi
+
 jobs:
   cleanup_pypi:
     name: Remove Nightlies from PyPI
     runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.environment }}
     env:
       PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}}
       PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}}
@@ -19,6 +35,19 @@ jobs:
         with:
           fetch-depth: 0
 
+      - if: ${{ vars.PYPI_HOST == '' }}
+        run: |
+          echo "Error: PYPI_HOST is not set in CI environment '${{ inputs.environment }}'"
+          exit 1
+      - if: ${{ vars.PYPI_CLEANUP_USERNAME == '' }}
+        run: |
+          echo "Error: PYPI_CLEANUP_USERNAME is not set in CI environment '${{ inputs.environment }}'"
+          exit 1
+      - if: ${{ vars.PYPI_MAX_NIGHTLIES == '' }}
+        run: |
+          echo "Error: PYPI_MAX_NIGHTLIES is not set in CI environment '${{ inputs.environment }}'"
+          exit 1
+
       - name: Install Astral UV
         uses: astral-sh/setup-uv@v6
         with:
@@ -27,8 +56,8 @@ jobs:
       - name: Run Cleanup
         run: |
           set -x
-          uv -v sync --only-group pypi --no-install-project
-          uv -v run --no-sync -s scripts/pypi_cleanup.py ${{ inputs.dry-run && '--dry' || '' }} \
+          uv sync --only-group pypi --no-install-project
+          uv run --no-sync -s scripts/pypi_cleanup.py ${{ inputs.dry-run && '--dry' || '' }} \
             --index-hostname "${{ vars.PYPI_HOST }}" \
             --username "${{ vars.PYPI_CLEANUP_USERNAME }}" \
             --max-nightlies ${{ vars.PYPI_MAX_NIGHTLIES }}
diff --git a/.github/workflows/on_external_dispatch.yml b/.github/workflows/on_external_dispatch.yml
@@ -17,7 +17,7 @@ on:
         required: false
       publish-packages:
         type: boolean
-        description: Publish to S3
+        description: Upload packages to S3
         required: true
         default: false
 
@@ -77,10 +77,12 @@ jobs:
       duckdb-git-ref: ${{ inputs.duckdb-sha }}
       force-version: ${{ inputs.force-version }}
 
-  publish_s3:
-    name: Publish Artifacts to the S3 Staging Bucket
+  upload_s3:
+    name: Upload Artifacts to the S3 Staging Bucket
     runs-on: ubuntu-latest
-    needs: [ externally_triggered_build ]
+    needs: externally_triggered_build
+    outputs:
+      version: ${{ steps.s3_upload.outputs.version }}
     if: ${{ github.repository_owner == 'duckdb' && inputs.publish-packages }}
     steps:
       - name: Fetch artifacts
@@ -97,10 +99,18 @@ jobs:
           aws-access-key-id: ${{ secrets.S3_DUCKDB_STAGING_ID }}
           aws-secret-access-key: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
 
-      - name: Upload artifacts to S3 bucket
-        # semantics: if a version is forced then we upload into a folder by the version name, otherwise we upload
-        #   into a folder that is named <run id>-<run-attempt>. Only the latter will be discovered be
-        #   upload_to_pypi.yml.
+      - name: Upload Artifacts
+        id: s3_upload
         run: |
-          FOLDER="${{ inputs.force-version != '' && inputs.force-version || format('{0}-{1}', github.run_id, github.run_attempt) }}"
-          aws s3 cp artifacts s3://duckdb-staging/${{ github.repository }}/${FOLDER}/ --recursive
+          version=$(basename artifacts/*.tar.gz | sed 's/duckdb-\(.*\).tar.gz/\1/g')
+          aws s3 cp artifacts s3://duckdb-staging/${{ github.repository }}/${version}/ --recursive
+          echo "version=${version}" >> $GITHUB_OUTPUT
+
+  publish_to_pypi:
+    name: Upload Artifacts to PyPI
+    needs: upload_s3
+    if: ${{ force-version == '' }}
+    uses: ./.github/workflows/upload_to_pypi.yml
+    with:
+      version: ${{ needs.upload_s3.outputs.version }}
+      environment: pypi.production
diff --git a/.github/workflows/upload_to_pypi.yml b/.github/workflows/upload_to_pypi.yml
@@ -1,87 +1,47 @@
 name: Upload Artifacts to PyPI
 on:
-  # this workflow runs after the "External Dispatch" workflow is completed
-  workflow_run:
-    workflows: [ External Dispatch ]
-    types: [ completed ]
-    branches:
-      - main
-      - v*.*-*
+  workflow_call:
+    inputs:
+      environment:
+        description: CI environment to run in (test.pypi or production.pypi)
+        type: string
+        required: true
+      version:
+        description: The version to upload (must be present in the S3 staging bucket)
+        type: string
+        required: true
   workflow_dispatch:
     inputs:
       environment:
-        description: Environment to run in ()
+        description: CI environment to run in (test.pypi or production.pypi)
         type: choice
         required: true
         default: test.pypi
         options:
           - test.pypi
           - production.pypi
-      artifact-folder:
-        description: The S3 folder that contains the artifacts (s3://duckdb-staging/duckdb/duckdb-python/<artifact-folder>)
+      version:
+        description: The version to upload (must be present in the S3 staging bucket)
         type: string
         required: true
 
-concurrency: ${{ inputs.artifact-folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }}
+concurrency:
+  group: ${{ inputs.version }}
+  cancel-in-progress: true
 
 jobs:
-  prepare:
-    name: Prepare and guard upload
-    if: ${{ github.repository_owner == 'duckdb' && ( github.event.workflow_run.conclusion == 'success' || github.event_name != 'workflow_run' ) }}
-    runs-on: ubuntu-latest
-    outputs:
-      s3_prefix: ${{ steps.get_s3_prefix.outputs.s3_prefix }}
-    steps:
-      - name: Determine S3 Prefix
-        id: get_s3_prefix
-        run: |
-          artifact_folder="${{ inputs.artifact-folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }}"
-          if [[ -n "${artifact_folder}" ]]; then
-            s3_prefix="${{ github.repository }}/${artifact_folder}"
-            echo "Created S3 prefix: ${s3_prefix}"
-            echo "s3_prefix=${s3_prefix}" >> $GITHUB_OUTPUT
-          else
-            echo "Can't determine S3 prefix for event: ${{ github.event_name }}. Quitting."
-            exit 1
-          fi
-
-      - name: Authenticate With AWS
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-region: 'us-east-2'
-          aws-access-key-id: ${{ secrets.S3_DUCKDB_STAGING_ID }}
-          aws-secret-access-key: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
-
-      - name: Check S3 Prefix
-        shell: bash
-        run: |
-          if [[ $(aws s3api list-objects-v2 \
-            --bucket duckdb-staging \
-            --prefix "${{ steps.get_s3_prefix.outputs.s3_prefix }}/" \
-            --max-items 1 \
-            --query 'Contents[0].Key' \
-            --output text) == "None" ]]; then
-            echo "Prefix does not exist: ${{ steps.get_s3_prefix.outputs.s3_prefix }}"
-            echo "${{ github.event_name == 'workflow_run' && 'Possibly built a stable release?' || 'Unexpected error' }}"
-            exit 1
-          fi
-
   publish-pypi:
     name: Publish Artifacts to PyPI
-    needs: [ prepare ]
     runs-on: ubuntu-latest
     environment:
-      name: ${{ github.event_name == 'workflow_dispatch' && inputs.environment || 'test.pypi' }}
+      name: ${{ inputs.environment }}
     permissions:
       # this is needed for the OIDC flow that is used with trusted publishing on PyPI
       id-token: write
     steps:
-      - name: Fail if PYPI_HOST is not set
-        if: ${{ vars.PYPI_HOST == '' }}
-        shell: bash
+      - if: ${{ vars.PYPI_HOST == '' }}
         run: |
-          env_name="${{ github.event_name == 'workflow_dispatch' && inputs.environment || 'test.pypi' }}"
-          echo "Error: vars.PYPI_HOST is not set in the resolved environment (${env_name})"
+          echo "Error: PYPI_HOST is not set in CI environment '${{ inputs.environment }}'"
           exit 1
 
       - name: Authenticate With AWS
@@ -93,7 +53,7 @@ jobs:
 
       - name: Download Artifacts From S3
         env:
-          S3_URL: 's3://duckdb-staging/${{ needs.prepare.outputs.s3_prefix }}/'
+          S3_URL: 's3://duckdb-staging/${{ github.repository }}/${{ inputs.version }}/'
           AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }}
         run: |
@@ -110,3 +70,5 @@ jobs:
     name: Remove Nightlies from PyPI
     needs: publish-pypi
     uses: ./.github/workflows/cleanup_pypi.yml
+    with:
+      environment: ${{ inputs.environment }}
