Release #33
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Release is called by duckdb's InvokeCI -> NotifyExternalRepositories job | |
| name: Release | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| duckdb-sha: | |
| type: string | |
| description: The DuckDB submodule commit to build against | |
| required: true | |
| stable-version: | |
| type: string | |
| description: Release a stable version (vX.Y.Z-((rc|post)N)) | |
| required: false | |
| pypi-index: | |
| type: choice | |
| description: Which PyPI to use | |
| required: true | |
| options: | |
| - test | |
| - prod | |
| store-s3: | |
| type: boolean | |
| description: Also store test packages in S3 | |
| default: false | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| build_sdist: | |
| name: Build an sdist and determine versions | |
| uses: ./.github/workflows/packaging_sdist.yml | |
| with: | |
| testsuite: all | |
| git-ref: ${{ github.ref }} | |
| duckdb-git-ref: ${{ inputs.duckdb-sha }} | |
| set-version: ${{ inputs.stable-version }} | |
| check_pypi: | |
| name: Check whether this version was already released | |
| needs: build_sdist | |
| outputs: | |
| can_upload: ${{ steps.index_check.outputs.can_upload }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: index_check | |
| run: | | |
| # Check PyPI whether the release we're building is already present | |
| pypi_hostname="${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org" | |
| pkg_version=${{ needs.build_sdist.outputs.package-version }} | |
| http_status=$( curl -s -o /dev/null -w "%{http_code}" https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json ) | |
| if [ "$http_status" == "200" ]; then | |
| echo "::warning::DuckDB ${pkg_version} is already present on ${pypi_hostname}, will not release" | |
| echo "can_upload=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "can_upload=false" >> $GITHUB_OUTPUT | |
| fi | |
| build_wheels: | |
| name: Build and test releases | |
| needs: check_pypi | |
| if: ${{ needs.check_pypi.outputs.can_upload }} | |
| uses: ./.github/workflows/packaging_wheels.yml | |
| with: | |
| minimal: false | |
| testsuite: all | |
| git-ref: ${{ github.ref }} | |
| duckdb-git-ref: ${{ inputs.duckdb-sha }} | |
| set-version: ${{ inputs.stable-version }} | |
| upload_s3: | |
| name: Upload Artifacts to S3 | |
| runs-on: ubuntu-latest | |
| needs: [build_sdist, build_wheels] | |
| if: ${{ github.repository_owner == 'duckdb' && ( inputs.pypi-index == 'prod' || inputs.store-s3 ) }} | |
| outputs: | |
| s3-upload-url: ${{ steps.input.outputs.s3_upload_url }} | |
| steps: | |
| - name: Fetch artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '{sdist,wheel}*' | |
| path: artifacts/ | |
| merge-multiple: true | |
| - name: Compute upload input | |
| id: input | |
| run: | | |
| sha=${{ github.sha }} | |
| dsha=${{ inputs.duckdb-sha }} | |
| version=${{ needs.build_sdist.outputs.package-version }} | |
| url="s3://duckdb-staging/python/${version}/${sha:0:10}-duckdb-${dsha:0:10}/" | |
| echo "s3_upload_url=${url}" >> $GITHUB_OUTPUT | |
| - name: Authenticate with AWS | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: 'us-east-2' | |
| aws-access-key-id: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| aws-secret-access-key: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| - name: Upload Artifacts | |
| run: | | |
| aws s3 cp artifacts ${{ steps.input.outputs.s3_upload_url }} --recursive | |
| determine_environment: | |
| name: Determine the Github Actions environment to use | |
| runs-on: ubuntu-latest | |
| needs: [build_sdist, build_wheels] | |
| outputs: | |
| env_name: ${{ steps.set-env.outputs.env_name }} | |
| steps: | |
| - name: Set environment name | |
| id: set-env | |
| run: | | |
| set -euo pipefail | |
| case "${{ inputs.pypi-index }}" in | |
| test) | |
| echo "env_name=pypi-test" >> "$GITHUB_OUTPUT" | |
| ;; | |
| prod) | |
| if [[ -n "${{ inputs.stable-version }}" ]]; then | |
| echo "env_name=pypi-prod" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "env_name=pypi-prod-nightly" >> "$GITHUB_OUTPUT" | |
| fi | |
| ;; | |
| *) | |
| echo "::error::invalid combination of inputs.pypi-index='${{ inputs.pypi-index }}' and inputs.stable-version='${{ inputs.stable-version }}'" | |
| exit 1 | |
| ;; | |
| esac | |
| publish_pypi: | |
| name: Publish Artifacts to PyPI | |
| runs-on: ubuntu-latest | |
| needs: determine_environment | |
| environment: | |
| name: ${{ needs.determine_environment.outputs.env_name }} | |
| permissions: | |
| # this is needed for the OIDC flow that is used with trusted publishing on PyPI | |
| id-token: write | |
| steps: | |
| - if: ${{ vars.PYPI_HOST == '' }} | |
| run: | | |
| echo "Error: PYPI_HOST is not set in CI environment '${{ needs.determine_environment.outputs.env_name }}'" | |
| exit 1 | |
| - name: Fetch artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '{sdist,wheel}*' | |
| path: packages/ | |
| merge-multiple: true | |
| - name: Upload artifacts to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| repository-url: 'https://${{ vars.PYPI_HOST }}/legacy/' | |
| packages-dir: packages | |
| verbose: 'true' | |
| cleanup_nightlies: | |
| name: Remove Nightlies from PyPI | |
| needs: [ determine_environment, publish_pypi ] | |
| if: ${{ inputs.stable-version == '' }} | |
| uses: ./.github/workflows/cleanup_pypi.yml | |
| with: | |
| environment: ${{ needs.determine_environment.outputs.env_name }} | |
| secrets: | |
| # reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206 | |
| PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}} | |
| PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}} | |
| summary: | |
| name: Release summary | |
| runs-on: ubuntu-latest | |
| needs: [build_sdist, check_pypi, upload_s3, determine_environment] | |
| if: true | |
| steps: | |
| - run: | | |
| sha=${{ github.sha }} | |
| dsha=${{ inputs.duckdb-sha }} | |
| echo "## Release Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "* Package Version: ${{ needs.build_sdist.outputs.package-version }} (${sha:0:10})" >> $GITHUB_STEP_SUMMARY | |
| echo "* DuckDB Version: ${{ needs.build_sdist.outputs.duckdb-version }} (${dsha:0:10})" >> $GITHUB_STEP_SUMMARY | |
| echo "* PyPI: ${{ vars.PYPI_HOST }}" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ needs.check_pypi.outputs.can_upload }}" == "false" ]; then | |
| echo " * Package already exists on this PyPI, nothing was uploaded" >> $GITHUB_STEP_SUMMARY | |
| exit 0 | |
| fi | |
| echo "* CI Environment: ${{ needs.determine_environment.outputs.env_name }}" >> $GITHUB_STEP_SUMMARY | |
| echo "* S3 URL: ${{ needs.upload_s3.outputs.s3_upload_url }}" >> $GITHUB_STEP_SUMMARY |