External Dispatch #69
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # External Dispatch is called by duckdb's InvokeCI -> NotifyExternalRepositories job | |
| name: External Dispatch | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| duckdb-sha: | |
| type: string | |
| description: The DuckDB submodule commit to build against | |
| required: true | |
| commit-duckdb-sha: | |
| type: boolean | |
| description: Commit and push the DuckDB submodule ref | |
| default: false | |
| force-version: | |
| type: string | |
| description: Force version (vX.Y.Z-((rc|post)N)) | |
| required: false | |
| publish-packages: | |
| type: boolean | |
| description: Upload packages to S3 | |
| required: true | |
| default: false | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| commit_submodule: | |
| name: Commit the submodule to the given DuckDB sha | |
| outputs: | |
| sha-after-commit: ${{ steps.git_commit_sha.outputs.commit_sha }} | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout DuckDB Python | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.ref }} | |
| fetch-depth: 0 | |
| submodules: true | |
| - name: Checkout DuckDB | |
| run: | | |
| cd external/duckdb | |
| git fetch origin | |
| git checkout ${{ inputs.duckdb-sha }} | |
| - name: Commit and push new submodule ref | |
| if: ${{ inputs.commit-duckdb-sha }} | |
| # see https://github.com/actions/checkout?tab=readme-ov-file#push-a-commit-to-a-pr-using-the-built-in-token | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add external/duckdb | |
| if git diff --cached --quiet; then | |
| echo "No changes to commit: submodule ref is unchanged." | |
| else | |
| git commit -m "Update submodule ref" | |
| git push | |
| fi | |
| - name: Get the SHA of the latest commit | |
| id: git_commit_sha | |
| run: | | |
| echo "commit_sha=$( git rev-parse HEAD )" >> $GITHUB_OUTPUT | |
| externally_triggered_build: | |
| name: Build and test releases | |
| needs: commit_submodule | |
| uses: ./.github/workflows/packaging.yml | |
| with: | |
| minimal: false | |
| testsuite: all | |
| git-ref: ${{ needs.commit_submodule.outputs.sha-after-commit }} | |
| duckdb-git-ref: ${{ inputs.duckdb-sha }} | |
| force-version: ${{ inputs.force-version }} | |
| upload_s3: | |
| name: Upload Artifacts to the S3 Staging Bucket | |
| runs-on: ubuntu-latest | |
| needs: [commit_submodule, externally_triggered_build] | |
| outputs: | |
| version: ${{ steps.s3_upload.outputs.version }} | |
| if: ${{ github.repository_owner == 'duckdb' && inputs.publish-packages }} | |
| steps: | |
| - name: Fetch artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '{sdist,wheel}*' | |
| path: artifacts/ | |
| merge-multiple: true | |
| - name: Authenticate with AWS | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: 'us-east-2' | |
| aws-access-key-id: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| aws-secret-access-key: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| - name: Upload Artifacts | |
| id: s3_upload | |
| run: | | |
| sha=${{ needs.commit_submodule.outputs.sha-after-commit }} | |
| aws s3 cp artifacts s3://duckdb-staging/${{ github.repository }}/${sha:0:10}/ --recursive | |
| echo "version=${version}" >> $GITHUB_OUTPUT | |
| - name: S3 Upload Summary | |
| run : | | |
| sha=${{ needs.commit_submodule.outputs.sha-after-commit }} | |
| version=$(basename artifacts/*.tar.gz | sed 's/duckdb-\(.*\).tar.gz/\1/g') | |
| echo "## S3 Upload Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "* Version: ${version}" >> $GITHUB_STEP_SUMMARY | |
| echo "* SHA: ${sha:0:10}" >> $GITHUB_STEP_SUMMARY | |
| echo "* S3 URL: s3://duckdb-staging/${{ github.repository }}/${sha:0:10}/" >> $GITHUB_STEP_SUMMARY | |
| publish_to_pypi: | |
| name: Upload Artifacts to PyPI | |
| needs: [ commit_submodule, upload_s3 ] | |
| if: ${{ inputs.force-version == '' }} | |
| uses: ./.github/workflows/upload_to_pypi.yml | |
| secrets: inherit | |
| with: | |
| sha: ${{ needs.commit_submodule.outputs.sha-after-commit }} | |
| environment: production.pypi |