Merge v1.3-ossivalis into main

.github/workflows/on_external_dispatch.yml

@@ -6,32 +6,68 @@ on:
         type: string
         description: The DuckDB SHA to build against
         required: true
-      force_version:
+      commit-duckdb-sha:
+        type: boolean
+        description: Set (commit) the DuckDB submodule to the given SHA
+        default: true
+      force-version:
         type: string
         description: Force version (vX.Y.Z-((rc|post)N))
         required: false
-      publish_packages:
+      publish-packages:
         type: boolean
         description: Publish to S3
         required: true
         default: false
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
+  commit_duckdb_submodule_sha:
+    name: Commit the submodule to the given DuckDB sha
+    if: ${{ inputs.commit-duckdb-sha }}
+    runs-on: ubuntu-24.04
+    steps:
+
+      - name: Checkout DuckDB Python
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+          fetch-depth: 0
+          submodules: true
+
+      - name: Checkout DuckDB
+        run: |
+          cd external/duckdb
+          git fetch origin
+          git checkout ${{ inputs.duckdb_git_ref }}
+
+      - name: Commit and push new submodule ref
+        # see https://github.com/actions/checkout?tab=readme-ov-file#push-a-commit-to-a-pr-using-the-built-in-token
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add external/duckdb
+          git commit -m "Update submodule ref"
+          git push
+
   externally_triggered_build:
     name: Build and test releases
-    uses: ./.github/workflows/pypi_packaging.yml
+    uses:  ./.github/workflows/packaging.yml
     with:
       minimal: false
       testsuite: all
-      git_ref: ${{ github.ref }}
-      duckdb_git_ref: ${{ inputs.duckdb-sha }}
-      force_version: ${{ inputs.force_version }}
+      git-ref: ${{ github.ref }}
+      duckdb-git-ref: ${{ inputs.duckdb-sha }}
+      force-version: ${{ inputs.force-version }}
 
-  publish-s3:
+  publish_s3:
     name: Publish Artifacts to the S3 Staging Bucket
     runs-on: ubuntu-latest
     needs: [ externally_triggered_build ]
-    if: ${{ github.repository_owner == 'duckdb' && inputs.publish_packages }}
+    if: ${{ github.repository_owner == 'duckdb' && inputs.publish-packages }}
     steps:
       - name: Fetch artifacts
         uses: actions/download-artifact@v4
@@ -51,7 +87,6 @@ jobs:
         # semantics: if a version is forced then we upload into a folder by the version name, otherwise we upload
         #   into a folder that is named <run id>-<run-attempt>. Only the latter will be discovered be
         #   upload_to_pypi.yml.
-        shell: bash
         run: |
-          FOLDER="${{ inputs.force_version != '' && inputs.force_version || format('{0}-{1}', github.run_id, github.run_attempt) }}"
+          FOLDER="${{ inputs.force-version != '' && inputs.force-version || format('{0}-{1}', github.run_id, github.run_attempt) }}"
           aws s3 cp artifacts s3://duckdb-staging/${{ github.repository }}/${FOLDER}/ --recursive

.github/workflows/on_pr.yml

@@ -24,11 +24,11 @@ jobs:
     name: Build a minimal set of packages and run all tests on them
     # Skip packaging tests for draft PRs
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
-    uses: ./.github/workflows/pypi_packaging.yml
+    uses: ./.github/workflows/packaging.yml
     with:
       minimal: true
       testsuite: all
-      duckdb_git_ref: ${{ github.base_ref }}
+      duckdb-git-ref: ${{ github.base_ref }}
 
   coverage_test:
     name: Run coverage tests

.github/workflows/on_push_postrelease.yml

@@ -33,10 +33,10 @@ jobs:
   packaging_test:
     name: Build and test post release packages and upload to S3
     needs: extract_duckdb_tag
-    uses: ./.github/workflows/pypi_packaging.yml
+    uses: ./.github/workflows/packaging.yml
     with:
       minimal: false
       testsuite: all
-      git_ref: ${{ github.ref }}
-      duckdb_git_ref: ${{ needs.extract_duckdb_tag.outputs.duckdb_version }}
-      force_version: ${{ github.ref_name }}
+      git-ref: ${{ github.ref }}
+      duckdb-git-ref: ${{ needs.extract_duckdb_tag.outputs.duckdb_version }}
+      force-version: ${{ github.ref_name }}

.github/workflows/pypi_packaging.yml → .github/workflows/packaging.yml

@@ -1,5 +1,5 @@
 name: Packaging
-run-name: Build ${{ inputs.minimal && 'minimal set of' || 'all' }} packages (version=${{ inputs.force_version != '' && inputs.force_version || 'dev' }}, tests=${{ inputs.testsuite }}, ref=${{ inputs.git_ref }}, duckdb ref=${{ inputs.duckdb_git_ref }})
+run-name: Build ${{ inputs.minimal && 'minimal set of' || 'all' }} packages (version=${{ inputs.force-version != '' && inputs.force-version || 'dev' }}, tests=${{ inputs.testsuite }}, ref=${{ inputs.git-ref }}, duckdb ref=${{ inputs.duckdb-git-ref }})
 on:
   workflow_dispatch:
     inputs:
@@ -16,16 +16,16 @@ on:
           - none
           - fast
           - all
-      git_ref:
+      git-ref:
         type: string
         description: Git ref of the DuckDB python package
         required: false
-      duckdb_git_ref:
+      duckdb-git-ref:
         type: string
         description: Git ref of DuckDB
         required: true
         default: refs/heads/main
-      force_version:
+      force-version:
         type: string
         description: Force version (vX.Y.Z-((rc|post)N))
         required: false
@@ -40,15 +40,15 @@ on:
         description: Testsuite to run (none, fast, all)
         required: true
         default: all
-      git_ref:
+      git-ref:
         type: string
         description: Git ref of the DuckDB python package
         required: false
-      duckdb_git_ref:
+      duckdb-git-ref:
         type: string
         description: Git ref of DuckDB
         required: false
-      force_version:
+      force-version:
         description: Force version (vX.Y.Z-((rc|post)N))
         required: false
         type: string
@@ -70,7 +70,7 @@ jobs:
       - name: Checkout DuckDB Python
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.git_ref }}
+          ref: ${{ inputs.git-ref }}
           fetch-depth: 0
           submodules: true
 
@@ -79,11 +79,11 @@ jobs:
         run: |
           cd external/duckdb
           git fetch origin
-          git checkout ${{ inputs.duckdb_git_ref }}
+          git checkout ${{ inputs.duckdb-git-ref }}
 
       - name: Set OVERRIDE_GIT_DESCRIBE
-        if: ${{ inputs.force_version != '' }}
-        run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.force_version }}" >> $GITHUB_ENV
+        if: ${{ inputs.force-version != '' }}
+        run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.force-version }}" >> $GITHUB_ENV
 
       - name: Install Astral UV
         uses: astral-sh/setup-uv@v6
@@ -153,7 +153,7 @@ jobs:
       - name: Checkout DuckDB Python
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.git_ref }}
+          ref: ${{ inputs.git-ref }}
           fetch-depth: 0
           submodules: true
 
@@ -162,12 +162,12 @@ jobs:
         run: |
           cd external/duckdb
           git fetch origin
-          git checkout ${{ inputs.duckdb_git_ref }}
+          git checkout ${{ inputs.duckdb-git-ref }}
 
       # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
       - name: Set OVERRIDE_GIT_DESCRIBE
-        if: ${{ inputs.force_version != '' }}
-        run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.force_version }}" >> $GITHUB_ENV
+        if: ${{ inputs.force-version != '' }}
+        run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.force-version }}" >> $GITHUB_ENV
 
       # Install Astral UV, which will be used as build-frontend for cibuildwheel
       - uses: astral-sh/setup-uv@v6

.github/workflows/upload_to_pypi.yml

@@ -17,12 +17,12 @@ on:
         options:
           - test.pypi
           - production.pypi
-      artifact_folder:
-        description: The S3 folder that contains the artifacts (s3://duckdb-staging/duckdb/duckdb-python/<artifact_folder>)
+      artifact-folder:
+        description: The S3 folder that contains the artifacts (s3://duckdb-staging/duckdb/duckdb-python/<artifact-folder>)
         type: string
         required: true
 
-concurrency: ${{ inputs.artifact_folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }}
+concurrency: ${{ inputs.artifact-folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }}
 
 jobs:
   prepare:
@@ -35,7 +35,7 @@ jobs:
       - name: Determine S3 Prefix
         id: get_s3_prefix
         run: |
-          artifact_folder="${{ inputs.artifact_folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }}"
+          artifact_folder="${{ inputs.artifact-folder || format('{0}-{1}', github.event.workflow_run.id, github.event.workflow_run.run_attempt) }}"
           if [[ -n "${artifact_folder}" ]]; then
             s3_prefix="${{ github.repository }}/${artifact_folder}"
             echo "Created S3 prefix: ${s3_prefix}"