update duckdb

Author: whzruc

.clang-format

@@ -1,5 +1,6 @@
 ---
 BasedOnStyle: LLVM
+SortIncludes: false
 TabWidth: 4
 IndentWidth: 4
 ColumnLimit: 120

.gitattributes

@@ -0,0 +1 @@
+*.test linguist-language=sql

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -16,19 +16,25 @@ body:
             * [duckdb-odbc](https://github.com/duckdb/duckdb-odbc/issues/new)
             * [duckdb-r](https://github.com/duckdb/duckdb-r/issues/new)
             * [duckdb-rs](https://github.com/duckdb/duckdb-rs/issues/new)
+            * [duckdb-swift](https://github.com/duckdb/duckdb-swift/issues/new)
             * [duckdb-wasm](https://github.com/duckdb/duckdb-wasm/issues/new)
             * [go-duckdb](https://github.com/marcboeker/go-duckdb/issues/new)
           * Extensions:
-            * [Arrow extension](https://github.com/duckdb/arrow/issues/new)
-            * [AWS extension](https://github.com/duckdb/duckdb_aws/issues/new)
-            * [Azure extension](https://github.com/duckdb/duckdb_azure/issues/new)
-            * [Delta extension](https://github.com/duckdb/duckdb_delta/issues/new)
-            * [Iceberg extension](https://github.com/duckdb/duckdb_iceberg/issues/new)
-            * [MySQL extension](https://github.com/duckdb/duckdb_mysql/issues/new)
-            * [Postgres scanner](https://github.com/duckdb/postgres_scanner/issues/new)
-            * [Spatial extension](https://github.com/duckdb/duckdb_spatial/issues/new)
-            * [SQLite scanner](https://github.com/duckdb/sqlite_scanner/issues/new)
-            * [VSS extension](https://github.com/duckdb/duckdb_vss/issues/new)
+            * [AWS extension](https://github.com/duckdb/duckdb-aws/issues/new)
+            * [Azure extension](https://github.com/duckdb/duckdb-azure/issues/new)
+            * [Delta extension](https://github.com/duckdb/duckdb-delta/issues/new)
+            * [Encodings extension](https://github.com/duckdb/duckdb-encodings/issues/new)
+            * [Excel extension](https://github.com/duckdb/duckdb-excel/issues/new)
+            * [fts (full text search) extension](https://github.com/duckdb/duckdb-fts/issues/new)
+            * [httpfs extension](https://github.com/duckdb/duckdb-httpfs/issues/new)
+            * [Iceberg extension](https://github.com/duckdb/duckdb-iceberg/issues/new)
+            * [inet extension](https://github.com/duckdb/duckdb-inet/issues/new)
+            * [MySQL extension](https://github.com/duckdb/duckdb-mysql/issues/new)
+            * [Postgres scanner](https://github.com/duckdb/duckdb-postgres/issues/new)
+            * [Spatial extension](https://github.com/duckdb/duckdb-spatial/issues/new)
+            * [SQLite scanner](https://github.com/duckdb/duckdb-sqlite/issues/new)
+            * [UI](https://github.com/duckdb/duckdb-ui/issues/new)
+            * [VSS extension](https://github.com/duckdb/duckdb-vss/issues/new)
           * Connectors:
             * [dbt-duckdb](https://github.com/duckdb/dbt-duckdb)
 

.github/actions/build_extensions/action.yml

@@ -4,14 +4,14 @@ inputs:
   # Test config
   run_tests:
     description: 'Run extension tests after build'
-    default: 1
+    default: '1'
   run_autoload_tests:
     description: 'Runs the autoloading tests'
-    default: 1
+    default: '1'
 
   # Deploy config
   deploy_as:
-    description: 'Binary architecture name for deploy step'
+    description: 'Binary architecture name for deploy step - DEPRECATED'
     default: ''
   deploy_version:
     description: 'Version tag or commit short hash for deploy step'
@@ -32,19 +32,19 @@ inputs:
     default: ''
   static_link_build:
     description: 'Links DuckDB statically to the loadable extensions'
-    default: 1
+    default: '1'
   no_static_linking:
     description: 'Disables linking extensions into DuckDB for testing'
-    default: 0
+    default: '0'
   vcpkg_build:
     description: 'Installs vcpkg and pass its toolchain to CMakes'
-    default: 1
+    default: '1'
   build_dir:
     description: 'DuckDB source directory to run the build in'
     default: '.'
   ninja:
     description: 'Use ninja for building'
-    default: 0
+    default: '0'
   openssl_path:
     description: 'Directory of OpenSSL installation'
     default: ''
@@ -53,22 +53,28 @@ inputs:
     default: ''
   treat_warn_as_error:
     description: 'Treat compilation warnings as errors'
-    default: 1
+    default: '1'
   build_in_tree_extensions:
     description: 'Build in-tree extensions'
-    default: 1
+    default: '1'
   build_out_of_tree_extensions:
     description: 'Build out-of-tree extensions'
-    default: 1
+    default: '1'
+  build_complete_extensions_set:
+    description: 'Whether all extensions needs to be built'
+    default: '1'
+  bundle_static_lib_mode:
+    description: 'Build the default bundled extensions to publish the static libs'
+    default: '0'
   osx_universal:
     description: 'Build Universal Binary for OSX'
-    default: 0
+    default: '0'
   osx_arch:
     description: 'Build specific architecture for OSX'
     default: ''
   aarch64_cross_compile:
     description: 'Enable Linux aarch64 cross-compiling'
-    default: 0
+    default: '0'
   vcpkg_target_triplet:
     description: 'Target triplet for installing vcpkg dependencies'
     default: ''
@@ -91,22 +97,24 @@ runs:
     - name: Setup DuckDB extension build config
       shell: bash
       run: |
+        export EXTENSION_CONFIGS="$EXTENSION_CONFIGS;${{ inputs.bundle_static_lib_mode == 1 && '.github/config/bundled_extensions.cmake' || ''}}"
         export EXTENSION_CONFIGS="$EXTENSION_CONFIGS;${{ inputs.build_in_tree_extensions == 1 && '.github/config/in_tree_extensions.cmake' || ''}}"
         export EXTENSION_CONFIGS="$EXTENSION_CONFIGS;${{ inputs.build_out_of_tree_extensions == 1 && '.github/config/out_of_tree_extensions.cmake' || '' }}"
         echo "EXTENSION_CONFIGS=$EXTENSION_CONFIGS" >> $GITHUB_ENV
 
     - name: Setup vcpkg
-      if: inputs.vcpkg_build == 1
+      if: ${{ inputs.vcpkg_build == 1 }}
       uses: lukka/run-vcpkg@v11.1
       with:
-        vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+        vcpkgGitCommitId: ce613c41372b23b1f51333815feb3edd87ef8a8b
 
     - name: Set vcpkg env variables
-      if: inputs.vcpkg_build == 1
+      if: ${{ inputs.vcpkg_build == 1 }}
       shell: bash
       run: |
         echo "VCPKG_TOOLCHAIN_PATH=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake" >> $GITHUB_ENV
         echo "VCPKG_TARGET_TRIPLET=${{ inputs.vcpkg_target_triplet }}" >> $GITHUB_ENV
+        echo "BUILD_COMPLETE_EXTENSION_SET=${{ inputs.build_complete_extensions_set }}" >> docker_env.txt
 
     - name: workaround for https://github.com/duckdb/duckdb/issues/8360
       if: inputs.vcpkg_target_triplet == 'x64-windows-static-md'
@@ -126,7 +134,7 @@ runs:
         echo "OPENSSL_ROOT_DIR=${{ inputs.openssl_path }}" >> $GITHUB_ENV
 
     - name: Create combined vcpkg manifest
-      if: inputs.vcpkg_build == 1 && inputs.build_out_of_tree_extensions == 1
+      if: ${{ inputs.vcpkg_build == '1' && inputs.build_out_of_tree_extensions == '1' }}
       shell: bash
       run: |
         make extension_configuration
@@ -146,6 +154,7 @@ runs:
         LOCAL_EXTENSION_REPO: ${{ inputs.run_autoload_tests == 1 && github.workspace || ''}}
         GEN: ${{ inputs.ninja == 1 && 'ninja' || '' }}
         USE_MERGED_VCPKG_MANIFEST: 1
+        # TODO we should no longer override this but we should probably check that it is what we expect
         DUCKDB_PLATFORM: ${{ inputs.duckdb_arch }}
         CMAKE_VARS_BUILD: ${{ inputs.cmake_flags }}
 
@@ -165,7 +174,7 @@ runs:
     # - statically linked extensions are disable on startup but a sqlogictest require will call their load function
     # - loadable-only extensions have their loadable extension loaded with the LOAD statement on a sqlogictest require
     - name: Test statically linked extensions
-      if: ${{ inputs.run_tests == 1 && inputs.no_static_linking == 0}}
+      if: ${{ inputs.run_tests == '1' && inputs.no_static_linking == '0' }}
       shell: bash
       run: |
         ${{ inputs.unittest_script }}
@@ -175,19 +184,19 @@ runs:
       shell: bash
       run: |
         ls
-        cd  ${{ inputs.build_dir}}
+        cd  ${{ inputs.build_dir }}
         ${{ inputs.post_install }}
 
     # The reason we need to rebuild is we need to test auto-loading extensions: this is only possible without the other
     # extensions linked
     - name: Rebuild DuckDB without any extensions, but with all extension tests
-      if: ${{ inputs.run_autoload_tests == 1 }}
+      if: ${{ inputs.run_autoload_tests == '1' }}
       shell: bash
       env:
         EXTENSION_TESTS_ONLY: 1
         ENABLE_EXTENSION_AUTOLOADING: 1
         ENABLE_EXTENSION_AUTOINSTALL: 1
-        GEN: ${{ inputs.ninja == 1 && 'ninja' || '' }}
+        GEN: ${{ inputs.ninja == '1' && 'ninja' || '' }}
         USE_MERGED_VCPKG_MANIFEST: 1
       run: |
         cd  ${{ inputs.build_dir}}
@@ -198,10 +207,11 @@ runs:
 
     # Run all unittests (including the out-of-tree tests) without any extensions linked, relying on the autoloader
     - name: Run tests with auto loading
-      if: ${{ inputs.run_autoload_tests == 1 }}
+      if: ${{ inputs.run_autoload_tests == '1' }}
       shell: bash
       env:
-        LOCAL_EXTENSION_REPO: ${{ inputs.run_autoload_tests == 1 && github.workspace || ''}}
+        LOCAL_EXTENSION_REPO: ${{ inputs.run_autoload_tests == '1' && github.workspace || ''}}
+        DUCKDB_TEST_DESCRIPTION: 'Extension autoloading tests. All `require` calls are ignored and auto-loading is tested. Use require no_extension_autoloading in the test to skip tests.'
       run: |
         cd  ${{ inputs.build_dir}}
         python3 scripts/get_test_list.py --file-contains 'require ' --list '"*.test"' > test.list
@@ -213,35 +223,5 @@ runs:
     - name: Deploy
       if: ${{ inputs.deploy_as != '' }}
       shell: bash
-      env:
-        AWS_ACCESS_KEY_ID: ${{ inputs.s3_id }}
-        AWS_SECRET_ACCESS_KEY: ${{ inputs.s3_key }}
-        DUCKDB_EXTENSION_SIGNING_PK: ${{ inputs.signing_pk }}
-        AWS_DEFAULT_REGION: us-east-1
-        DUCKDB_DEPLOY_SCRIPT_MODE: for_real
-      run: |
-        cd  ${{ inputs.build_dir}}
-        if [[ "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
-          if [[ ! -z "${{ inputs.deploy_version }}" ]] ; then
-            ./scripts/extension-upload-all.sh ${{ inputs.deploy_as }} ${{ inputs.deploy_version }}
-          elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then
-            ./scripts/extension-upload-all.sh ${{ inputs.deploy_as }} ${{ github.ref_name }}
-          elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then
-            ./scripts/extension-upload-all.sh ${{ inputs.deploy_as }} `git log -1 --format=%h`
-          fi
-        fi
-
-    # Run the unittests (excluding the out-of-tree tests) with the extensions that we deployed to S3
-    - name: Test deployed extensions
-      if: ${{ inputs.run_tests == 1 }}
-      shell: bash
-      env:
-        AWS_ACCESS_KEY_ID: ${{ inputs.s3_id }}
-        AWS_SECRET_ACCESS_KEY: ${{ inputs.s3_key }}
-        AWS_DEFAULT_REGION: us-east-1
       run: |
-        rm -rf ~/.duckdb
-        cd  ${{ inputs.build_dir}}
-        if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then
-          ./scripts/extension-upload-test.sh
-        fi
+        exit 1

.github/actions/build_extensions_dockerized/action.yml

@@ -0,0 +1,113 @@
+name: "Build Extensions using Docker"
+description: "Build and test the DuckDB extensions"
+inputs:
+  # Build config
+  run_tests:
+    description: 'Run extension tests after build'
+    default: 'true'
+  duckdb_arch:
+    description: 'Provide DUCKDB_PLATFORM to build system for cross compilation'
+    default: ''
+  vcpkg_target_triplet:
+    description: 'Target triplet for installing vcpkg dependencies'
+    default: ''
+  override_git_describe:
+    description: 'Override git describe'
+    default: ''
+  build_complete_extensions_set:
+    description: 'Whether all extensions needs to be built'
+    default: '1'
+  save_cache:
+    description: 'Should cache be saved'
+    default: '1'
+
+runs:
+  using: "composite"
+  steps:
+      - uses: actions/checkout@v4
+        with:
+          path: 'duckdb'
+          fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
+
+      - uses: actions/checkout@v4
+        name: Checkout Extension CI tools
+        with:
+          path: 'extension-ci-tools'
+          ref: 'main'
+          repository: 'duckdb/extension-ci-tools'
+          fetch-depth: 0
+
+      - name: Populate composed extension
+        shell: bash
+        run: |
+          cat duckdb/.github/config/in_tree_extensions.cmake duckdb/.github/config/out_of_tree_extensions.cmake > extension_config.cmake
+          echo "{\"dependencies\": []}" > vcpkg.json
+          cp duckdb/.github/helper_makefile/build_all_extensions Makefile
+
+      - name: Build Docker image
+        shell: bash
+        run: |
+          docker build \
+            --build-arg 'vcpkg_url=https://github.com/microsoft/vcpkg.git' \
+            --build-arg 'vcpkg_commit=ce613c41372b23b1f51333815feb3edd87ef8a8b' \
+            --build-arg 'extra_toolchains=;python3;' \
+            -t duckdb/${{ inputs.duckdb_arch }} \
+            ./extension-ci-tools/docker/${{ inputs.duckdb_arch }}
+
+      - name: Create env file for docker
+        shell: bash
+        run: |
+          touch docker_env.txt
+          echo "VCPKG_TARGET_TRIPLET=${{ inputs.vcpkg_target_triplet }}" >> docker_env.txt 
+          echo "BUILD_SHELL=1" >> docker_env.txt
+          echo "OPENSSL_ROOT_DIR=/duckdb_build_dir/build/release/vcpkg_installed/${{ inputs.vcpkg_target_triplet }}" >> docker_env.txt
+          echo "OPENSSL_DIR=/duckdb_build_dir/build/release/vcpkg_installed/${{ inputs.vcpkg_target_triplet }}" >> docker_env.txt
+          echo "OPENSSL_USE_STATIC_LIBS=true" >> docker_env.txt
+          # TODO this should also turn into a check
+          echo "DUCKDB_PLATFORM=${{ inputs.duckdb_arch }}" >> docker_env.txt
+          echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.override_git_describe }}" >> docker_env.txt
+          echo "LINUX_CI_IN_DOCKER=1" >> docker_env.txt
+          echo "BUILD_COMPLETE_EXTENSION_SET=${{ inputs.build_complete_extensions_set }}" >> docker_env.txt
+
+      - name: Generate timestamp for Ccache entry
+        shell: bash
+        id: ccache_timestamp
+        run: |
+          date --utc +'timestamp=%Y-%m-%d-%H;%M;%S' >> "$GITHUB_OUTPUT"
+
+      - name: Create Ccache directory
+        shell: bash
+        run: |
+          mkdir ccache_dir
+
+      - name: Load Ccache
+        uses: actions/cache/restore@v4
+        with:
+          path: ./ccache_dir
+          key: ccache-extension-distribution-${{ inputs.duckdb_arch }}-${{ steps.ccache_timestamp.outputs.timestamp }}
+          restore-keys: |
+            ccache-extension-distribution-${{ inputs.duckdb_arch }}-
+
+      - name: Run configure (inside Docker)
+        shell: bash
+        run: |
+          docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ inputs.duckdb_arch }} make configure_ci
+
+      - name: Build extension (inside Docker)
+        shell: bash
+        run: |
+          docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ inputs.duckdb_arch }} make release
+
+      - name: Save Ccache
+        if: ${{ inputs.save_cache == '1' }}
+        uses: actions/cache/save@v4
+        with:
+          path: ./ccache_dir
+          key: ccache-extension-distribution-${{ inputs.duckdb_arch }}-${{ steps.ccache_timestamp.outputs.timestamp }}
+
+      - name: Test extension (inside docker)
+        shell: bash
+        if: ${{ inputs.run_tests == 'true' && inputs.duckdb_arch != 'linux_arm64'}}
+        run: |
+          docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ inputs.duckdb_arch }} make test_release