Merge pull request #109 from duckdb/v1.4-andium

Update main to use duckdb v1.4

Author: samansmink

.github/workflows/MainDistributionPipeline.yml

@@ -17,7 +17,7 @@ jobs:
     uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
     with:
       extension_name: httpfs
-      duckdb_version: v1.3.2
+      duckdb_version: v1.4-andium
       ci_tools_version: main
 
 
@@ -28,6 +28,6 @@ jobs:
     secrets: inherit
     with:
       extension_name: httpfs
-      duckdb_version: v1.3.2
+      duckdb_version: v1.4-andium
       ci_tools_version: main
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

.github/workflows/MinioTests.yml

@@ -18,8 +18,12 @@ jobs:
       AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
       DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000
       DUCKDB_S3_USE_SSL: false
+      CORE_EXTENSIONS: 'parquet;json'
       GEN: ninja
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
       VCPKG_TARGET_TRIPLET: x64-linux
+      PYTHON_HTTP_SERVER_URL: http://localhost:8008
+      PYTHON_HTTP_SERVER_DIR: /tmp/python_test_server
 
     steps:
       - uses: actions/checkout@v4
@@ -48,7 +52,9 @@ jobs:
 
       - name: Build
         shell: bash
-        run: make
+        run: |
+          echo -e "\nduckdb_extension_load(tpch)\n" >> extension_config.cmake
+          make
 
       - name: Start S3/HTTP test server
         shell: bash
@@ -60,6 +66,13 @@ jobs:
           source ./scripts/run_s3_test_server.sh
           sleep 30
 
+      - name: Run & Populate test server
+        shell: bash
+        run: |
+          mkdir -p $PYTHON_HTTP_SERVER_DIR
+          cd $PYTHON_HTTP_SERVER_DIR
+          python3 -m http.server 8008 &
+
       - name: Test
         shell: bash
         run: |

CMakeLists.txt

@@ -10,7 +10,7 @@ include_directories(extension/httpfs/include
                     ${DUCKDB_MODULE_BASE_DIR}/third_party/httplib)
 
 if (NOT EMSCRIPTEN)
-  set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_client.cpp)
+  set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_httplib_client.cpp extension/httpfs/httpfs_curl_client.cpp)
   add_definitions(-DOVERRIDE_ENCRYPTION_UTILS=1)
 else()
   set(EXTRA_SOURCES extension/httpfs/httpfs_client_wasm.cpp)
@@ -48,21 +48,29 @@ if(MINGW)
 endif()
 
 find_package(OpenSSL REQUIRED)
+find_package(CURL REQUIRED)
 include_directories(${OPENSSL_INCLUDE_DIR})
+include_directories(${CURL_INCLUDE_DIRS})
 if(EMSCRIPTEN)
   target_link_libraries(httpfs_loadable_extension duckdb_mbedtls)
 else()
   target_link_libraries(httpfs_loadable_extension duckdb_mbedtls
                         ${OPENSSL_LIBRARIES})
   target_link_libraries(httpfs_extension duckdb_mbedtls ${OPENSSL_LIBRARIES})
 
+  # Link dependencies into extension
+  target_link_libraries(httpfs_loadable_extension ${CURL_LIBRARIES})
+  target_link_libraries(httpfs_extension ${CURL_LIBRARIES})
+
+
   if(MINGW)
     find_package(ZLIB)
     target_link_libraries(httpfs_loadable_extension ZLIB::ZLIB -lcrypt32)
     target_link_libraries(httpfs_extension ZLIB::ZLIB -lcrypt32)
   endif()
 endif()
 
+
 install(
   TARGETS httpfs_extension
   EXPORT "${DUCKDB_EXPORT_SET}"

Makefile

@@ -8,3 +8,17 @@ CORE_EXTENSIONS=''
 
 # Include the Makefile from extension-ci-tools
 include extension-ci-tools/makefiles/duckdb_extension.Makefile
+
+
+## Add some more extra tests
+test_release_internal:
+	./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*"
+	./build/release/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json
+
+test_debug_internal:
+	./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*"
+	./build/debug/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json
+
+test_reldebug_internal:
+	./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*"
+	./build/reldebug/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json

duckdb

@@ -1,15 +1,15 @@
 #include "create_secret_functions.hpp"
 #include "s3fs.hpp"
-#include "duckdb/main/extension_util.hpp"
+#include "duckdb/main/extension/extension_loader.hpp"
 #include "duckdb/common/local_file_system.hpp"
 
 namespace duckdb {
 
-void CreateS3SecretFunctions::Register(DatabaseInstance &instance) {
-	RegisterCreateSecretFunction(instance, "s3");
-	RegisterCreateSecretFunction(instance, "aws");
-	RegisterCreateSecretFunction(instance, "r2");
-	RegisterCreateSecretFunction(instance, "gcs");
+void CreateS3SecretFunctions::Register(ExtensionLoader &loader) {
+	RegisterCreateSecretFunction(loader, "s3");
+	RegisterCreateSecretFunction(loader, "aws");
+	RegisterCreateSecretFunction(loader, "r2");
+	RegisterCreateSecretFunction(loader, "gcs");
 }
 
 static Value MapToStruct(const Value &map) {
@@ -220,39 +220,39 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function,
 	}
 }
 
-void CreateS3SecretFunctions::RegisterCreateSecretFunction(DatabaseInstance &instance, string type) {
+void CreateS3SecretFunctions::RegisterCreateSecretFunction(ExtensionLoader &loader, string type) {
 	// Register the new type
 	SecretType secret_type;
 	secret_type.name = type;
 	secret_type.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
 	secret_type.default_provider = "config";
 	secret_type.extension = "httpfs";
 
-	ExtensionUtil::RegisterSecretType(instance, secret_type);
+	loader.RegisterSecretType(secret_type);
 
 	CreateSecretFunction from_empty_config_fun2 = {type, "config", CreateS3SecretFromConfig};
 	SetBaseNamedParams(from_empty_config_fun2, type);
-	ExtensionUtil::RegisterFunction(instance, from_empty_config_fun2);
+	loader.RegisterFunction(from_empty_config_fun2);
 }
 
-void CreateBearerTokenFunctions::Register(DatabaseInstance &instance) {
+void CreateBearerTokenFunctions::Register(ExtensionLoader &loader) {
 	// HuggingFace secret
 	SecretType secret_type_hf;
 	secret_type_hf.name = HUGGINGFACE_TYPE;
 	secret_type_hf.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
 	secret_type_hf.default_provider = "config";
 	secret_type_hf.extension = "httpfs";
-	ExtensionUtil::RegisterSecretType(instance, secret_type_hf);
+	loader.RegisterSecretType(secret_type_hf);
 
 	// Huggingface config provider
 	CreateSecretFunction hf_config_fun = {HUGGINGFACE_TYPE, "config", CreateBearerSecretFromConfig};
 	hf_config_fun.named_parameters["token"] = LogicalType::VARCHAR;
-	ExtensionUtil::RegisterFunction(instance, hf_config_fun);
+	loader.RegisterFunction(hf_config_fun);
 
 	// Huggingface credential_chain provider
 	CreateSecretFunction hf_cred_fun = {HUGGINGFACE_TYPE, "credential_chain",
 	                                    CreateHuggingFaceSecretFromCredentialChain};
-	ExtensionUtil::RegisterFunction(instance, hf_cred_fun);
+	loader.RegisterFunction(hf_cred_fun);
 }
 
 unique_ptr<BaseSecret> CreateBearerTokenFunctions::CreateSecretFunctionInternal(ClientContext &context,