Skip to content

Commit 0989823

Browse files
samansminksamansmink
authored
Merge pull request duckdb#109 from duckdb/v1.4-andium
Update main to use duckdb v1.4
2 parents 810f64e + 4019119 commit 0989823

29 files changed

+1245
-214
lines changed

‎.github/workflows/MainDistributionPipeline.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
1818
with:
1919
extension_name: httpfs
20-
duckdb_version: v1.3.2
20+
duckdb_version: v1.4-andium
2121
ci_tools_version: main
2222

2323

@@ -28,6 +28,6 @@ jobs:
2828
secrets: inherit
2929
with:
3030
extension_name: httpfs
31-
duckdb_version: v1.3.2
31+
duckdb_version: v1.4-andium
3232
ci_tools_version: main
3333
deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

‎.github/workflows/MinioTests.yml‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,12 @@ jobs:
1818
AWS_SECRET_ACCESS_KEY: minio_duckdb_user_password
1919
DUCKDB_S3_ENDPOINT: duckdb-minio.com:9000
2020
DUCKDB_S3_USE_SSL: false
21+
CORE_EXTENSIONS: 'parquet;json'
2122
GEN: ninja
23+
VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
2224
VCPKG_TARGET_TRIPLET: x64-linux
25+
PYTHON_HTTP_SERVER_URL: http://localhost:8008
26+
PYTHON_HTTP_SERVER_DIR: /tmp/python_test_server
2327

2428
steps:
2529
- uses: actions/checkout@v4
@@ -48,7 +52,9 @@ jobs:
4852

4953
- name: Build
5054
shell: bash
51-
run: make
55+
run: |
56+
echo -e "\nduckdb_extension_load(tpch)\n" >> extension_config.cmake
57+
make
5258
5359
- name: Start S3/HTTP test server
5460
shell: bash
@@ -60,6 +66,13 @@ jobs:
6066
source ./scripts/run_s3_test_server.sh
6167
sleep 30
6268
69+
- name: Run & Populate test server
70+
shell: bash
71+
run: |
72+
mkdir -p $PYTHON_HTTP_SERVER_DIR
73+
cd $PYTHON_HTTP_SERVER_DIR
74+
python3 -m http.server 8008 &
75+
6376
- name: Test
6477
shell: bash
6578
run: |

‎CMakeLists.txt‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ include_directories(extension/httpfs/include
1010
${DUCKDB_MODULE_BASE_DIR}/third_party/httplib)
1111

1212
if (NOT EMSCRIPTEN)
13-
set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_client.cpp)
13+
set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_httplib_client.cpp extension/httpfs/httpfs_curl_client.cpp)
1414
add_definitions(-DOVERRIDE_ENCRYPTION_UTILS=1)
1515
else()
1616
set(EXTRA_SOURCES extension/httpfs/httpfs_client_wasm.cpp)
@@ -48,21 +48,29 @@ if(MINGW)
4848
endif()
4949

5050
find_package(OpenSSL REQUIRED)
51+
find_package(CURL REQUIRED)
5152
include_directories(${OPENSSL_INCLUDE_DIR})
53+
include_directories(${CURL_INCLUDE_DIRS})
5254
if(EMSCRIPTEN)
5355
target_link_libraries(httpfs_loadable_extension duckdb_mbedtls)
5456
else()
5557
target_link_libraries(httpfs_loadable_extension duckdb_mbedtls
5658
${OPENSSL_LIBRARIES})
5759
target_link_libraries(httpfs_extension duckdb_mbedtls ${OPENSSL_LIBRARIES})
5860

61+
# Link dependencies into extension
62+
target_link_libraries(httpfs_loadable_extension ${CURL_LIBRARIES})
63+
target_link_libraries(httpfs_extension ${CURL_LIBRARIES})
64+
65+
5966
if(MINGW)
6067
find_package(ZLIB)
6168
target_link_libraries(httpfs_loadable_extension ZLIB::ZLIB -lcrypt32)
6269
target_link_libraries(httpfs_extension ZLIB::ZLIB -lcrypt32)
6370
endif()
6471
endif()
6572

73+
6674
install(
6775
TARGETS httpfs_extension
6876
EXPORT "${DUCKDB_EXPORT_SET}"

‎Makefile‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,17 @@ CORE_EXTENSIONS=''
88

99
# Include the Makefile from extension-ci-tools
1010
include extension-ci-tools/makefiles/duckdb_extension.Makefile
11+
12+
13+
## Add some more extra tests
14+
test_release_internal:
15+
./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*"
16+
./build/release/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json
17+
18+
test_debug_internal:
19+
./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*"
20+
./build/debug/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json
21+
22+
test_reldebug_internal:
23+
./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*"
24+
./build/reldebug/$(TEST_PATH) --test-dir duckdb --test-config test/configs/duckdb-tests.json

‎duckdb‎

Submodule duckdb updated 3333 files

‎extension/httpfs/create_secret_functions.cpp‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
#include "create_secret_functions.hpp"
22
#include "s3fs.hpp"
3-
#include "duckdb/main/extension_util.hpp"
3+
#include "duckdb/main/extension/extension_loader.hpp"
44
#include "duckdb/common/local_file_system.hpp"
55

66
namespace duckdb {
77

8-
void CreateS3SecretFunctions::Register(DatabaseInstance &instance) {
9-
RegisterCreateSecretFunction(instance, "s3");
10-
RegisterCreateSecretFunction(instance, "aws");
11-
RegisterCreateSecretFunction(instance, "r2");
12-
RegisterCreateSecretFunction(instance, "gcs");
8+
void CreateS3SecretFunctions::Register(ExtensionLoader &loader) {
9+
RegisterCreateSecretFunction(loader, "s3");
10+
RegisterCreateSecretFunction(loader, "aws");
11+
RegisterCreateSecretFunction(loader, "r2");
12+
RegisterCreateSecretFunction(loader, "gcs");
1313
}
1414

1515
static Value MapToStruct(const Value &map) {
@@ -220,39 +220,39 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function,
220220
}
221221
}
222222

223-
void CreateS3SecretFunctions::RegisterCreateSecretFunction(DatabaseInstance &instance, string type) {
223+
void CreateS3SecretFunctions::RegisterCreateSecretFunction(ExtensionLoader &loader, string type) {
224224
// Register the new type
225225
SecretType secret_type;
226226
secret_type.name = type;
227227
secret_type.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
228228
secret_type.default_provider = "config";
229229
secret_type.extension = "httpfs";
230230

231-
ExtensionUtil::RegisterSecretType(instance, secret_type);
231+
loader.RegisterSecretType(secret_type);
232232

233233
CreateSecretFunction from_empty_config_fun2 = {type, "config", CreateS3SecretFromConfig};
234234
SetBaseNamedParams(from_empty_config_fun2, type);
235-
ExtensionUtil::RegisterFunction(instance, from_empty_config_fun2);
235+
loader.RegisterFunction(from_empty_config_fun2);
236236
}
237237

238-
void CreateBearerTokenFunctions::Register(DatabaseInstance &instance) {
238+
void CreateBearerTokenFunctions::Register(ExtensionLoader &loader) {
239239
// HuggingFace secret
240240
SecretType secret_type_hf;
241241
secret_type_hf.name = HUGGINGFACE_TYPE;
242242
secret_type_hf.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
243243
secret_type_hf.default_provider = "config";
244244
secret_type_hf.extension = "httpfs";
245-
ExtensionUtil::RegisterSecretType(instance, secret_type_hf);
245+
loader.RegisterSecretType(secret_type_hf);
246246

247247
// Huggingface config provider
248248
CreateSecretFunction hf_config_fun = {HUGGINGFACE_TYPE, "config", CreateBearerSecretFromConfig};
249249
hf_config_fun.named_parameters["token"] = LogicalType::VARCHAR;
250-
ExtensionUtil::RegisterFunction(instance, hf_config_fun);
250+
loader.RegisterFunction(hf_config_fun);
251251

252252
// Huggingface credential_chain provider
253253
CreateSecretFunction hf_cred_fun = {HUGGINGFACE_TYPE, "credential_chain",
254254
CreateHuggingFaceSecretFromCredentialChain};
255-
ExtensionUtil::RegisterFunction(instance, hf_cred_fun);
255+
loader.RegisterFunction(hf_cred_fun);
256256
}
257257

258258
unique_ptr<BaseSecret> CreateBearerTokenFunctions::CreateSecretFunctionInternal(ClientContext &context,

0 commit comments

Comments
 (0)