Merge branch 'main' into code_quality

Author: carlopi

.github/workflows/MainDistributionPipeline.yml

@@ -19,7 +19,6 @@ jobs:
       extension_name: httpfs
       duckdb_version: v1.3-ossivalis
       ci_tools_version: main
-      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads'
 
 
   duckdb-stable-deploy:
@@ -32,7 +31,6 @@ jobs:
       duckdb_version: v1.3-ossivalis
       ci_tools_version: main
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
-      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads'
 
   code-quality:
     name: Code Quality
@@ -42,4 +40,4 @@ jobs:
       duckdb_version: v1.3-ossivalis
       ci_tools_version: main
       extra_toolchains: 'python3'
-      format_checks: 'format;tidy'
+      format_checks: 'format;tidy'

CMakeLists.txt

@@ -9,16 +9,24 @@ add_extension_definitions()
 include_directories(extension/httpfs/include
                     ${DUCKDB_MODULE_BASE_DIR}/third_party/httplib)
 
+if (NOT EMSCRIPTEN)
+  set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_client.cpp)
+  add_definitions(-DOVERRIDE_ENCRYPTION_UTILS=1)
+else()
+  set(EXTRA_SOURCES extension/httpfs/httpfs_client_wasm.cpp)
+endif()
+
 build_static_extension(
   httpfs
   extension/httpfs/hffs.cpp
   extension/httpfs/s3fs.cpp
   extension/httpfs/httpfs.cpp
-  extension/httpfs/httpfs_client.cpp
   extension/httpfs/http_state.cpp
   extension/httpfs/crypto.cpp
+  extension/httpfs/hash_functions.cpp
   extension/httpfs/create_secret_functions.cpp
-  extension/httpfs/httpfs_extension.cpp)
+  extension/httpfs/httpfs_extension.cpp
+  ${EXTRA_SOURCES} )
 
 set(PARAMETERS "-warnings")
 build_loadable_extension(
@@ -27,11 +35,12 @@ build_loadable_extension(
   extension/httpfs/hffs.cpp
   extension/httpfs/s3fs.cpp
   extension/httpfs/httpfs.cpp
-  extension/httpfs/httpfs_client.cpp
   extension/httpfs/http_state.cpp
   extension/httpfs/crypto.cpp
+  extension/httpfs/hash_functions.cpp
   extension/httpfs/create_secret_functions.cpp
-  extension/httpfs/httpfs_extension.cpp)
+  extension/httpfs/httpfs_extension.cpp
+  ${EXTRA_SOURCES} )
 
 if(MINGW)
   set(OPENSSL_USE_STATIC_LIBS TRUE)

extension/httpfs/README.md

@@ -1 +1 @@
-Documentation on S3 tests setup can be found [here](../../test/sql/copy/s3/README.md)
+Documentation on S3 tests setup can be found [in the duckdb/duckdb repository](https://github.com/duckdb/duckdb/blob/main/test/sql/copy/s3/README.md)

extension/httpfs/crypto.cpp

@@ -1,4 +1,5 @@
 #include "crypto.hpp"
+#include "hash_functions.hpp"
 #include "mbedtls_wrapper.hpp"
 #include <iostream>
 #include "duckdb/common/common.hpp"
@@ -9,28 +10,6 @@
 
 namespace duckdb {
 
-void sha256(const char *in, size_t in_len, hash_bytes &out) {
-	duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(in, in_len, (char *)out);
-}
-
-void hmac256(const std::string &message, const char *secret, size_t secret_len, hash_bytes &out) {
-	duckdb_mbedtls::MbedTlsWrapper::Hmac256(secret, secret_len, message.data(), message.size(), (char *)out);
-}
-
-void hmac256(std::string message, hash_bytes secret, hash_bytes &out) {
-	hmac256(message, (char *)secret, sizeof(hash_bytes), out);
-}
-
-void hex256(hash_bytes &in, hash_str &out) {
-	const char *hex = "0123456789abcdef";
-	unsigned char *pin = in;
-	unsigned char *pout = out;
-	for (; pin < in + sizeof(in); pout += 2, pin++) {
-		pout[0] = hex[(*pin >> 4) & 0xF];
-		pout[1] = hex[*pin & 0xF];
-	}
-}
-
 AESStateSSL::AESStateSSL(const std::string *key) : context(EVP_CIPHER_CTX_new()) {
 	if (!(context)) {
 		throw InternalException("AES GCM failed with initializing context");

extension/httpfs/hash_functions.cpp

@@ -0,0 +1,28 @@
+#include "mbedtls_wrapper.hpp"
+#include "hash_functions.hpp"
+
+namespace duckdb {
+
+void sha256(const char *in, size_t in_len, hash_bytes &out) {
+	duckdb_mbedtls::MbedTlsWrapper::ComputeSha256Hash(in, in_len, (char *)out);
+}
+
+void hmac256(const std::string &message, const char *secret, size_t secret_len, hash_bytes &out) {
+	duckdb_mbedtls::MbedTlsWrapper::Hmac256(secret, secret_len, message.data(), message.size(), (char *)out);
+}
+
+void hmac256(std::string message, hash_bytes secret, hash_bytes &out) {
+	hmac256(message, (char *)secret, sizeof(hash_bytes), out);
+}
+
+void hex256(hash_bytes &in, hash_str &out) {
+	const char *hex = "0123456789abcdef";
+	unsigned char *pin = in;
+	unsigned char *pout = out;
+	for (; pin < in + sizeof(in); pout += 2, pin++) {
+		pout[0] = hex[(*pin >> 4) & 0xF];
+		pout[1] = hex[*pin & 0xF];
+	}
+}
+
+} // namespace duckdb

extension/httpfs/httpfs.cpp

@@ -728,4 +728,9 @@ void HTTPFileHandle::StoreClient(unique_ptr<HTTPClient> client) {
 HTTPFileHandle::~HTTPFileHandle() {
 	DUCKDB_LOG_FILE_SYSTEM_CLOSE((*this));
 };
+
+string HTTPFSUtil::GetName() const {
+	return "HTTPFS";
+}
+
 } // namespace duckdb

extension/httpfs/httpfs_client.cpp

@@ -160,8 +160,4 @@ unordered_map<string, string> HTTPFSUtil::ParseGetParameters(const string &text)
 	return result;
 }
 
-string HTTPFSUtil::GetName() const {
-	return "HTTPFS";
-}
-
 } // namespace duckdb

extension/httpfs/httpfs_client_wasm.cpp

@@ -0,0 +1,16 @@
+#include "httpfs_client.hpp"
+#include "http_state.hpp"
+
+namespace duckdb {
+
+unique_ptr<HTTPClient> HTTPFSUtil::InitializeClient(HTTPParams &http_params, const string &proto_host_port) {
+	throw InternalException("HTTPFSUtil::InitializeClient is not expected to be called");
+}
+
+unordered_map<string, string> HTTPFSUtil::ParseGetParameters(const string &text) {
+	unordered_map<string, string> result;
+	//TODO: HTTPFSUtil::ParseGetParameters is currently not implemented
+	return result;
+}
+
+} // namespace duckdb

extension/httpfs/httpfs_extension.cpp

@@ -6,7 +6,9 @@
 #include "duckdb.hpp"
 #include "s3fs.hpp"
 #include "hffs.hpp"
+#ifdef OVERRIDE_ENCRYPTION_UTILS
 #include "crypto.hpp"
+#endif // OVERRIDE_ENCRYPTION_UTILS
 
 namespace duckdb {
 
@@ -61,16 +63,23 @@ static void LoadInternal(DatabaseInstance &instance) {
 	// HuggingFace options
 	config.AddExtensionOption("hf_max_per_page", "Debug option to limit number of items returned in list requests",
 	                          LogicalType::UBIGINT, Value::UBIGINT(0));
-	config.http_util = make_shared_ptr<HTTPFSUtil>();
+
+	if (config.http_util && config.http_util->GetName() == "WasmHTTPUtils") {
+		// Already handled, do not override
+	} else {
+		config.http_util = make_shared_ptr<HTTPFSUtil>();
+	}
 
 	auto provider = make_uniq<AWSEnvironmentCredentialsProvider>(config);
 	provider->SetAll();
 
 	CreateS3SecretFunctions::Register(instance);
 	CreateBearerTokenFunctions::Register(instance);
 
+#ifdef OVERRIDE_ENCRYPTION_UTILS
 	// set pointer to OpenSSL encryption state
 	config.encryption_util = make_shared_ptr<AESStateSSLFactory>();
+#endif // OVERRIDE_ENCRYPTION_UTILS
 }
 void HttpfsExtension::Load(DuckDB &db) {
 	LoadInternal(*db.instance);

extension/httpfs/include/hash_functions.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "duckdb/common/helper.hpp"
+
+namespace duckdb {
+
+typedef unsigned char hash_bytes[32];
+typedef unsigned char hash_str[64];
+
+void sha256(const char *in, size_t in_len, hash_bytes &out);
+
+void hmac256(const std::string &message, const char *secret, size_t secret_len, hash_bytes &out);
+
+void hmac256(std::string message, hash_bytes secret, hash_bytes &out);
+
+void hex256(hash_bytes &in, hash_str &out);
+
+} // namespace duckdb