Merge pull request duckdb#133 from Mytherin/movesource

Move sources to src/ directory

Author: Mytherin

CMakeLists.txt

@@ -1,48 +1,15 @@
 cmake_minimum_required(VERSION 2.8.12...3.29)
 
-set(HTTPFS_BASE_FOLDER "extension/httpfs")
-
 project(HTTPFsExtension)
 
 add_extension_definitions()
 
-include_directories(extension/httpfs/include
-                    ${DUCKDB_MODULE_BASE_DIR}/third_party/httplib)
-
 if (NOT EMSCRIPTEN)
-  set(EXTRA_SOURCES extension/httpfs/crypto.cpp extension/httpfs/httpfs_httplib_client.cpp extension/httpfs/httpfs_curl_client.cpp)
   add_definitions(-DOVERRIDE_ENCRYPTION_UTILS=1)
 else()
-  set(EXTRA_SOURCES extension/httpfs/httpfs_client_wasm.cpp)
-  set (DUCKDB_EXTENSION_HTTPFS_LINKED_LIBS "../../third_party/mbedtls/libduckdb_mbedtls.a")
+  set(DUCKDB_EXTENSION_HTTPFS_LINKED_LIBS "../../third_party/mbedtls/libduckdb_mbedtls.a")
 endif()
 
-build_static_extension(
-  httpfs
-  extension/httpfs/hffs.cpp
-  extension/httpfs/s3fs.cpp
-  extension/httpfs/httpfs.cpp
-  extension/httpfs/http_state.cpp
-  extension/httpfs/crypto.cpp
-  extension/httpfs/hash_functions.cpp
-  extension/httpfs/create_secret_functions.cpp
-  extension/httpfs/httpfs_extension.cpp
-  ${EXTRA_SOURCES} )
-
-set(PARAMETERS "-warnings")
-build_loadable_extension(
-  httpfs
-  ${PARAMETERS}
-  extension/httpfs/hffs.cpp
-  extension/httpfs/s3fs.cpp
-  extension/httpfs/httpfs.cpp
-  extension/httpfs/http_state.cpp
-  extension/httpfs/crypto.cpp
-  extension/httpfs/hash_functions.cpp
-  extension/httpfs/create_secret_functions.cpp
-  extension/httpfs/httpfs_extension.cpp
-  ${EXTRA_SOURCES} )
-
 if(MINGW)
   set(OPENSSL_USE_STATIC_LIBS TRUE)
 endif()
@@ -51,6 +18,17 @@ find_package(OpenSSL REQUIRED)
 find_package(CURL REQUIRED)
 include_directories(${OPENSSL_INCLUDE_DIR})
 include_directories(${CURL_INCLUDE_DIRS})
+
+include_directories(src/include
+                    ${DUCKDB_MODULE_BASE_DIR}/third_party/httplib)
+add_subdirectory(src)
+set(EXTENSION_SOURCES ${ALL_OBJECT_FILES})
+
+build_static_extension(httpfs ${EXTENSION_SOURCES})
+
+set(PARAMETERS "-warnings")
+build_loadable_extension(httpfs ${PARAMETERS} ${EXTENSION_SOURCES})
+
 if(EMSCRIPTEN)
   target_link_libraries(httpfs_loadable_extension duckdb_mbedtls)
 else()

extension_config.cmake

@@ -6,5 +6,5 @@ duckdb_extension_load(parquet)
 
 duckdb_extension_load(httpfs
 	SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}
-	INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/extension/httpfs/include
+	INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/src/include
 )

src/CMakeLists.txt

@@ -0,0 +1,20 @@
+set(HTTPFS_SOURCES
+    hffs.cpp
+    s3fs.cpp
+    httpfs.cpp
+    http_state.cpp
+    crypto.cpp
+    hash_functions.cpp
+    create_secret_functions.cpp
+    httpfs_extension.cpp)
+if(NOT EMSCRIPTEN)
+  set(HTTPFS_SOURCES ${HTTPFS_SOURCES} crypto.cpp httpfs_httplib_client.cpp
+                     httpfs_curl_client.cpp)
+else()
+  set(HTTPFS_SOURCES ${HTTPFS_SOURCES} httpfs_client_wasm.cpp)
+endif()
+
+add_library(httpfs_library OBJECT ${HTTPFS_SOURCES})
+set(ALL_OBJECT_FILES
+    ${ALL_OBJECT_FILES} $<TARGET_OBJECTS:httpfs_library>
+    PARENT_SCOPE)

extension/httpfs/README.md renamed to src/README.md

@@ -23,7 +23,8 @@
 
 namespace duckdb {
 
-AESStateSSL::AESStateSSL(EncryptionTypes::CipherType  cipher_p, idx_t key_len) : EncryptionState(cipher_p, key_len), context(EVP_CIPHER_CTX_new()) {
+AESStateSSL::AESStateSSL(EncryptionTypes::CipherType cipher_p, idx_t key_len)
+    : EncryptionState(cipher_p, key_len), context(EVP_CIPHER_CTX_new()) {
 	if (!(context)) {
 		throw InternalException("OpenSSL AES failed with initializing context");
 	}
@@ -83,7 +84,8 @@ void AESStateSSL::GenerateRandomData(data_ptr_t data, idx_t len) {
 	RAND_bytes(data, len);
 }
 
-void AESStateSSL::InitializeEncryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len_p, const_data_ptr_t aad, idx_t aad_len) {
+void AESStateSSL::InitializeEncryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len_p,
+                                       const_data_ptr_t aad, idx_t aad_len) {
 	mode = EncryptionTypes::ENCRYPT;
 
 	if (key_len_p != key_len) {
@@ -101,14 +103,15 @@ void AESStateSSL::InitializeEncryption(const_data_ptr_t iv, idx_t iv_len, const_
 	}
 
 	int len;
-	if (aad_len > 0){
+	if (aad_len > 0) {
 		if (!EVP_DecryptUpdate(context, NULL, &len, aad, aad_len)) {
 			throw InternalException("Setting Additional Authenticated Data  failed");
 		}
 	}
 }
 
-void AESStateSSL::InitializeDecryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len_p, const_data_ptr_t aad, idx_t aad_len) {
+void AESStateSSL::InitializeDecryption(const_data_ptr_t iv, idx_t iv_len, const_data_ptr_t key, idx_t key_len_p,
+                                       const_data_ptr_t aad, idx_t aad_len) {
 	mode = EncryptionTypes::DECRYPT;
 	if (key_len_p != key_len) {
 		throw InternalException("Invalid encryption key length, expected %llu, got %llu", key_len, key_len_p);
@@ -126,7 +129,7 @@ void AESStateSSL::InitializeDecryption(const_data_ptr_t iv, idx_t iv_len, const_
 		throw InternalException("EVP_DecryptInit_ex failed to set iv/key");
 	}
 	int len;
-	if (aad_len > 0){
+	if (aad_len > 0) {
 		if (!EVP_DecryptUpdate(context, NULL, &len, aad, aad_len)) {
 			throw InternalException("Setting Additional Authenticated Data  failed");
 		}

extension/httpfs/create_secret_functions.cpp renamed to src/create_secret_functions.cpp

@@ -44,13 +44,14 @@ unique_ptr<HTTPParams> HTTPFSUtil::InitializeParameters(optional_ptr<FileOpener>
 	// Setting lookups
 	FileOpener::TryGetCurrentSetting(opener, "http_timeout", result->timeout, info);
 	FileOpener::TryGetCurrentSetting(opener, "force_download", result->force_download, info);
-	FileOpener::TryGetCurrentSetting(opener, "auto_fallback_to_full_download", result->auto_fallback_to_full_download, info);
+	FileOpener::TryGetCurrentSetting(opener, "auto_fallback_to_full_download", result->auto_fallback_to_full_download,
+	                                 info);
 	FileOpener::TryGetCurrentSetting(opener, "http_retries", result->retries, info);
 	FileOpener::TryGetCurrentSetting(opener, "http_retry_wait_ms", result->retry_wait_ms, info);
 	FileOpener::TryGetCurrentSetting(opener, "http_retry_backoff", result->retry_backoff, info);
 	FileOpener::TryGetCurrentSetting(opener, "http_keep_alive", result->keep_alive, info);
-	FileOpener::TryGetCurrentSetting(opener, "enable_curl_server_cert_verification", result->enable_curl_server_cert_verification,
-	                                 info);
+	FileOpener::TryGetCurrentSetting(opener, "enable_curl_server_cert_verification",
+	                                 result->enable_curl_server_cert_verification, info);
 	FileOpener::TryGetCurrentSetting(opener, "enable_server_cert_verification", result->enable_server_cert_verification,
 	                                 info);
 	FileOpener::TryGetCurrentSetting(opener, "ca_cert_file", result->ca_cert_file, info);
@@ -262,19 +263,23 @@ unique_ptr<HTTPResponse> HTTPFileSystem::GetRangeRequest(FileHandle &handle, str
 			    out_offset = 0;
 
 			    if (!hfh.http_params.unsafe_disable_etag_checks && !hfh.etag.empty() && response.HasHeader("ETag")) {
-                                string responseEtag = response.GetHeaderValue("ETag");
-
-                                if (!responseEtag.empty() && responseEtag != hfh.etag) {
-                                    throw HTTPException(response, "ETag was initially %s and now it returned %s, this likely means the remote file has changed.\nTry to restart the read or close the file-handle and read the file again (e.g. `DETACH` in the file is a database file).\nYou can disable checking etags via `SET unsafe_disable_etag_checks = true;`", hfh.etag, response.GetHeaderValue("ETag"));
-				}
-                            }
-
-
+				    string responseEtag = response.GetHeaderValue("ETag");
+
+				    if (!responseEtag.empty() && responseEtag != hfh.etag) {
+					    throw HTTPException(
+					        response,
+					        "ETag was initially %s and now it returned %s, this likely means the remote file has "
+					        "changed.\nTry to restart the read or close the file-handle and read the file again (e.g. "
+					        "`DETACH` in the file is a database file).\nYou can disable checking etags via `SET "
+					        "unsafe_disable_etag_checks = true;`",
+					        hfh.etag, response.GetHeaderValue("ETag"));
+				    }
+			    }
 
 			    if (response.HasHeader("Content-Length")) {
 				    auto content_length = stoll(response.GetHeaderValue("Content-Length"));
 				    if ((idx_t)content_length != buffer_out_len) {
-				    	RangeRequestNotSupportedException::Throw();
+					    RangeRequestNotSupportedException::Throw();
 				    }
 			    }
 		    }
@@ -388,31 +393,33 @@ unique_ptr<FileHandle> HTTPFileSystem::OpenFileExtended(const OpenFileInfo &file
 	return std::move(handle);
 }
 
-bool HTTPFileSystem::TryRangeRequest(FileHandle &handle, string url, HTTPHeaders header_map, idx_t file_offset, char *buffer_out, idx_t buffer_out_len) {
+bool HTTPFileSystem::TryRangeRequest(FileHandle &handle, string url, HTTPHeaders header_map, idx_t file_offset,
+                                     char *buffer_out, idx_t buffer_out_len) {
 	auto res = GetRangeRequest(handle, url, header_map, file_offset, buffer_out, buffer_out_len);
 
 	if (res) {
 		// Request succeeded TODO: fix upstream that 206 is not considered success
-		if (res->Success() || res->status == HTTPStatusCode::PartialContent_206 || res->status == HTTPStatusCode::Accepted_202) {
+		if (res->Success() || res->status == HTTPStatusCode::PartialContent_206 ||
+		    res->status == HTTPStatusCode::Accepted_202) {
 			return true;
 		}
 
 		// Request failed and we have a request error
 		if (res->HasRequestError()) {
-			ErrorData error (res->GetRequestError());
+			ErrorData error(res->GetRequestError());
 
 			// Special case: we can do a retry with a full file download
-			if (error.Type() == RangeRequestNotSupportedException::TYPE && error.RawMessage() == RangeRequestNotSupportedException::MESSAGE) {
+			if (error.Type() == RangeRequestNotSupportedException::TYPE &&
+			    error.RawMessage() == RangeRequestNotSupportedException::MESSAGE) {
 				auto &hfh = handle.Cast<HTTPFileHandle>();
 				if (hfh.http_params.auto_fallback_to_full_download) {
 					return false;
 				}
-
 			}
 			error.Throw();
 		}
-		throw HTTPException(*res, "Request returned HTTP %d for HTTP %s to '%s'",
-									static_cast<int>(res->status), EnumUtil::ToString(RequestType::GET_REQUEST), res->url);
+		throw HTTPException(*res, "Request returned HTTP %d for HTTP %s to '%s'", static_cast<int>(res->status),
+		                    EnumUtil::ToString(RequestType::GET_REQUEST), res->url);
 	}
 	throw IOException("Unknown error for HTTP %s to '%s'", EnumUtil::ToString(RequestType::GET_REQUEST), url);
 }
@@ -485,15 +492,17 @@ bool HTTPFileSystem::ReadInternal(FileHandle &handle, void *buffer, int64_t nr_b
 
 			// Bypass buffer if we read more than buffer size
 			if (to_read > new_buffer_available) {
-				if (!TryRangeRequest(hfh, hfh.path, {}, location + buffer_offset, (char *)buffer + buffer_offset, to_read)) {
+				if (!TryRangeRequest(hfh, hfh.path, {}, location + buffer_offset, (char *)buffer + buffer_offset,
+				                     to_read)) {
 					return false;
 				}
 				hfh.buffer_available = 0;
 				hfh.buffer_idx = 0;
 				start_offset += to_read;
 				break;
 			} else {
-				if (!TryRangeRequest(hfh, hfh.path, {}, start_offset, (char *)hfh.read_buffer.get(), new_buffer_available)) {
+				if (!TryRangeRequest(hfh, hfh.path, {}, start_offset, (char *)hfh.read_buffer.get(),
+				                     new_buffer_available)) {
 					return false;
 				}
 				hfh.buffer_available = new_buffer_available;
@@ -520,7 +529,10 @@ void HTTPFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, id
 	// attempt to download the full file and retry.
 
 	if (handle.logger) {
-		DUCKDB_LOG_WARN(handle.logger, "Falling back to full file download for file '%s': the server does not support HTTP range requests. Performance and memory usage are potentially degraded.", handle.path);
+		DUCKDB_LOG_WARN(handle.logger,
+		                "Falling back to full file download for file '%s': the server does not support HTTP range "
+		                "requests. Performance and memory usage are potentially degraded.",
+		                handle.path);
 	}
 
 	auto &hfh = handle.Cast<HTTPFileHandle>();
@@ -744,7 +756,6 @@ void HTTPFileHandle::LoadFileInfo() {
 	initialized = true;
 }
 
-
 void HTTPFileHandle::TryAddLogger(FileOpener &opener) {
 	auto context = opener.TryGetClientContext();
 	if (context) {