Merge pull request #52 from carlopi/httputil_port_changes

HTTPUtil: port changes from duckdb's #17486

Author: Mytherin

duckdb

@@ -45,7 +45,7 @@ static string ParseNextUrlFromLinkHeader(const string &link_header_content) {
 HFFileHandle::~HFFileHandle() {};
 
 unique_ptr<HTTPClient> HFFileHandle::CreateClient() {
-	return http_params.http_util->InitializeClient(http_params, parsed_url.endpoint);
+	return http_params.http_util.InitializeClient(http_params, parsed_url.endpoint);
 }
 
 string HuggingFaceFileSystem::ListHFRequest(ParsedHFUrl &url, HTTPFSParams &http_params, string &next_page_url,
@@ -69,7 +69,7 @@ string HuggingFaceFileSystem::ListHFRequest(ParsedHFUrl &url, HTTPFSParams &http
 		    response << string(const_char_ptr_cast(data), data_length);
 		    return true;
 	    });
-	auto res = http_params.http_util->Request(get_request);
+	auto res = http_params.http_util.Request(get_request);
 	if (res->status != HTTPStatusCode::OK_200) {
 		throw IOException(res->GetError() + " error for HTTP GET to '" + next_page_url + "'");
 	}
@@ -205,7 +205,9 @@ vector<OpenFileInfo> HuggingFaceFileSystem::Glob(const string &path, FileOpener
 
 	FileOpenerInfo info;
 	info.file_path = path;
-	auto http_params = HTTPFSParams::ReadFrom(opener, info);
+	auto http_util = HTTPFSUtil::GetHTTPUtil(opener);
+	auto params = http_util->InitializeParameters(opener, info);
+	auto &http_params = params->Cast<HTTPFSParams>();
 	SetParams(http_params, path, opener);
 	auto http_state = HTTPState::TryGetState(opener).get();
 
@@ -278,10 +280,11 @@ unique_ptr<HTTPFileHandle> HuggingFaceFileSystem::CreateHandle(const OpenFileInf
 	FileOpenerInfo info;
 	info.file_path = file.path;
 
-	auto params = HTTPFSParams::ReadFrom(opener, info);
-	SetParams(params, file.path, opener);
+	auto http_util = HTTPFSUtil::GetHTTPUtil(opener);
+	auto params = http_util->InitializeParameters(opener, info);
+	SetParams(params->Cast<HTTPFSParams>(), file.path, opener);
 
-	return duckdb::make_uniq<HFFileHandle>(*this, std::move(parsed_url), file, flags, params);
+	return duckdb::make_uniq<HFFileHandle>(*this, std::move(parsed_url), file, flags, std::move(params));
 }
 
 void HuggingFaceFileSystem::SetParams(HTTPFSParams &params, const string &path, optional_ptr<FileOpener> opener) {

extension/httpfs/hffs.cpp

@@ -21,7 +21,7 @@
 
 namespace duckdb {
 
-shared_ptr<HTTPUtil> GetHTTPUtil(optional_ptr<FileOpener> opener) {
+shared_ptr<HTTPUtil> HTTPFSUtil::GetHTTPUtil(optional_ptr<FileOpener> opener) {
 	if (opener) {
 		auto db = opener->TryGetDatabase();
 		if (db) {
@@ -32,65 +32,55 @@ shared_ptr<HTTPUtil> GetHTTPUtil(optional_ptr<FileOpener> opener) {
 	return make_shared_ptr<HTTPFSUtil>();
 }
 
-HTTPFSParams HTTPFSParams::ReadFrom(optional_ptr<FileOpener> opener, optional_ptr<FileOpenerInfo> info) {
-	HTTPFSParams result;
-	result.http_util = GetHTTPUtil(opener);
+unique_ptr<HTTPParams> HTTPFSUtil::InitializeParameters(optional_ptr<FileOpener> opener, optional_ptr<FileOpenerInfo> info) {
+	auto result = make_uniq<HTTPFSParams>(*this);
+	result->Initialize(opener);
 
 	// No point in continueing without an opener
 	if (!opener) {
-		return result;
+		return std::move(result);
 	}
 
 	Value value;
 
 	// Setting lookups
-	FileOpener::TryGetCurrentSetting(opener, "http_timeout", result.timeout, info);
-	FileOpener::TryGetCurrentSetting(opener, "force_download", result.force_download, info);
-	FileOpener::TryGetCurrentSetting(opener, "http_retries", result.retries, info);
-	FileOpener::TryGetCurrentSetting(opener, "http_retry_wait_ms", result.retry_wait_ms, info);
-	FileOpener::TryGetCurrentSetting(opener, "http_retry_backoff", result.retry_backoff, info);
-	FileOpener::TryGetCurrentSetting(opener, "http_keep_alive", result.keep_alive, info);
-	FileOpener::TryGetCurrentSetting(opener, "enable_server_cert_verification", result.enable_server_cert_verification,
+	FileOpener::TryGetCurrentSetting(opener, "http_timeout", result->timeout, info);
+	FileOpener::TryGetCurrentSetting(opener, "force_download", result->force_download, info);
+	FileOpener::TryGetCurrentSetting(opener, "http_retries", result->retries, info);
+	FileOpener::TryGetCurrentSetting(opener, "http_retry_wait_ms", result->retry_wait_ms, info);
+	FileOpener::TryGetCurrentSetting(opener, "http_retry_backoff", result->retry_backoff, info);
+	FileOpener::TryGetCurrentSetting(opener, "http_keep_alive", result->keep_alive, info);
+	FileOpener::TryGetCurrentSetting(opener, "enable_server_cert_verification", result->enable_server_cert_verification,
 	                                 info);
-	FileOpener::TryGetCurrentSetting(opener, "ca_cert_file", result.ca_cert_file, info);
-	FileOpener::TryGetCurrentSetting(opener, "hf_max_per_page", result.hf_max_per_page, info);
+	FileOpener::TryGetCurrentSetting(opener, "ca_cert_file", result->ca_cert_file, info);
+	FileOpener::TryGetCurrentSetting(opener, "hf_max_per_page", result->hf_max_per_page, info);
 
 	// HTTP Secret lookups
 	KeyValueSecretReader settings_reader(*opener, info, "http");
 
-	auto client_context = FileOpener::TryGetClientContext(opener);
-	if (client_context) {
-		result.Initialize(*client_context);
-	} else {
-		auto db = FileOpener::TryGetDatabase(opener);
-		if (db) {
-			result.Initialize(*db);
-		}
-	}
-
 	string proxy_setting;
 	if (settings_reader.TryGetSecretKey<string>("http_proxy", proxy_setting) && !proxy_setting.empty()) {
 		idx_t port;
 		string host;
 		HTTPUtil::ParseHTTPProxyHost(proxy_setting, host, port);
-		result.http_proxy = host;
-		result.http_proxy_port = port;
+		result->http_proxy = host;
+		result->http_proxy_port = port;
 	}
-	settings_reader.TryGetSecretKey<string>("http_proxy_username", result.http_proxy_username);
-	settings_reader.TryGetSecretKey<string>("http_proxy_password", result.http_proxy_password);
-	settings_reader.TryGetSecretKey<string>("bearer_token", result.bearer_token);
+	settings_reader.TryGetSecretKey<string>("http_proxy_username", result->http_proxy_username);
+	settings_reader.TryGetSecretKey<string>("http_proxy_password", result->http_proxy_password);
+	settings_reader.TryGetSecretKey<string>("bearer_token", result->bearer_token);
 
 	Value extra_headers;
 	if (settings_reader.TryGetSecretKey("extra_http_headers", extra_headers)) {
 		auto children = MapValue::GetChildren(extra_headers);
 		for (const auto &child : children) {
 			auto kv = StructValue::GetChildren(child);
 			D_ASSERT(kv.size() == 2);
-			result.extra_headers[kv[0].GetValue<string>()] = kv[1].GetValue<string>();
+			result->extra_headers[kv[0].GetValue<string>()] = kv[1].GetValue<string>();
 		}
 	}
 
-	return result;
+	return std::move(result);
 }
 
 unique_ptr<HTTPClient> HTTPClientCache::GetClient() {
@@ -113,7 +103,7 @@ unique_ptr<HTTPResponse> HTTPFileSystem::PostRequest(FileHandle &handle, string
                                                      string &buffer_out, char *buffer_in, idx_t buffer_in_len,
                                                      string params) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
-	auto &http_util = *hfh.http_params.http_util;
+	auto &http_util = hfh.http_params.http_util;
 	PostRequestInfo post_request(url, header_map, hfh.http_params, const_data_ptr_cast(buffer_in), buffer_in_len);
 	auto result = http_util.Request(post_request);
 	buffer_out = std::move(post_request.buffer_out);
@@ -123,7 +113,7 @@ unique_ptr<HTTPResponse> HTTPFileSystem::PostRequest(FileHandle &handle, string
 unique_ptr<HTTPResponse> HTTPFileSystem::PutRequest(FileHandle &handle, string url, HTTPHeaders header_map,
                                                     char *buffer_in, idx_t buffer_in_len, string params) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
-	auto &http_util = *hfh.http_params.http_util;
+	auto &http_util = hfh.http_params.http_util;
 	string content_type = "application/octet-stream";
 	PutRequestInfo put_request(url, header_map, hfh.http_params, (const_data_ptr_t)buffer_in, buffer_in_len,
 	                           content_type);
@@ -132,7 +122,7 @@ unique_ptr<HTTPResponse> HTTPFileSystem::PutRequest(FileHandle &handle, string u
 
 unique_ptr<HTTPResponse> HTTPFileSystem::HeadRequest(FileHandle &handle, string url, HTTPHeaders header_map) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
-	auto &http_util = *hfh.http_params.http_util;
+	auto &http_util = hfh.http_params.http_util;
 	auto http_client = hfh.GetClient();
 
 	HeadRequestInfo head_request(url, header_map, hfh.http_params);
@@ -144,7 +134,7 @@ unique_ptr<HTTPResponse> HTTPFileSystem::HeadRequest(FileHandle &handle, string
 
 unique_ptr<HTTPResponse> HTTPFileSystem::DeleteRequest(FileHandle &handle, string url, HTTPHeaders header_map) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
-	auto &http_util = *hfh.http_params.http_util;
+	auto &http_util = hfh.http_params.http_util;
 	auto http_client = hfh.GetClient();
 	DeleteRequestInfo delete_request(url, header_map, hfh.http_params);
 	auto response = http_util.Request(delete_request, http_client);
@@ -166,7 +156,7 @@ HTTPException HTTPFileSystem::GetHTTPError(FileHandle &, const HTTPResponse &res
 
 unique_ptr<HTTPResponse> HTTPFileSystem::GetRequest(FileHandle &handle, string url, HTTPHeaders header_map) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
-	auto &http_util = *hfh.http_params.http_util;
+	auto &http_util = hfh.http_params.http_util;
 
 	D_ASSERT(hfh.cached_file_handle);
 
@@ -215,7 +205,7 @@ unique_ptr<HTTPResponse> HTTPFileSystem::GetRequest(FileHandle &handle, string u
 unique_ptr<HTTPResponse> HTTPFileSystem::GetRangeRequest(FileHandle &handle, string url, HTTPHeaders header_map,
                                                          idx_t file_offset, char *buffer_out, idx_t buffer_out_len) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
-	auto &http_util = *hfh.http_params.http_util;
+	auto &http_util = hfh.http_params.http_util;
 
 	// send the Range header to read only subset of file
 	string range_expr = "bytes=" + to_string(file_offset) + "-" + to_string(file_offset + buffer_out_len - 1);
@@ -286,8 +276,8 @@ void TimestampToTimeT(timestamp_t timestamp, time_t &result) {
 }
 
 HTTPFileHandle::HTTPFileHandle(FileSystem &fs, const OpenFileInfo &file, FileOpenFlags flags,
-                               HTTPFSParams http_params_p)
-    : FileHandle(fs, file.path, flags), http_params(std::move(http_params_p)), flags(flags), length(0),
+                               unique_ptr<HTTPParams> params_p)
+    : FileHandle(fs, file.path, flags), params(std::move(params_p)), http_params(params->Cast<HTTPFSParams>()), flags(flags), length(0),
       buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0) {
 	// check if the handle has extended properties that can be set directly in the handle
 	// if we have these properties we don't need to do a head request to obtain them later
@@ -318,7 +308,9 @@ unique_ptr<HTTPFileHandle> HTTPFileSystem::CreateHandle(const OpenFileInfo &file
 
 	FileOpenerInfo info;
 	info.file_path = file.path;
-	auto params = HTTPFSParams::ReadFrom(opener, info);
+
+	auto http_util = HTTPFSUtil::GetHTTPUtil(opener);
+	auto params = http_util->InitializeParameters(opener, info);
 
 	auto secret_manager = FileOpener::TryGetSecretManager(opener);
 	auto transaction = FileOpener::TryGetCatalogTransaction(opener);
@@ -327,10 +319,11 @@ unique_ptr<HTTPFileHandle> HTTPFileSystem::CreateHandle(const OpenFileInfo &file
 
 		if (secret_match.HasMatch()) {
 			const auto &kv_secret = dynamic_cast<const KeyValueSecret &>(*secret_match.secret_entry->secret);
-			params.bearer_token = kv_secret.TryGetValue("token", true).ToString();
+			auto &httpfs_params = params->Cast<HTTPFSParams>();
+			httpfs_params.bearer_token = kv_secret.TryGetValue("token", true).ToString();
 		}
 	}
-	return duckdb::make_uniq<HTTPFileHandle>(*this, file, flags, params);
+	return duckdb::make_uniq<HTTPFileHandle>(*this, file, flags, std::move(params));
 }
 
 unique_ptr<FileHandle> HTTPFileSystem::OpenFileExtended(const OpenFileInfo &file, FileOpenFlags flags,
@@ -711,7 +704,7 @@ unique_ptr<HTTPClient> HTTPFileHandle::CreateClient() {
 	// Create a new client
 	string path_out, proto_host_port;
 	HTTPUtil::DecomposeURL(path, path_out, proto_host_port);
-	return http_params.http_util->InitializeClient(http_params, proto_host_port);
+	return http_params.http_util.InitializeClient(http_params, proto_host_port);
 }
 
 void HTTPFileHandle::StoreClient(unique_ptr<HTTPClient> client) {

extension/httpfs/httpfs.cpp

@@ -167,137 +167,8 @@ unordered_map<string, string> HTTPFSUtil::ParseGetParameters(const string &text)
 	return result;
 }
 
-string HTTPFSUtil::GetStatusMessage(HTTPStatusCode status) {
-	switch (status) {
-	case HTTPStatusCode::Continue_100:
-		return "Continue";
-	case HTTPStatusCode::SwitchingProtocol_101:
-		return "Switching Protocol";
-	case HTTPStatusCode::Processing_102:
-		return "Processing";
-	case HTTPStatusCode::EarlyHints_103:
-		return "Early Hints";
-	case HTTPStatusCode::OK_200:
-		return "OK";
-	case HTTPStatusCode::Created_201:
-		return "Created";
-	case HTTPStatusCode::Accepted_202:
-		return "Accepted";
-	case HTTPStatusCode::NonAuthoritativeInformation_203:
-		return "Non-Authoritative Information";
-	case HTTPStatusCode::NoContent_204:
-		return "No Content";
-	case HTTPStatusCode::ResetContent_205:
-		return "Reset Content";
-	case HTTPStatusCode::PartialContent_206:
-		return "Partial Content";
-	case HTTPStatusCode::MultiStatus_207:
-		return "Multi-Status";
-	case HTTPStatusCode::AlreadyReported_208:
-		return "Already Reported";
-	case HTTPStatusCode::IMUsed_226:
-		return "IM Used";
-	case HTTPStatusCode::MultipleChoices_300:
-		return "Multiple Choices";
-	case HTTPStatusCode::MovedPermanently_301:
-		return "Moved Permanently";
-	case HTTPStatusCode::Found_302:
-		return "Found";
-	case HTTPStatusCode::SeeOther_303:
-		return "See Other";
-	case HTTPStatusCode::NotModified_304:
-		return "Not Modified";
-	case HTTPStatusCode::UseProxy_305:
-		return "Use Proxy";
-	case HTTPStatusCode::unused_306:
-		return "unused";
-	case HTTPStatusCode::TemporaryRedirect_307:
-		return "Temporary Redirect";
-	case HTTPStatusCode::PermanentRedirect_308:
-		return "Permanent Redirect";
-	case HTTPStatusCode::BadRequest_400:
-		return "Bad Request";
-	case HTTPStatusCode::Unauthorized_401:
-		return "Unauthorized";
-	case HTTPStatusCode::PaymentRequired_402:
-		return "Payment Required";
-	case HTTPStatusCode::Forbidden_403:
-		return "Forbidden";
-	case HTTPStatusCode::NotFound_404:
-		return "Not Found";
-	case HTTPStatusCode::MethodNotAllowed_405:
-		return "Method Not Allowed";
-	case HTTPStatusCode::NotAcceptable_406:
-		return "Not Acceptable";
-	case HTTPStatusCode::ProxyAuthenticationRequired_407:
-		return "Proxy Authentication Required";
-	case HTTPStatusCode::RequestTimeout_408:
-		return "Request Timeout";
-	case HTTPStatusCode::Conflict_409:
-		return "Conflict";
-	case HTTPStatusCode::Gone_410:
-		return "Gone";
-	case HTTPStatusCode::LengthRequired_411:
-		return "Length Required";
-	case HTTPStatusCode::PreconditionFailed_412:
-		return "Precondition Failed";
-	case HTTPStatusCode::PayloadTooLarge_413:
-		return "Payload Too Large";
-	case HTTPStatusCode::UriTooLong_414:
-		return "URI Too Long";
-	case HTTPStatusCode::UnsupportedMediaType_415:
-		return "Unsupported Media Type";
-	case HTTPStatusCode::RangeNotSatisfiable_416:
-		return "Range Not Satisfiable";
-	case HTTPStatusCode::ExpectationFailed_417:
-		return "Expectation Failed";
-	case HTTPStatusCode::ImATeapot_418:
-		return "I'm a teapot";
-	case HTTPStatusCode::MisdirectedRequest_421:
-		return "Misdirected Request";
-	case HTTPStatusCode::UnprocessableContent_422:
-		return "Unprocessable Content";
-	case HTTPStatusCode::Locked_423:
-		return "Locked";
-	case HTTPStatusCode::FailedDependency_424:
-		return "Failed Dependency";
-	case HTTPStatusCode::TooEarly_425:
-		return "Too Early";
-	case HTTPStatusCode::UpgradeRequired_426:
-		return "Upgrade Required";
-	case HTTPStatusCode::PreconditionRequired_428:
-		return "Precondition Required";
-	case HTTPStatusCode::TooManyRequests_429:
-		return "Too Many Requests";
-	case HTTPStatusCode::RequestHeaderFieldsTooLarge_431:
-		return "Request Header Fields Too Large";
-	case HTTPStatusCode::UnavailableForLegalReasons_451:
-		return "Unavailable For Legal Reasons";
-	case HTTPStatusCode::NotImplemented_501:
-		return "Not Implemented";
-	case HTTPStatusCode::BadGateway_502:
-		return "Bad Gateway";
-	case HTTPStatusCode::ServiceUnavailable_503:
-		return "Service Unavailable";
-	case HTTPStatusCode::GatewayTimeout_504:
-		return "Gateway Timeout";
-	case HTTPStatusCode::HttpVersionNotSupported_505:
-		return "HTTP Version Not Supported";
-	case HTTPStatusCode::VariantAlsoNegotiates_506:
-		return "Variant Also Negotiates";
-	case HTTPStatusCode::InsufficientStorage_507:
-		return "Insufficient Storage";
-	case HTTPStatusCode::LoopDetected_508:
-		return "Loop Detected";
-	case HTTPStatusCode::NotExtended_510:
-		return "Not Extended";
-	case HTTPStatusCode::NetworkAuthenticationRequired_511:
-		return "Network Authentication Required";
-
-	default:
-	case HTTPStatusCode::InternalServerError_500:
-		return "Internal Server Error";
-	}
+string HTTPFSUtil::GetName() const {
+	return "HTTPFS";
 }
 
 } // namespace duckdb

extension/httpfs/httpfs_client.cpp

@@ -57,8 +57,8 @@ class HFFileHandle : public HTTPFileHandle {
 
 public:
 	HFFileHandle(FileSystem &fs, ParsedHFUrl hf_url, const OpenFileInfo &file, FileOpenFlags flags,
-	             const HTTPFSParams &http_params)
-	    : HTTPFileHandle(fs, file, flags, http_params), parsed_url(std::move(hf_url)) {
+	             unique_ptr<HTTPParams> http_params)
+	    : HTTPFileHandle(fs, file, flags, std::move(http_params)), parsed_url(std::move(hf_url)) {
 	}
 	~HFFileHandle() override;