Merge pull request #117 from carlopi/duckdb_user_agent

Set User-Agent to DuckDB's custom db.config.UserAgent()

Author: samansmink

extension/httpfs/httpfs.cpp

@@ -57,6 +57,13 @@ unique_ptr<HTTPParams> HTTPFSUtil::InitializeParameters(optional_ptr<FileOpener>
 	FileOpener::TryGetCurrentSetting(opener, "hf_max_per_page", result->hf_max_per_page, info);
 	FileOpener::TryGetCurrentSetting(opener, "unsafe_disable_etag_checks", result->unsafe_disable_etag_checks, info);
 
+	{
+		auto db = FileOpener::TryGetDatabase(opener);
+		if (db) {
+			result->user_agent = StringUtil::Format("%s %s", db->config.UserAgent(), DuckDB::SourceID());
+		}
+	}
+
 	// HTTP Secret lookups
 	KeyValueSecretReader settings_reader(*opener, info, "http");
 
@@ -101,11 +108,20 @@ void HTTPClientCache::StoreClient(unique_ptr<HTTPClient> client) {
 	clients.push_back(std::move(client));
 }
 
+static void AddUserAgentIfAvailable(HTTPFSParams &http_params, HTTPHeaders &header_map) {
+	if (!http_params.user_agent.empty()) {
+		header_map.Insert("User-Agent", http_params.user_agent);
+	}
+}
+
 unique_ptr<HTTPResponse> HTTPFileSystem::PostRequest(FileHandle &handle, string url, HTTPHeaders header_map,
                                                      string &buffer_out, char *buffer_in, idx_t buffer_in_len,
                                                      string params) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
 	auto &http_util = hfh.http_params.http_util;
+
+	AddUserAgentIfAvailable(hfh.http_params, header_map);
+
 	PostRequestInfo post_request(url, header_map, hfh.http_params, const_data_ptr_cast(buffer_in), buffer_in_len);
 	auto result = http_util.Request(post_request);
 	buffer_out = std::move(post_request.buffer_out);
@@ -116,6 +132,9 @@ unique_ptr<HTTPResponse> HTTPFileSystem::PutRequest(FileHandle &handle, string u
                                                     char *buffer_in, idx_t buffer_in_len, string params) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
 	auto &http_util = hfh.http_params.http_util;
+
+	AddUserAgentIfAvailable(hfh.http_params, header_map);
+
 	string content_type = "application/octet-stream";
 	PutRequestInfo put_request(url, header_map, hfh.http_params, (const_data_ptr_t)buffer_in, buffer_in_len,
 	                           content_type);
@@ -125,6 +144,9 @@ unique_ptr<HTTPResponse> HTTPFileSystem::PutRequest(FileHandle &handle, string u
 unique_ptr<HTTPResponse> HTTPFileSystem::HeadRequest(FileHandle &handle, string url, HTTPHeaders header_map) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
 	auto &http_util = hfh.http_params.http_util;
+
+	AddUserAgentIfAvailable(hfh.http_params, header_map);
+
 	auto http_client = hfh.GetClient();
 
 	HeadRequestInfo head_request(url, header_map, hfh.http_params);
@@ -137,6 +159,9 @@ unique_ptr<HTTPResponse> HTTPFileSystem::HeadRequest(FileHandle &handle, string
 unique_ptr<HTTPResponse> HTTPFileSystem::DeleteRequest(FileHandle &handle, string url, HTTPHeaders header_map) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
 	auto &http_util = hfh.http_params.http_util;
+
+	AddUserAgentIfAvailable(hfh.http_params, header_map);
+
 	auto http_client = hfh.GetClient();
 	DeleteRequestInfo delete_request(url, header_map, hfh.http_params);
 	auto response = http_util.Request(delete_request, http_client);
@@ -160,6 +185,8 @@ unique_ptr<HTTPResponse> HTTPFileSystem::GetRequest(FileHandle &handle, string u
 	auto &hfh = handle.Cast<HTTPFileHandle>();
 	auto &http_util = hfh.http_params.http_util;
 
+	AddUserAgentIfAvailable(hfh.http_params, header_map);
+
 	D_ASSERT(hfh.cached_file_handle);
 
 	auto http_client = hfh.GetClient();
@@ -209,6 +236,8 @@ unique_ptr<HTTPResponse> HTTPFileSystem::GetRangeRequest(FileHandle &handle, str
 	auto &hfh = handle.Cast<HTTPFileHandle>();
 	auto &http_util = hfh.http_params.http_util;
 
+	AddUserAgentIfAvailable(hfh.http_params, header_map);
+
 	// send the Range header to read only subset of file
 	string range_expr = "bytes=" + to_string(file_offset) + "-" + to_string(file_offset + buffer_out_len - 1);
 	header_map.Insert("Range", range_expr);

extension/httpfs/include/httpfs_client.hpp

@@ -22,6 +22,7 @@ struct HTTPFSParams : public HTTPParams {
 	bool enable_server_cert_verification = DEFAULT_ENABLE_SERVER_CERT_VERIFICATION;
 	bool enable_curl_server_cert_verification = true;
 	idx_t hf_max_per_page = DEFAULT_HF_MAX_PER_PAGE;
+	string user_agent = {""};
 	string ca_cert_file;
 	string bearer_token;
 	bool unsafe_disable_etag_checks {false};