Apply logger patch

Author: Mytherin

duckdb

@@ -158,7 +158,7 @@ bool CreateS3SecretFunctions::TryRefreshS3Secret(ClientContext &context, const S
 	try {
 		auto res = secret_manager.CreateSecret(context, refresh_input);
 		auto &new_secret = dynamic_cast<const KeyValueSecret &>(*res->secret);
-		DUCKDB_LOG_INFO(context, "httpfs.SecretRefresh", "Successfully refreshed secret: %s, new key_id: %s",
+		DUCKDB_LOG_INFO(context, "Successfully refreshed secret: %s, new key_id: %s",
 		                secret_to_refresh.secret->GetName(), new_secret.TryGetValue("key_id").ToString());
 		return true;
 	} catch (std::exception &ex) {

extension/httpfs/create_secret_functions.cpp

@@ -8,7 +8,7 @@
 #include "duckdb/common/thread.hpp"
 #include "duckdb/common/types/hash.hpp"
 #include "duckdb/function/scalar/strftime_format.hpp"
-#include "duckdb/logging/http_logger.hpp"
+#include "duckdb/logging/file_system_logger.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/main/secret/secret_manager.hpp"
@@ -32,7 +32,8 @@ shared_ptr<HTTPUtil> HTTPFSUtil::GetHTTPUtil(optional_ptr<FileOpener> opener) {
 	return make_shared_ptr<HTTPFSUtil>();
 }
 
-unique_ptr<HTTPParams> HTTPFSUtil::InitializeParameters(optional_ptr<FileOpener> opener, optional_ptr<FileOpenerInfo> info) {
+unique_ptr<HTTPParams> HTTPFSUtil::InitializeParameters(optional_ptr<FileOpener> opener,
+                                                        optional_ptr<FileOpenerInfo> info) {
 	auto result = make_uniq<HTTPFSParams>(*this);
 	result->Initialize(opener);
 
@@ -277,8 +278,8 @@ void TimestampToTimeT(timestamp_t timestamp, time_t &result) {
 
 HTTPFileHandle::HTTPFileHandle(FileSystem &fs, const OpenFileInfo &file, FileOpenFlags flags,
                                unique_ptr<HTTPParams> params_p)
-    : FileHandle(fs, file.path, flags), params(std::move(params_p)), http_params(params->Cast<HTTPFSParams>()), flags(flags), length(0),
-      buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0) {
+    : FileHandle(fs, file.path, flags), params(std::move(params_p)), http_params(params->Cast<HTTPFSParams>()),
+      flags(flags), length(0), buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0) {
 	// check if the handle has extended properties that can be set directly in the handle
 	// if we have these properties we don't need to do a head request to obtain them later
 	if (file.extended_info) {
@@ -342,6 +343,9 @@ unique_ptr<FileHandle> HTTPFileSystem::OpenFileExtended(const OpenFileInfo &file
 
 	auto handle = CreateHandle(file, flags, opener);
 	handle->Initialize(opener);
+
+	DUCKDB_LOG_FILE_SYSTEM_OPEN((*handle));
+
 	return std::move(handle);
 }
 
@@ -356,6 +360,8 @@ void HTTPFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, id
 			throw InternalException("Cached file not initialized properly");
 		}
 		memcpy(buffer, hfh.cached_file_handle->GetData() + location, nr_bytes);
+		DUCKDB_LOG_FILE_SYSTEM_READ(handle, nr_bytes, location);
+		hfh.file_offset = location + nr_bytes;
 		return;
 	}
 
@@ -366,17 +372,19 @@ void HTTPFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, id
 	bool skip_buffer = hfh.flags.DirectIO() || hfh.flags.RequireParallelAccess();
 	if (skip_buffer && to_read > 0) {
 		GetRangeRequest(hfh, hfh.path, {}, location, (char *)buffer, to_read);
-
+		DUCKDB_LOG_FILE_SYSTEM_READ(handle, nr_bytes, location);
 		// Update handle status within critical section for parallel access.
 		if (hfh.flags.RequireParallelAccess()) {
 			std::lock_guard<std::mutex> lck(hfh.mu);
 			hfh.buffer_available = 0;
 			hfh.buffer_idx = 0;
+			hfh.file_offset = location + nr_bytes;
 			return;
 		}
 
 		hfh.buffer_available = 0;
 		hfh.buffer_idx = 0;
+		hfh.file_offset = location + nr_bytes;
 		return;
 	}
 
@@ -423,14 +431,15 @@ void HTTPFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, id
 			}
 		}
 	}
+	hfh.file_offset = location + nr_bytes;
+	DUCKDB_LOG_FILE_SYSTEM_READ(handle, nr_bytes, location);
 }
 
 int64_t HTTPFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
 	auto &hfh = handle.Cast<HTTPFileHandle>();
 	idx_t max_read = hfh.length - hfh.file_offset;
 	nr_bytes = MinValue<idx_t>(max_read, nr_bytes);
 	Read(handle, buffer, nr_bytes, hfh.file_offset);
-	hfh.file_offset += nr_bytes;
 	return nr_bytes;
 }
 
@@ -642,6 +651,10 @@ void HTTPFileHandle::Initialize(optional_ptr<FileOpener> opener) {
 		http_params.state = make_shared_ptr<HTTPState>();
 	}
 
+	if (opener) {
+		TryAddLogger(*opener);
+	}
+
 	auto current_cache = TryGetMetadataCache(opener, hfs);
 
 	bool should_write_cache = false;
@@ -711,5 +724,7 @@ void HTTPFileHandle::StoreClient(unique_ptr<HTTPClient> client) {
 	client_cache.StoreClient(std::move(client));
 }
 
-HTTPFileHandle::~HTTPFileHandle() = default;
+HTTPFileHandle::~HTTPFileHandle() {
+	DUCKDB_LOG_FILE_SYSTEM_CLOSE((*this));
+};
 } // namespace duckdb

extension/httpfs/httpfs.cpp

@@ -1,6 +1,5 @@
 #include "httpfs_client.hpp"
 #include "http_state.hpp"
-#include "duckdb/logging/http_logger.hpp"
 
 #define CPPHTTPLIB_OPENSSL_SUPPORT
 #include "httplib.hpp"
@@ -21,9 +20,6 @@ class HTTPFSClient : public HTTPClient {
 		client->set_read_timeout(http_params.timeout, http_params.timeout_usec);
 		client->set_connection_timeout(http_params.timeout, http_params.timeout_usec);
 		client->set_decompress(false);
-		if (http_params.logger) {
-			SetLogger(*http_params.logger);
-		}
 		if (!http_params.bearer_token.empty()) {
 			client->set_bearer_token_auth(http_params.bearer_token.c_str());
 		}
@@ -38,9 +34,6 @@ class HTTPFSClient : public HTTPClient {
 		state = http_params.state;
 	}
 
-	void SetLogger(HTTPLogger &logger) {
-		client->set_logger(logger.GetLogger<duckdb_httplib_openssl::Request, duckdb_httplib_openssl::Response>());
-	}
 	unique_ptr<HTTPResponse> Get(GetRequestInfo &info) override {
 		if (state) {
 			state->get_count++;

extension/httpfs/httpfs_client.cpp

@@ -9,7 +9,8 @@ struct FileOpenerInfo;
 class HTTPState;
 
 struct HTTPFSParams : public HTTPParams {
-	HTTPFSParams(HTTPUtil &http_util) : HTTPParams(http_util) {}
+	HTTPFSParams(HTTPUtil &http_util) : HTTPParams(http_util) {
+	}
 
 	static constexpr bool DEFAULT_ENABLE_SERVER_CERT_VERIFICATION = false;
 	static constexpr uint64_t DEFAULT_HF_MAX_PER_PAGE = 0;
@@ -25,7 +26,8 @@ struct HTTPFSParams : public HTTPParams {
 
 class HTTPFSUtil : public HTTPUtil {
 public:
-	unique_ptr<HTTPParams> InitializeParameters(optional_ptr<FileOpener> opener, optional_ptr<FileOpenerInfo> info) override;
+	unique_ptr<HTTPParams> InitializeParameters(optional_ptr<FileOpener> opener,
+	                                            optional_ptr<FileOpenerInfo> info) override;
 	unique_ptr<HTTPClient> InitializeClient(HTTPParams &http_params, const string &proto_host_port) override;
 
 	static unordered_map<string, string> ParseGetParameters(const string &text);

extension/httpfs/include/httpfs_client.hpp

@@ -4,6 +4,8 @@
 #include "duckdb.hpp"
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/exception/http_exception.hpp"
+#include "duckdb/logging/log_type.hpp"
+#include "duckdb/logging/file_system_logger.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/thread.hpp"
 #include "duckdb/common/types/timestamp.hpp"
@@ -838,6 +840,8 @@ void S3FileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx
 		s3fh.file_offset += bytes_to_write;
 		bytes_written += bytes_to_write;
 	}
+
+	DUCKDB_LOG_FILE_SYSTEM_WRITE(handle, bytes_written, s3fh.file_offset - bytes_written);
 }
 
 static bool Match(vector<string>::const_iterator key, vector<string>::const_iterator key_end,
@@ -918,8 +922,8 @@ vector<OpenFileInfo> S3FileSystem::Glob(const string &glob_pattern, FileOpener *
 			string common_prefix_continuation_token;
 			do {
 				auto prefix_res =
-				    AWSListObjectV2::Request(prefix_path, *http_params, s3_auth_params, common_prefix_continuation_token,
-				                             HTTPState::TryGetState(opener).get());
+				    AWSListObjectV2::Request(prefix_path, *http_params, s3_auth_params,
+				                             common_prefix_continuation_token, HTTPState::TryGetState(opener).get());
 				AWSListObjectV2::ParseFileList(prefix_res, s3_keys);
 				auto more_prefixes = AWSListObjectV2::ParseCommonPrefix(prefix_res);
 				common_prefixes.insert(common_prefixes.end(), more_prefixes.begin(), more_prefixes.end());

extension/httpfs/s3fs.cpp

@@ -61,7 +61,7 @@ statement ok
 FROM "s3://test-bucket/test-file.parquet"
 
 query I
-SELECT message[0:46] FROM duckdb_logs WHERE type='httpfs.SecretRefresh'
+SELECT message[0:46] FROM duckdb_logs WHERE message like '%Successfully refreshed secret%'
 ----
 Successfully refreshed secret: s1, new key_id:
 
@@ -84,7 +84,7 @@ FROM "s3://test-bucket/test-file.parquet"
 HTTP 403
 
 query I
-SELECT message[0:46] FROM duckdb_logs WHERE type='httpfs.SecretRefresh'
+SELECT message[0:46] FROM duckdb_logs WHERE message like '%Successfully refreshed secret%'
 ----
 Successfully refreshed secret: s1, new key_id:
 
@@ -125,5 +125,5 @@ HTTP 403
 
 # -> log empty
 query II
-SELECT log_level, message FROM duckdb_logs WHERE type='httpfs.SecretRefresh'
+SELECT log_level, message FROM duckdb_logs WHERE message like '%Successfully refreshed secret%'
 ----

test/sql/secret/secret_refresh.test

@@ -45,6 +45,6 @@ ATTACH 's3://test-bucket/presigned/attach.db' AS db (READONLY 1);
 
 # Secret refresh has been triggered
 query II
-SELECT log_level, message FROM duckdb_logs WHERE type='httpfs.SecretRefresh'
+SELECT log_level, message FROM duckdb_logs WHERE message like '%Successfully refreshed secret%'
 ----
 INFO	Successfully refreshed secret: uhuh_this_mah_sh, new key_id: all the girls