Merge branch 'main' of https://github.com/duckdb/duckdb-httpfs

# Conflicts:
#	extension/httpfs/s3fs.cpp

Author: prmoore77

extension/httpfs/create_secret_functions.cpp

@@ -111,7 +111,7 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
 		} else if (lower_name == "requester_pays") {
 			if (named_param.second.type() != LogicalType::BOOLEAN) {
 				throw InvalidInputException("Invalid type past to secret option: '%s', found '%s', expected: 'BOOLEAN'",
-											lower_name, named_param.second.type().ToString());
+				                            lower_name, named_param.second.type().ToString());
 			}
 			secret->secret_map["requester_pays"] = Value::BOOLEAN(named_param.second.GetValue<bool>());
 		} else if (lower_name == "bearer_token" && input.type == "gcs") {
@@ -195,7 +195,7 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function,
 	function.named_parameters["use_ssl"] = LogicalType::BOOLEAN;
 	function.named_parameters["kms_key_id"] = LogicalType::VARCHAR;
 	function.named_parameters["url_compatibility_mode"] = LogicalType::BOOLEAN;
-    function.named_parameters["requester_pays"] = LogicalType::BOOLEAN;
+	function.named_parameters["requester_pays"] = LogicalType::BOOLEAN;
 
 	// Whether a secret refresh attempt should be made when the secret appears to be incorrect
 	function.named_parameters["refresh"] = LogicalType::VARCHAR;
@@ -214,7 +214,7 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function,
 	if (type == "r2") {
 		function.named_parameters["account_id"] = LogicalType::VARCHAR;
 	}
-	
+
 	if (type == "gcs") {
 		function.named_parameters["bearer_token"] = LogicalType::VARCHAR;
 	}

extension/httpfs/httpfs.cpp

@@ -51,6 +51,7 @@ unique_ptr<HTTPParams> HTTPFSUtil::InitializeParameters(optional_ptr<FileOpener>
 	                                 info);
 	FileOpener::TryGetCurrentSetting(opener, "ca_cert_file", result->ca_cert_file, info);
 	FileOpener::TryGetCurrentSetting(opener, "hf_max_per_page", result->hf_max_per_page, info);
+	FileOpener::TryGetCurrentSetting(opener, "unsafe_disable_etag_checks", result->unsafe_disable_etag_checks, info);
 
 	// HTTP Secret lookups
 	KeyValueSecretReader settings_reader(*opener, info, "http");
@@ -226,6 +227,17 @@ unique_ptr<HTTPResponse> HTTPFileSystem::GetRangeRequest(FileHandle &handle, str
 		    }
 		    if (static_cast<int>(response.status) < 300) { // done redirecting
 			    out_offset = 0;
+
+			    if (!hfh.http_params.unsafe_disable_etag_checks && hfh.etag.empty() && response.HasHeader("ETag")) {
+                                string responseEtag = response.GetHeaderValue("ETag");
+
+                                if (!responseEtag.empty() && responseEtag != hfh.etag) {
+                                    throw HTTPException(response, "ETag was initially %s and now it returned %s, this likely means the remote file has changed.\nTry to restart the read or close the file-handle and read the file again (e.g. `DETACH` in the file is a database file).\nYou can disable checking etags via `SET unsafe_disable_etag_checks = true;`", hfh.etag, response.GetHeaderValue("ETag"));
+				}
+                            }
+
+
+
 			    if (response.HasHeader("Content-Length")) {
 				    auto content_length = stoll(response.GetHeaderValue("Content-Length"));
 				    if ((idx_t)content_length != buffer_out_len) {
@@ -275,7 +287,8 @@ void TimestampToTimeT(timestamp_t timestamp, time_t &result) {
 HTTPFileHandle::HTTPFileHandle(FileSystem &fs, const OpenFileInfo &file, FileOpenFlags flags,
                                unique_ptr<HTTPParams> params_p)
     : FileHandle(fs, file.path, flags), params(std::move(params_p)), http_params(params->Cast<HTTPFSParams>()),
-      flags(flags), length(0), force_full_download(false), buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0) {
+      flags(flags), length(0), force_full_download(false), buffer_available(0), buffer_idx(0), file_offset(0),
+      buffer_start(0), buffer_end(0) {
 	// check if the handle has extended properties that can be set directly in the handle
 	// if we have these properties we don't need to do a head request to obtain them later
 	if (file.extended_info) {

extension/httpfs/httpfs_client_wasm.cpp

@@ -9,7 +9,7 @@ unique_ptr<HTTPClient> HTTPFSUtil::InitializeClient(HTTPParams &http_params, con
 
 unordered_map<string, string> HTTPFSUtil::ParseGetParameters(const string &text) {
 	unordered_map<string, string> result;
-	//TODO: HTTPFSUtil::ParseGetParameters is currently not implemented
+	// TODO: HTTPFSUtil::ParseGetParameters is currently not implemented
 	return result;
 }
 

extension/httpfs/httpfs_extension.cpp

@@ -13,23 +13,23 @@
 namespace duckdb {
 
 static void SetHttpfsClientImplementation(DBConfig &config, const string &value) {
-		if (config.http_util && config.http_util->GetName() == "WasmHTTPUtils") {
-			if (value == "wasm" || value == "default") {
-				// Already handled, do not override
-				return;
-			}
-			throw InvalidInputException("Unsupported option for httpfs_client_implementation, only `wasm` and "
-			                            "`default` are currently supported for duckdb-wasm");
-		}
-		if (value == "httplib" || value == "default") {
-			if (!config.http_util || config.http_util->GetName() != "HTTPFSUtil") {
-				config.http_util = make_shared_ptr<HTTPFSUtil>();
-			}
+	if (config.http_util && config.http_util->GetName() == "WasmHTTPUtils") {
+		if (value == "wasm" || value == "default") {
+			// Already handled, do not override
 			return;
 		}
-		throw InvalidInputException("Unsupported option for httpfs_client_implementation, only `curl`, `httplib` and "
-		                            "`default` are currently supported");
+		throw InvalidInputException("Unsupported option for httpfs_client_implementation, only `wasm` and "
+		                            "`default` are currently supported for duckdb-wasm");
+	}
+	if (value == "httplib" || value == "default") {
+		if (!config.http_util || config.http_util->GetName() != "HTTPFSUtil") {
+			config.http_util = make_shared_ptr<HTTPFSUtil>();
+		}
+		return;
 	}
+	throw InvalidInputException("Unsupported option for httpfs_client_implementation, only `curl`, `httplib` and "
+	                            "`default` are currently supported");
+}
 
 static void LoadInternal(DatabaseInstance &instance) {
 	auto &fs = instance.GetFileSystem();
@@ -79,6 +79,8 @@ static void LoadInternal(DatabaseInstance &instance) {
 	                          LogicalType::UBIGINT, Value(10000));
 	config.AddExtensionOption("s3_uploader_thread_limit", "S3 Uploader global thread limit", LogicalType::UBIGINT,
 	                          Value(50));
+	config.AddExtensionOption("unsafe_disable_etag_checks", "Disable checks on ETag consistency",
+	                          LogicalType::BOOLEAN, Value(false));
 
 	// HuggingFace options
 	config.AddExtensionOption("hf_max_per_page", "Debug option to limit number of items returned in list requests",

extension/httpfs/include/httpfs_client.hpp

@@ -21,6 +21,7 @@ struct HTTPFSParams : public HTTPParams {
 	idx_t hf_max_per_page = DEFAULT_HF_MAX_PER_PAGE;
 	string ca_cert_file;
 	string bearer_token;
+	bool unsafe_disable_etag_checks {false};
 	shared_ptr<HTTPState> state;
 };
 

extension/httpfs/include/s3fs.hpp

@@ -30,8 +30,8 @@ struct S3AuthParams {
 	string url_style;
 	bool use_ssl = true;
 	bool s3_url_compatibility_mode = false;
-    bool requester_pays = false;
-	string oauth2_bearer_token;  // OAuth2 bearer token for GCS
+	bool requester_pays = false;
+	string oauth2_bearer_token; // OAuth2 bearer token for GCS
 
 	static S3AuthParams ReadFrom(optional_ptr<FileOpener> opener, FileOpenerInfo &info);
 };
@@ -47,7 +47,6 @@ struct AWSEnvironmentCredentialsProvider {
 	static constexpr const char *DUCKDB_KMS_KEY_ID_ENV_VAR = "DUCKDB_S3_KMS_KEY_ID";
 	static constexpr const char *DUCKDB_REQUESTER_PAYS_ENV_VAR = "DUCKDB_S3_REQUESTER_PAYS";
 
-
 	explicit AWSEnvironmentCredentialsProvider(DBConfig &config) : config(config) {};
 
 	DBConfig &config;