Merge pull request duckdb#114 from carlopi/compare_etag_with_option

When a HTTPFIleHandle has an Etag, and that changes over time, throw an error

Author: samansmink

extension/httpfs/httpfs.cpp

@@ -51,6 +51,7 @@ unique_ptr<HTTPParams> HTTPFSUtil::InitializeParameters(optional_ptr<FileOpener>
 	                                 info);
 	FileOpener::TryGetCurrentSetting(opener, "ca_cert_file", result->ca_cert_file, info);
 	FileOpener::TryGetCurrentSetting(opener, "hf_max_per_page", result->hf_max_per_page, info);
+	FileOpener::TryGetCurrentSetting(opener, "unsafe_disable_etag_checks", result->unsafe_disable_etag_checks, info);
 
 	// HTTP Secret lookups
 	KeyValueSecretReader settings_reader(*opener, info, "http");
@@ -226,6 +227,17 @@ unique_ptr<HTTPResponse> HTTPFileSystem::GetRangeRequest(FileHandle &handle, str
 		    }
 		    if (static_cast<int>(response.status) < 300) { // done redirecting
 			    out_offset = 0;
+
+			    if (!hfh.http_params.unsafe_disable_etag_checks && hfh.etag.empty() && response.HasHeader("ETag")) {
+                                string responseEtag = response.GetHeaderValue("ETag");
+
+                                if (!responseEtag.empty() && responseEtag != hfh.etag) {
+                                    throw HTTPException(response, "ETag was initially %s and now it returned %s, this likely means the remote file has changed.\nTry to restart the read or close the file-handle and read the file again (e.g. `DETACH` in the file is a database file).\nYou can disable checking etags via `SET unsafe_disable_etag_checks = true;`", hfh.etag, response.GetHeaderValue("ETag"));
+				}
+                            }
+
+
+
 			    if (response.HasHeader("Content-Length")) {
 				    auto content_length = stoll(response.GetHeaderValue("Content-Length"));
 				    if ((idx_t)content_length != buffer_out_len) {

extension/httpfs/httpfs_extension.cpp

@@ -79,6 +79,8 @@ static void LoadInternal(DatabaseInstance &instance) {
 	                          LogicalType::UBIGINT, Value(10000));
 	config.AddExtensionOption("s3_uploader_thread_limit", "S3 Uploader global thread limit", LogicalType::UBIGINT,
 	                          Value(50));
+	config.AddExtensionOption("unsafe_disable_etag_checks", "Disable checks on ETag consistency",
+	                          LogicalType::BOOLEAN, Value(false));
 
 	// HuggingFace options
 	config.AddExtensionOption("hf_max_per_page", "Debug option to limit number of items returned in list requests",

extension/httpfs/include/httpfs_client.hpp

@@ -21,6 +21,7 @@ struct HTTPFSParams : public HTTPParams {
 	idx_t hf_max_per_page = DEFAULT_HF_MAX_PER_PAGE;
 	string ca_cert_file;
 	string bearer_token;
+	bool unsafe_disable_etag_checks {false};
 	shared_ptr<HTTPState> state;
 };