Merge pull request #113 from samansmink/remove-unnecessary-head-request

Avoid sending HEAD request on writes

Author: samansmink

extension/httpfs/httpfs.cpp

@@ -333,6 +333,11 @@ unique_ptr<FileHandle> HTTPFileSystem::OpenFileExtended(const OpenFileInfo &file
 	}
 
 	auto handle = CreateHandle(file, flags, opener);
+
+	if (flags.OpenForWriting() && !flags.OpenForAppending() && !flags.OpenForReading()) {
+		handle->write_overwrite_mode = true;
+	}
+
 	handle->Initialize(opener);
 
 	DUCKDB_LOG_FILE_SYSTEM_OPEN((*handle));
@@ -584,6 +589,14 @@ void HTTPFileHandle::LoadFileInfo() {
 		// already initialized or we specifically do not want to perform a head request and just run a direct download
 		return;
 	}
+
+	// In write_overwrite_mode we dgaf about the size, so no head request is needed
+	if (write_overwrite_mode) {
+		length = 0;
+		initialized = true;
+		return;
+	}
+
 	auto &hfs = file_system.Cast<HTTPFileSystem>();
 	auto res = hfs.HeadRequest(*this, path, {});
 	if (res->status != HTTPStatusCode::OK_200) {

extension/httpfs/include/httpfs.hpp

@@ -51,6 +51,9 @@ class HTTPFileHandle : public FileHandle {
 	bool force_full_download;
 	bool initialized = false;
 
+	// In write overwrite mode, we are not interested in the current state of the file: we're overwriting it.
+	bool write_overwrite_mode = false;
+
 	// When using full file download, the full file will be written to a cached file handle
 	unique_ptr<CachedFileHandle> cached_file_handle;
 

test/sql/copy/no_head_on_write.test

@@ -0,0 +1,66 @@
+# name: test/sql/copy/no_head_on_write.test
+# description: Confirm that we don't send head requests for writes
+# group: [secret]
+
+require-env S3_TEST_SERVER_AVAILABLE 1
+
+require-env AWS_DEFAULT_REGION
+
+require-env AWS_ACCESS_KEY_ID
+
+require-env AWS_SECRET_ACCESS_KEY
+
+require-env DUCKDB_S3_ENDPOINT
+
+require-env DUCKDB_S3_USE_SSL
+
+require httpfs
+
+require parquet
+
+statement ok
+SET enable_logging=true
+
+statement ok
+set s3_use_ssl='${DUCKDB_S3_USE_SSL}'
+
+statement ok
+set s3_endpoint='${DUCKDB_S3_ENDPOINT}'
+
+statement ok
+set s3_region='${AWS_DEFAULT_REGION}'
+
+# Create some test data
+statement ok
+CREATE SECRET s1 (
+    TYPE S3,
+    KEY_ID '${AWS_ACCESS_KEY_ID}',
+    SECRET '${AWS_SECRET_ACCESS_KEY}',
+    REQUESTER_PAYS true
+)
+
+statement ok
+CALL enable_logging('HTTP');
+
+statement ok
+copy (select 1 as a) to 's3://test-bucket/test-file.parquet'
+
+query I
+select request.type FROM duckdb_logs_parsed('HTTP')
+----
+POST
+PUT
+POST
+
+statement ok
+CALL truncate_duckdb_logs();
+
+statement ok
+copy (select 1 as a) to 's3://test-bucket/test-file.csv'
+
+query I
+select request.type FROM duckdb_logs_parsed('HTTP')
+----
+POST
+PUT
+POST