Merge pull request #67 from Tmonster/allow_direct_complete_download

Allow complete downloads to happen without first making a GET request

Author: samansmink

extension/httpfs/httpfs.cpp

@@ -279,7 +279,7 @@ void TimestampToTimeT(timestamp_t timestamp, time_t &result) {
 HTTPFileHandle::HTTPFileHandle(FileSystem &fs, const OpenFileInfo &file, FileOpenFlags flags,
                                unique_ptr<HTTPParams> params_p)
     : FileHandle(fs, file.path, flags), params(std::move(params_p)), http_params(params->Cast<HTTPFSParams>()),
-      flags(flags), length(0), buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0) {
+      flags(flags), length(0), force_full_download(false), buffer_available(0), buffer_idx(0), file_offset(0), buffer_start(0), buffer_end(0) {
 	// check if the handle has extended properties that can be set directly in the handle
 	// if we have these properties we don't need to do a head request to obtain them later
 	if (file.extended_info) {
@@ -296,6 +296,10 @@ HTTPFileHandle::HTTPFileHandle(FileSystem &fs, const OpenFileInfo &file, FileOpe
 		if (fs_entry != info.end()) {
 			length = fs_entry->second.GetValue<uint64_t>();
 		}
+		auto force_full_download_entry = info.find("force_full_download");
+		if (force_full_download_entry != info.end()) {
+			force_full_download = force_full_download_entry->second.GetValue<bool>();
+		}
 		if (lm_entry != info.end() && etag_entry != info.end() && fs_entry != info.end()) {
 			// we found all relevant entries (last_modified, etag and file size)
 			// skip head request
@@ -595,8 +599,8 @@ optional_idx TryParseContentLength(const HTTPHeaders &headers) {
 }
 
 void HTTPFileHandle::LoadFileInfo() {
-	if (initialized) {
-		// already initialized
+	if (initialized || force_full_download) {
+		// already initialized or we specifically do not want to perform a head request and just run a direct download
 		return;
 	}
 	auto &hfs = file_system.Cast<HTTPFileSystem>();
@@ -685,7 +689,7 @@ void HTTPFileHandle::Initialize(optional_ptr<FileOpener> opener) {
 	LoadFileInfo();
 
 	if (flags.OpenForReading()) {
-		if (http_params.state && length == 0) {
+		if ((http_params.state && length == 0) || force_full_download) {
 			FullDownload(hfs, should_write_cache);
 		}
 		if (should_write_cache) {

extension/httpfs/include/httpfs.hpp

@@ -48,6 +48,7 @@ class HTTPFileHandle : public FileHandle {
 	idx_t length;
 	time_t last_modified;
 	string etag;
+	bool force_full_download;
 	bool initialized = false;
 
 	// When using full file download, the full file will be written to a cached file handle