Skip to content

Fixup sigv4 for bulk deletes AND fixup CURL content type to be overwritable #234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
carlopi merged 1 commit into from
Jan 29, 2026
Merged

Fixup sigv4 for bulk deletes AND fixup CURL content type to be overwritable #234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/httpfs_curl_client.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,8 +373,10 @@ class HTTPFSCurlClient : public HTTPClient {
}

auto curl_headers = TransformHeadersCurl(info.headers, info.params);
const string content_type = "Content-Type: application/octet-stream";
curl_headers.Add(content_type.c_str());
if (!info.headers.HasHeader("Content-Type")) {
const string content_type = "Content-Type: application/octet-stream";
curl_headers.Add(content_type.c_str());
}
// transform parameters
request_info->url = info.url;

Expand Down
2 changes: 1 addition & 1 deletion src/include/s3fs.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,5 +298,5 @@ struct AWSListObjectV2 {

HTTPHeaders CreateS3Header(string url, string query, string host, string service, string method,
const S3AuthParams &auth_params, string date_now = "", string datetime_now = "",
string payload_hash = "", string content_type = "");
string payload_hash = "", string content_type = "", string content_md5 = "");
} // namespace duckdb
20 changes: 13 additions & 7 deletions src/s3fs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ namespace duckdb {

HTTPHeaders CreateS3Header(string url, string query, string host, string service, string method,
const S3AuthParams &auth_params, string date_now, string datetime_now, string payload_hash,
string content_type) {
string content_type, string content_md5) {

HTTPHeaders res;
res["Host"] = host;
Expand Down Expand Up @@ -74,11 +74,15 @@ HTTPHeaders CreateS3Header(string url, string query, string host, string service
string signed_headers = "";
hash_bytes canonical_request_hash;
hash_str canonical_request_hash_str;
if (content_md5.length() > 0) {
signed_headers += "content-md5;";
res["content-md5"] = content_md5;
}
if (content_type.length() > 0) {
signed_headers += "content-type;";
#ifdef EMSCRIPTEN
res["content-type"] = content_type;
#endif
if (content_type != "application/octet-stream") {
res["content-type"] = content_type;
}
}
signed_headers += "host;x-amz-content-sha256;x-amz-date";
if (use_requester_pays) {
Expand All @@ -91,6 +95,9 @@ HTTPHeaders CreateS3Header(string url, string query, string host, string service
signed_headers += ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id";
}
auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query;
if (content_md5.length() > 0) {
canonical_request += "\ncontent-md5:" + content_md5;
}
if (content_type.length() > 0) {
canonical_request += "\ncontent-type:" + content_type;
}
Expand All @@ -107,6 +114,7 @@ HTTPHeaders CreateS3Header(string url, string query, string host, string service
}

canonical_request += "\n\n" + signed_headers + "\n" + payload_hash;

sha256(canonical_request.c_str(), canonical_request.length(), canonical_request_hash);

hex256(canonical_request_hash, canonical_request_hash_str);
Expand Down Expand Up @@ -1104,9 +1112,7 @@ void S3FileSystem::RemoveFiles(const vector<string> &paths, optional_ptr<FileOpe
auto payload_hash = GetPayloadHash(const_cast<char *>(body.data()), body.length());

auto headers = CreateS3Header(url_info.path, http_query_param_for_sig, url_info.host, "s3", "POST",
url_info.auth_params, "", "", payload_hash, "");
headers["Content-MD5"] = content_md5;
headers["Content-Type"] = "application/xml";
url_info.auth_params, "", "", payload_hash, "application/xml", content_md5);

string http_url = url_info.http_proto + url_info.host + S3FileSystem::UrlEncode(url_info.path) + "?" +
http_query_param_for_url;
Expand Down
Loading