Skip to content

Commit 19079b5

Browse files
utayutay
committed
Add support for SSE-C encryption on S3
1 parent 6b683b1 commit 19079b5

File tree

6 files changed

+53
-2
lines changed

6 files changed

+53
-2
lines changed

‎duckdb‎

Submodule duckdb updated 97 files

‎extension-ci-tools‎

‎src/create_secret_functions.cpp‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
8282
secret->secret_map["use_ssl"] = Value::BOOLEAN(named_param.second.GetValue<bool>());
8383
} else if (lower_name == "kms_key_id") {
8484
secret->secret_map["kms_key_id"] = named_param.second.ToString();
85+
} else if (lower_name == "sse_c_key") {
86+
secret->secret_map["sse_c_key"] = named_param.second.ToString();
87+
} else if (lower_name == "sse_c_key_md5") {
88+
secret->secret_map["sse_c_key_md5"] = named_param.second.ToString();
8589
} else if (lower_name == "url_compatibility_mode") {
8690
if (named_param.second.type() != LogicalType::BOOLEAN) {
8791
throw InvalidInputException("Invalid type past to secret option: '%s', found '%s', expected: 'BOOLEAN'",
@@ -124,6 +128,16 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
124128
}
125129
}
126130

131+
bool has_sse_c_key = secret->secret_map.find("sse_c_key") != secret->secret_map.end();
132+
bool has_sse_c_key_md5 = secret->secret_map.find("sse_c_key_md5") != secret->secret_map.end();
133+
if (has_sse_c_key != has_sse_c_key_md5) {
134+
throw InvalidInputException("Both `sse_c_key` and `sse_c_key_md5` must be set together, or neither should be set");
135+
}
136+
bool has_kms_key_id = secret->secret_map.find("kms_key_id") != secret->secret_map.end();
137+
if (has_kms_key_id && has_sse_c_key) {
138+
throw InvalidInputException("Cannot set `kms_key_id` and `sse_c_key` at the same time");
139+
}
140+
127141
return std::move(secret);
128142
}
129143

@@ -194,6 +208,8 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function,
194208
function.named_parameters["url_style"] = LogicalType::VARCHAR;
195209
function.named_parameters["use_ssl"] = LogicalType::BOOLEAN;
196210
function.named_parameters["kms_key_id"] = LogicalType::VARCHAR;
211+
function.named_parameters["sse_c_key"] = LogicalType::VARCHAR;
212+
function.named_parameters["sse_c_key_md5"] = LogicalType::VARCHAR;
197213
function.named_parameters["url_compatibility_mode"] = LogicalType::BOOLEAN;
198214
function.named_parameters["requester_pays"] = LogicalType::BOOLEAN;
199215

‎src/httpfs_extension.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@ static void LoadInternal(ExtensionLoader &loader) {
7878
config.AddExtensionOption("s3_url_style", "S3 URL style", LogicalType::VARCHAR, Value("vhost"));
7979
config.AddExtensionOption("s3_use_ssl", "S3 use SSL", LogicalType::BOOLEAN, Value(true));
8080
config.AddExtensionOption("s3_kms_key_id", "S3 KMS Key ID", LogicalType::VARCHAR);
81+
config.AddExtensionOption("s3_sse_c_key", "S3 SSE-C Key", LogicalType::VARCHAR);
82+
config.AddExtensionOption("s3_sse_c_key_md5", "S3 SSE-C Key MD5", LogicalType::VARCHAR);
8183
config.AddExtensionOption("s3_url_compatibility_mode", "Disable Globs and Query Parameters on S3 URLs",
8284
LogicalType::BOOLEAN, Value(false));
8385
config.AddExtensionOption("s3_requester_pays", "S3 use requester pays mode", LogicalType::BOOLEAN, Value(false));

‎src/include/s3fs.hpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ struct S3AuthParams {
2727
string session_token;
2828
string endpoint;
2929
string kms_key_id;
30+
string sse_c_key;
31+
string sse_c_key_md5;
3032
string url_style;
3133
bool use_ssl = true;
3234
bool s3_url_compatibility_mode = false;
@@ -45,6 +47,8 @@ struct AWSEnvironmentCredentialsProvider {
4547
static constexpr const char *DUCKDB_ENDPOINT_ENV_VAR = "DUCKDB_S3_ENDPOINT";
4648
static constexpr const char *DUCKDB_USE_SSL_ENV_VAR = "DUCKDB_S3_USE_SSL";
4749
static constexpr const char *DUCKDB_KMS_KEY_ID_ENV_VAR = "DUCKDB_S3_KMS_KEY_ID";
50+
static constexpr const char *DUCKDB_SSE_C_KEY_ENV_VAR = "DUCKDB_S3_SSE_C_KEY";
51+
static constexpr const char *DUCKDB_SSE_C_KEY_MD5_ENV_VAR = "DUCKDB_S3_SSE_C_KEY_MD5";
4852
static constexpr const char *DUCKDB_REQUESTER_PAYS_ENV_VAR = "DUCKDB_S3_REQUESTER_PAYS";
4953

5054
explicit AWSEnvironmentCredentialsProvider(DBConfig &config) : config(config) {};

‎src/s3fs.cpp‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
5454
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/specifying-kms-encryption.html#sse-request-headers-kms
5555
bool use_sse_kms = auth_params.kms_key_id.length() > 0 && (method == "POST" || method == "PUT") &&
5656
query.find("uploadId") == std::string::npos;
57+
bool use_sse_c = auth_params.sse_c_key.length() > 0 && auth_params.sse_c_key_md5.length() > 0;
5758

5859
res["x-amz-date"] = datetime_now;
5960
res["x-amz-content-sha256"] = payload_hash;
@@ -64,6 +65,11 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
6465
res["x-amz-server-side-encryption"] = "aws:kms";
6566
res["x-amz-server-side-encryption-aws-kms-key-id"] = auth_params.kms_key_id;
6667
}
68+
if (use_sse_c) {
69+
res["x-amz-server-side-encryption-customer-algorithm"] = "AES256";
70+
res["x-amz-server-side-encryption-customer-key"] = auth_params.sse_c_key;
71+
res["x-amz-server-side-encryption-customer-key-md5"] = auth_params.sse_c_key_md5;
72+
}
6773

6874
bool use_requester_pays = auth_params.requester_pays;
6975
if (use_requester_pays) {
@@ -89,6 +95,13 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
8995
if (use_sse_kms) {
9096
signed_headers += ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id";
9197
}
98+
if (use_sse_c) {
99+
signed_headers += ";x-amz-server-side-encryption-customer-algorithm;x-amz-server-side-encryption-customer-key;"
100+
"x-amz-server-side-encryption-customer-key-md5";
101+
}
102+
if (use_requester_pays) {
103+
signed_headers += ";x-amz-request-payer";
104+
}
92105
auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query;
93106
if (content_type.length() > 0) {
94107
canonical_request += "\ncontent-type:" + content_type;
@@ -104,6 +117,14 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
104117
canonical_request += "\nx-amz-server-side-encryption:aws:kms";
105118
canonical_request += "\nx-amz-server-side-encryption-aws-kms-key-id:" + auth_params.kms_key_id;
106119
}
120+
if (use_sse_c) {
121+
canonical_request += "\nx-amz-server-side-encryption-customer-algorithm:AES256";
122+
canonical_request += "\nx-amz-server-side-encryption-customer-key:" + auth_params.sse_c_key;
123+
canonical_request += "\nx-amz-server-side-encryption-customer-key-md5:" + auth_params.sse_c_key_md5;
124+
}
125+
if (use_requester_pays) {
126+
canonical_request += "\nx-amz-request-payer:requester";
127+
}
107128

108129
canonical_request += "\n\n" + signed_headers + "\n" + payload_hash;
109130
sha256(canonical_request.c_str(), canonical_request.length(), canonical_request_hash);
@@ -164,6 +185,8 @@ void AWSEnvironmentCredentialsProvider::SetAll() {
164185
this->SetExtensionOptionValue("s3_endpoint", DUCKDB_ENDPOINT_ENV_VAR);
165186
this->SetExtensionOptionValue("s3_use_ssl", DUCKDB_USE_SSL_ENV_VAR);
166187
this->SetExtensionOptionValue("s3_kms_key_id", DUCKDB_KMS_KEY_ID_ENV_VAR);
188+
this->SetExtensionOptionValue("s3_sse_c_key", DUCKDB_SSE_C_KEY_ENV_VAR);
189+
this->SetExtensionOptionValue("s3_sse_c_key_md5", DUCKDB_SSE_C_KEY_MD5_ENV_VAR);
167190
this->SetExtensionOptionValue("s3_requester_pays", DUCKDB_REQUESTER_PAYS_ENV_VAR);
168191
}
169192

@@ -177,6 +200,8 @@ S3AuthParams AWSEnvironmentCredentialsProvider::CreateParams() {
177200
params.session_token = SESSION_TOKEN_ENV_VAR;
178201
params.endpoint = DUCKDB_ENDPOINT_ENV_VAR;
179202
params.kms_key_id = DUCKDB_KMS_KEY_ID_ENV_VAR;
203+
params.sse_c_key = DUCKDB_SSE_C_KEY_ENV_VAR;
204+
params.sse_c_key_md5 = DUCKDB_SSE_C_KEY_MD5_ENV_VAR;
180205
params.use_ssl = DUCKDB_USE_SSL_ENV_VAR;
181206
params.requester_pays = DUCKDB_REQUESTER_PAYS_ENV_VAR;
182207

@@ -202,6 +227,8 @@ S3AuthParams S3AuthParams::ReadFrom(optional_ptr<FileOpener> opener, FileOpenerI
202227
secret_reader.TryGetSecretKeyOrSetting("region", "s3_region", result.region);
203228
secret_reader.TryGetSecretKeyOrSetting("use_ssl", "s3_use_ssl", result.use_ssl);
204229
secret_reader.TryGetSecretKeyOrSetting("kms_key_id", "s3_kms_key_id", result.kms_key_id);
230+
secret_reader.TryGetSecretKeyOrSetting("sse_c_key", "s3_sse_c_key", result.sse_c_key);
231+
secret_reader.TryGetSecretKeyOrSetting("sse_c_key_md5", "s3_sse_c_key_md5", result.sse_c_key_md5);
205232
secret_reader.TryGetSecretKeyOrSetting("s3_url_compatibility_mode", "s3_url_compatibility_mode",
206233
result.s3_url_compatibility_mode);
207234
secret_reader.TryGetSecretKeyOrSetting("requester_pays", "s3_requester_pays", result.requester_pays);
@@ -244,6 +271,8 @@ unique_ptr<KeyValueSecret> CreateSecret(vector<string> &prefix_paths_p, string &
244271
return_value->secret_map["url_style"] = params.url_style;
245272
return_value->secret_map["use_ssl"] = params.use_ssl;
246273
return_value->secret_map["kms_key_id"] = params.kms_key_id;
274+
return_value->secret_map["sse_c_key"] = params.sse_c_key;
275+
return_value->secret_map["sse_c_key_md5"] = params.sse_c_key_md5;
247276
return_value->secret_map["s3_url_compatibility_mode"] = params.s3_url_compatibility_mode;
248277
return_value->secret_map["requester_pays"] = params.requester_pays;
249278
return_value->secret_map["bearer_token"] = params.oauth2_bearer_token;

0 commit comments

Comments
 (0)