Merge pull request duckdb#93 from danklynn/gcs-oauth2-bearer-token

Add OAuth2 bearer token support for GCS

Author: samansmink

extension/httpfs/create_secret_functions.cpp

@@ -114,6 +114,10 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
 											lower_name, named_param.second.type().ToString());
 			}
 			secret->secret_map["requester_pays"] = Value::BOOLEAN(named_param.second.GetValue<bool>());
+		} else if (lower_name == "bearer_token" && input.type == "gcs") {
+			secret->secret_map["bearer_token"] = named_param.second.ToString();
+			// Mark it as sensitive
+			secret->redact_keys.insert("bearer_token");
 		} else {
 			throw InvalidInputException("Unknown named parameter passed to CreateSecretFunctionInternal: " +
 			                            lower_name);
@@ -210,6 +214,10 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function,
 	if (type == "r2") {
 		function.named_parameters["account_id"] = LogicalType::VARCHAR;
 	}
+	
+	if (type == "gcs") {
+		function.named_parameters["bearer_token"] = LogicalType::VARCHAR;
+	}
 }
 
 void CreateS3SecretFunctions::RegisterCreateSecretFunction(DatabaseInstance &instance, string type) {

extension/httpfs/include/s3fs.hpp

@@ -31,6 +31,7 @@ struct S3AuthParams {
 	bool use_ssl = true;
 	bool s3_url_compatibility_mode = false;
     bool requester_pays = false;
+	string oauth2_bearer_token;  // OAuth2 bearer token for GCS
 
 	static S3AuthParams ReadFrom(optional_ptr<FileOpener> opener, FileOpenerInfo &info);
 };
@@ -228,6 +229,7 @@ class S3FileSystem : public HTTPFileSystem {
 
 	static string GetS3BadRequestError(S3AuthParams &s3_auth_params);
 	static string GetS3AuthError(S3AuthParams &s3_auth_params);
+	static string GetGCSAuthError(S3AuthParams &s3_auth_params);
 	static HTTPException GetS3Error(S3AuthParams &s3_auth_params, const HTTPResponse &response, const string &url);
 
 protected:

extension/httpfs/s3fs.cpp

@@ -131,6 +131,11 @@ string S3FileSystem::UrlEncode(const string &input, bool encode_slash) {
 	return StringUtil::URLEncode(input, encode_slash);
 }
 
+static bool IsGCSRequest(const string &url) {
+	return StringUtil::StartsWith(url, "gcs://") || 
+	       StringUtil::StartsWith(url, "gs://");
+}
+
 void AWSEnvironmentCredentialsProvider::SetExtensionOptionValue(string key, const char *env_var_name) {
 	char *evar;
 
@@ -209,6 +214,8 @@ S3AuthParams S3AuthParams::ReadFrom(optional_ptr<FileOpener> opener, FileOpenerI
 		if (result.url_style.empty() || url_style_result.GetScope() != SettingScope::SECRET) {
 			result.url_style = "path";
 		}
+		// Read bearer token for GCS
+		secret_reader.TryGetSecretKey("bearer_token", result.oauth2_bearer_token);
 	}
 
 	if (!result.region.empty() && (result.endpoint.empty() || result.endpoint == "s3.amazonaws.com")) {
@@ -235,9 +242,13 @@ unique_ptr<KeyValueSecret> CreateSecret(vector<string> &prefix_paths_p, string &
 	return_value->secret_map["kms_key_id"] = params.kms_key_id;
 	return_value->secret_map["s3_url_compatibility_mode"] = params.s3_url_compatibility_mode;
 	return_value->secret_map["requester_pays"] = params.requester_pays;
+	return_value->secret_map["bearer_token"] = params.oauth2_bearer_token;
 
 	//! Set redact keys
 	return_value->redact_keys = {"secret", "session_token"};
+	if (!params.oauth2_bearer_token.empty()) {
+		return_value->redact_keys.insert("bearer_token");
+	}
 
 	return return_value;
 }
@@ -672,9 +683,19 @@ unique_ptr<HTTPResponse> S3FileSystem::PostRequest(FileHandle &handle, string ur
 	auto auth_params = handle.Cast<S3FileHandle>().auth_params;
 	auto parsed_s3_url = S3UrlParse(url, auth_params);
 	string http_url = parsed_s3_url.GetHTTPUrl(auth_params, http_params);
-	auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len);
-	auto headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "",
-	                                "", payload_hash, "application/octet-stream");
+	
+	HTTPHeaders headers;
+	if (IsGCSRequest(url) && !auth_params.oauth2_bearer_token.empty()) {
+		// Use bearer token for GCS
+		headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
+		headers["Host"] = parsed_s3_url.host;
+		headers["Content-Type"] = "application/octet-stream";
+	} else {
+		// Use existing S3 authentication
+		auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len);
+		headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "",
+		                          "", payload_hash, "application/octet-stream");
+	}
 
 	return HTTPFileSystem::PostRequest(handle, http_url, headers, result, buffer_in, buffer_in_len);
 }
@@ -685,28 +706,58 @@ unique_ptr<HTTPResponse> S3FileSystem::PutRequest(FileHandle &handle, string url
 	auto parsed_s3_url = S3UrlParse(url, auth_params);
 	string http_url = parsed_s3_url.GetHTTPUrl(auth_params, http_params);
 	auto content_type = "application/octet-stream";
-	auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len);
-
-	auto headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "",
-	                                "", payload_hash, content_type);
+	
+	HTTPHeaders headers;
+	if (IsGCSRequest(url) && !auth_params.oauth2_bearer_token.empty()) {
+		// Use bearer token for GCS
+		headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
+		headers["Host"] = parsed_s3_url.host;
+		headers["Content-Type"] = content_type;
+	} else {
+		// Use existing S3 authentication
+		auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len);
+		headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "",
+		                          "", payload_hash, content_type);
+	}
+	
 	return HTTPFileSystem::PutRequest(handle, http_url, headers, buffer_in, buffer_in_len);
 }
 
 unique_ptr<HTTPResponse> S3FileSystem::HeadRequest(FileHandle &handle, string s3_url, HTTPHeaders header_map) {
 	auto auth_params = handle.Cast<S3FileHandle>().auth_params;
 	auto parsed_s3_url = S3UrlParse(s3_url, auth_params);
 	string http_url = parsed_s3_url.GetHTTPUrl(auth_params);
-	auto headers =
-	    create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "HEAD", auth_params, "", "", "", "");
+	
+	HTTPHeaders headers;
+	if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) {
+		// Use bearer token for GCS
+		headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
+		headers["Host"] = parsed_s3_url.host;
+	} else {
+		// Use existing S3 authentication
+		headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, 
+		                          "s3", "HEAD", auth_params, "", "", "", "");
+	}
+	
 	return HTTPFileSystem::HeadRequest(handle, http_url, headers);
 }
 
 unique_ptr<HTTPResponse> S3FileSystem::GetRequest(FileHandle &handle, string s3_url, HTTPHeaders header_map) {
 	auto auth_params = handle.Cast<S3FileHandle>().auth_params;
 	auto parsed_s3_url = S3UrlParse(s3_url, auth_params);
 	string http_url = parsed_s3_url.GetHTTPUrl(auth_params);
-	auto headers =
-	    create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", "");
+	
+	HTTPHeaders headers;
+	if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) {
+		// Use bearer token for GCS
+		headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
+		headers["Host"] = parsed_s3_url.host;
+	} else {
+		// Use existing S3 authentication
+		headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, 
+		                          "s3", "GET", auth_params, "", "", "", "");
+	}
+	
 	return HTTPFileSystem::GetRequest(handle, http_url, headers);
 }
 
@@ -715,17 +766,37 @@ unique_ptr<HTTPResponse> S3FileSystem::GetRangeRequest(FileHandle &handle, strin
 	auto auth_params = handle.Cast<S3FileHandle>().auth_params;
 	auto parsed_s3_url = S3UrlParse(s3_url, auth_params);
 	string http_url = parsed_s3_url.GetHTTPUrl(auth_params);
-	auto headers =
-	    create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", "");
+	
+	HTTPHeaders headers;
+	if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) {
+		// Use bearer token for GCS
+		headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
+		headers["Host"] = parsed_s3_url.host;
+	} else {
+		// Use existing S3 authentication
+		headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, 
+		                          "s3", "GET", auth_params, "", "", "", "");
+	}
+	
 	return HTTPFileSystem::GetRangeRequest(handle, http_url, headers, file_offset, buffer_out, buffer_out_len);
 }
 
 unique_ptr<HTTPResponse> S3FileSystem::DeleteRequest(FileHandle &handle, string s3_url, HTTPHeaders header_map) {
 	auto auth_params = handle.Cast<S3FileHandle>().auth_params;
 	auto parsed_s3_url = S3UrlParse(s3_url, auth_params);
 	string http_url = parsed_s3_url.GetHTTPUrl(auth_params);
-	auto headers =
-	    create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "DELETE", auth_params, "", "", "", "");
+	
+	HTTPHeaders headers;
+	if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) {
+		// Use bearer token for GCS
+		headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token;
+		headers["Host"] = parsed_s3_url.host;
+	} else {
+		// Use existing S3 authentication
+		headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, 
+		                          "s3", "DELETE", auth_params, "", "", "", "");
+	}
+	
 	return HTTPFileSystem::DeleteRequest(handle, http_url, headers);
 }
 
@@ -774,7 +845,12 @@ void S3FileHandle::Initialize(optional_ptr<FileOpener> opener) {
 				}
 				if (entry->second == "403") {
 					// 403: FORBIDDEN
-					auto extra_text = S3FileSystem::GetS3AuthError(auth_params);
+					string extra_text;
+					if (IsGCSRequest(path)) {
+						extra_text = S3FileSystem::GetGCSAuthError(auth_params);
+					} else {
+						extra_text = S3FileSystem::GetS3AuthError(auth_params);
+					}
 					throw Exception(error.Type(), error.RawMessage() + extra_text, extra_info);
 				}
 			}
@@ -1036,6 +1112,24 @@ string S3FileSystem::GetS3AuthError(S3AuthParams &s3_auth_params) {
 	return extra_text;
 }
 
+string S3FileSystem::GetGCSAuthError(S3AuthParams &s3_auth_params) {
+	string extra_text = "\n\nAuthentication Failure - GCS authentication failed.";
+	if (s3_auth_params.oauth2_bearer_token.empty() && 
+	    s3_auth_params.secret_access_key.empty() && 
+	    s3_auth_params.access_key_id.empty()) {
+		extra_text += "\n* No credentials provided.";
+		extra_text += "\n* For OAuth2: CREATE SECRET (TYPE GCS, bearer_token 'your-token')";
+		extra_text += "\n* For HMAC: CREATE SECRET (TYPE GCS, key_id 'key', secret 'secret')";
+	} else if (!s3_auth_params.oauth2_bearer_token.empty()) {
+		extra_text += "\n* Bearer token was provided but authentication failed.";
+		extra_text += "\n* Ensure your OAuth2 token is valid and not expired.";
+	} else {
+		extra_text += "\n* HMAC credentials were provided but authentication failed.";
+		extra_text += "\n* Ensure your HMAC key_id and secret are correct.";
+	}
+	return extra_text;
+}
+
 HTTPException S3FileSystem::GetS3Error(S3AuthParams &s3_auth_params, const HTTPResponse &response, const string &url) {
 	string extra_text;
 	if (response.status == HTTPStatusCode::BadRequest_400) {
@@ -1051,6 +1145,15 @@ HTTPException S3FileSystem::GetS3Error(S3AuthParams &s3_auth_params, const HTTPR
 
 HTTPException S3FileSystem::GetHTTPError(FileHandle &handle, const HTTPResponse &response, const string &url) {
 	auto &s3_handle = handle.Cast<S3FileHandle>();
+	
+	// Use GCS-specific error for GCS URLs
+	if (IsGCSRequest(url) && response.status == HTTPStatusCode::Forbidden_403) {
+		string extra_text = GetGCSAuthError(s3_handle.auth_params);
+		auto status_message = HTTPFSUtil::GetStatusMessage(response.status);
+		throw HTTPException(response, "HTTP error on '%s' (HTTP %d %s)%s", url,
+		                    response.status, status_message, extra_text);
+	}
+	
 	return GetS3Error(s3_handle.auth_params, response, url);
 }
 string AWSListObjectV2::Request(string &path, HTTPParams &http_params, S3AuthParams &s3_auth_params,

test/sql/secret/gcs_oauth.test

@@ -0,0 +1,89 @@
+# name: test/sql/secret/gcs_oauth.test
+# description: Test GCS OAuth2 bearer token support
+# group: [secret]
+
+require httpfs
+
+statement ok
+PRAGMA enable_verification
+
+# Test creating a GCS secret with OAuth2 bearer token
+statement ok
+CREATE SECRET gcs_oauth_test (
+    TYPE GCS,
+    bearer_token 'test_oauth2_token_12345'
+);
+
+# Verify the secret was created
+query I
+SELECT COUNT(*) FROM duckdb_secrets() WHERE name = 'gcs_oauth_test' AND type = 'gcs';
+----
+1
+
+# Verify bearer token is redacted
+query I
+SELECT COUNT(*) FROM duckdb_secrets() WHERE name = 'gcs_oauth_test' AND secret_string LIKE '%bearer_token=redacted%';
+----
+1
+
+# Test creating a GCS secret with HMAC keys (backward compatibility)
+statement ok
+CREATE SECRET gcs_hmac_test (
+    TYPE GCS,
+    key_id 'test_key_id',
+    secret 'test_secret'
+);
+
+# Verify both secrets exist
+query II
+SELECT name, type FROM duckdb_secrets() WHERE name IN ('gcs_oauth_test', 'gcs_hmac_test') ORDER BY name;
+----
+gcs_hmac_test	gcs
+gcs_oauth_test	gcs
+
+# Test creating a GCS secret with both bearer token and HMAC (bearer token should take precedence)
+statement ok
+CREATE SECRET gcs_mixed_test (
+    TYPE GCS,
+    bearer_token 'oauth_token',
+    key_id 'hmac_key',
+    secret 'hmac_secret'
+);
+
+# Verify all three secrets exist
+query I
+SELECT COUNT(*) FROM duckdb_secrets() WHERE name LIKE 'gcs_%test';
+----
+3
+
+# Clean up
+statement ok
+DROP SECRET gcs_oauth_test;
+
+statement ok
+DROP SECRET gcs_hmac_test;
+
+statement ok
+DROP SECRET gcs_mixed_test;
+
+# Verify all secrets are removed
+query I
+SELECT COUNT(*) FROM duckdb_secrets() WHERE name LIKE 'gcs_%test';
+----
+0
+
+# Test that bearer_token parameter is not allowed for S3 secrets
+statement error Unknown named parameter
+CREATE SECRET s3_with_bearer (
+    TYPE S3,
+    bearer_token 'should_not_work'
+);
+----
+
+# Test that bearer_token parameter is not allowed for R2 secrets  
+statement error Unknown named parameter
+CREATE SECRET r2_with_bearer (
+    TYPE R2,
+    bearer_token 'should_not_work'
+);
+----