diff --git a/ci/cloudbuild/builds/lib/integration.sh b/ci/cloudbuild/builds/lib/integration.sh index 146222fb31382..96df17b3c7556 100644 --- a/ci/cloudbuild/builds/lib/integration.sh +++ b/ci/cloudbuild/builds/lib/integration.sh @@ -132,6 +132,21 @@ function integration::bazel_args() { "--test_env=GOOGLE_CLOUD_CPP_STORAGE_TEST_KEY_FILE_P12=${KEY_DIR}/${key_base}.p12" ) fi + + # Adds environment variables for SSL testing. + # Info on how to create/refresh these can be found at: + # https://cloud.google.com/certificate-authority-service/docs/create-certificate + gcloud storage cp --quiet "${SECRETS_BUCKET}/client.crt" "${KEY_DIR}/client.crt" >/dev/null 2>&1 || true + gcloud storage cp --quiet "${SECRETS_BUCKET}/client.chain.crt" "${KEY_DIR}/client.chain.crt" >/dev/null 2>&1 || true + gcloud storage cp --quiet "${SECRETS_BUCKET}/client.private.pem" "${KEY_DIR}/client.private.pem" >/dev/null 2>&1 || true + if [[ -r "${KEY_DIR}/client.crt" ]] && [[ -r "${KEY_DIR}/client.chain.crt" ]] && [[ -r "${KEY_DIR}/client.private.pem" ]]; then + args+=( + "--test_env=GOOGLE_CLOUD_CPP_CLIENT_SSL_CERT_FILE=${KEY_DIR}/client.crt" + "--test_env=GOOGLE_CLOUD_CPP_CLIENT_SSL_CERT_CHAIN_FILE=${KEY_DIR}/client.chain.crt" + "--test_env=GOOGLE_CLOUD_CPP_CLIENT_SSL_KEY_FILE=${KEY_DIR}/client.private.pem" + ) + fi + printf "%s\n" "${args[@]}" } diff --git a/google/cloud/credentials.h b/google/cloud/credentials.h index e56283017ee41..263176b7300cb 100644 --- a/google/cloud/credentials.h +++ b/google/cloud/credentials.h @@ -17,6 +17,7 @@ #include "google/cloud/common_options.h" #include "google/cloud/options.h" +#include "google/cloud/ssl_certificate.h" #include "google/cloud/version.h" #include #include @@ -30,6 +31,22 @@ namespace internal { class CredentialsVisitor; } // namespace internal +namespace experimental { +/** + * Represents a Client SSL certificate used in mTLS authentication. + * + * Providing this option enables both PEER and HOST verification. + * + * @note This option is currently experimental and only works with services + * using JSON/HTTP transport. + * + * @note Requires libcurl v7.71.0 or later. + */ +struct ClientSslCertificateOption { + using Type = SslCertificate; +}; +} // namespace experimental + /** * An opaque representation of the authentication configuration. * @@ -433,7 +450,8 @@ struct CARootsFilePathOption { using UnifiedCredentialsOptionList = OptionList; + UnifiedCredentialsOption, + experimental::ClientSslCertificateOption>; GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace cloud diff --git a/google/cloud/google_cloud_cpp_common.bzl b/google/cloud/google_cloud_cpp_common.bzl index a6dc86077eb76..2201aa030ef8d 100644 --- a/google/cloud/google_cloud_cpp_common.bzl +++ b/google/cloud/google_cloud_cpp_common.bzl @@ -107,6 +107,7 @@ google_cloud_cpp_common_hdrs = [ "project.h", "retry_policy.h", "rpc_metadata.h", + "ssl_certificate.h", "status.h", "status_or.h", "stream_range.h", diff --git a/google/cloud/google_cloud_cpp_common.cmake b/google/cloud/google_cloud_cpp_common.cmake index 8ef5f726b9331..773857ef720d1 100644 --- a/google/cloud/google_cloud_cpp_common.cmake +++ b/google/cloud/google_cloud_cpp_common.cmake @@ -167,6 +167,7 @@ add_library( project.h retry_policy.h rpc_metadata.h + ssl_certificate.h status.cc status.h status_or.h diff --git a/google/cloud/internal/curl_impl.cc b/google/cloud/internal/curl_impl.cc index 89d4091bb2829..1c70c65f5c0c7 100644 --- a/google/cloud/internal/curl_impl.cc +++ b/google/cloud/internal/curl_impl.cc @@ -14,6 +14,7 @@ #include "google/cloud/internal/curl_impl.h" #include "google/cloud/common_options.h" +#include "google/cloud/credentials.h" #include "google/cloud/internal/absl_str_cat_quiet.h" #include "google/cloud/internal/absl_str_join_quiet.h" #include "google/cloud/internal/algorithm.h" @@ -26,6 +27,7 @@ #include "google/cloud/rest_options.h" #include "absl/strings/match.h" #include "absl/strings/strip.h" +#include #include #include #include @@ -198,6 +200,10 @@ CurlImpl::CurlImpl(CurlHandle handle, proxy_username_ = CurlOptProxyUsername(options); proxy_password_ = CurlOptProxyPassword(options); + if (options.has()) { + client_ssl_cert_ = options.get(); + } + interface_ = CurlOptInterface(options); } @@ -327,6 +333,44 @@ Status CurlImpl::MakeRequest(HttpMethod method, RestContext& context, if (!status.ok()) return OnTransferError(context, std::move(status)); } + if (client_ssl_cert_.has_value()) { +#if CURL_AT_LEAST_VERSION(7, 71, 0) + status = handle_.SetOption(CURLOPT_SSL_VERIFYPEER, 1L); + if (!status.ok()) return OnTransferError(context, std::move(status)); + status = handle_.SetOption(CURLOPT_SSL_VERIFYHOST, 2L); + if (!status.ok()) return OnTransferError(context, std::move(status)); + + status = handle_.SetOption(CURLOPT_SSLCERTTYPE, + experimental::SslCertificate::ToString( + client_ssl_cert_->ssl_certificate_type()) + .c_str()); + if (!status.ok()) return OnTransferError(context, std::move(status)); + + struct curl_blob ssl_cert_blob; + ssl_cert_blob.data = + const_cast(client_ssl_cert_->ssl_certificate().data()); + ssl_cert_blob.len = client_ssl_cert_->ssl_certificate().length(); + ssl_cert_blob.flags = CURL_BLOB_COPY; + status = handle_.SetOption(CURLOPT_SSLCERT_BLOB, &ssl_cert_blob); + if (!status.ok()) return OnTransferError(context, std::move(status)); + + struct curl_blob ssl_key_blob; + ssl_key_blob.data = + const_cast(client_ssl_cert_->ssl_private_key().data()); + ssl_key_blob.len = client_ssl_cert_->ssl_private_key().length(); + ssl_key_blob.flags = CURL_BLOB_COPY; + status = handle_.SetOption(CURLOPT_SSLKEY_BLOB, &ssl_key_blob); + if (!status.ok()) return OnTransferError(context, std::move(status)); +#else + return OnTransferError( + context, + internal::InvalidArgumentError( + "libcurl 7.71.0 or higher required to use ClientSslCertificate", + GCP_ERROR_INFO().WithMetadata("current_libcurl_version", + LIBCURL_VERSION))); +#endif + } + if (method == HttpMethod::kGet) { status = handle_.SetOption(CURLOPT_NOPROGRESS, 1L); if (!status.ok()) return OnTransferError(context, std::move(status)); diff --git a/google/cloud/internal/curl_impl.h b/google/cloud/internal/curl_impl.h index a27318d9a175b..56e04c252ed71 100644 --- a/google/cloud/internal/curl_impl.h +++ b/google/cloud/internal/curl_impl.h @@ -23,6 +23,7 @@ #include "google/cloud/internal/rest_request.h" #include "google/cloud/internal/rest_response.h" #include "google/cloud/options.h" +#include "google/cloud/ssl_certificate.h" #include "google/cloud/status_or.h" #include "google/cloud/version.h" #include "absl/types/optional.h" @@ -143,6 +144,8 @@ class CurlImpl { absl::optional proxy_username_; absl::optional proxy_password_; + absl::optional client_ssl_cert_ = absl::nullopt; + absl::optional interface_; CurlReceivedHeaders received_headers_; diff --git a/google/cloud/ssl_certificate.h b/google/cloud/ssl_certificate.h new file mode 100644 index 0000000000000..5bf640fcd6679 --- /dev/null +++ b/google/cloud/ssl_certificate.h @@ -0,0 +1,79 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_SSL_CERTIFICATE_H +#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_SSL_CERTIFICATE_H + +#include "google/cloud/version.h" +#include + +namespace google { +namespace cloud { +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN +namespace experimental { + +/** + * Represents an SSL certificate used in TLS authentication. + */ +class SslCertificate { + public: + enum class SslCertificateType { kPEM, kDER, kP12 }; + + /// Creates and empty certificate. + SslCertificate() = default; + + /// Creates a PEM certificate from the values provided. + SslCertificate(std::string ssl_certificate, std::string ssl_private_key) + : ssl_certificate_(std::move(ssl_certificate)), + ssl_private_key_(std::move(ssl_private_key)) {} + + /// Creates a user specified type of certificate from the values provided. + SslCertificate(std::string ssl_certificate, std::string ssl_private_key, + SslCertificateType ssl_certificate_type) + : ssl_certificate_(std::move(ssl_certificate)), + ssl_private_key_(std::move(ssl_private_key)), + ssl_certificate_type_(ssl_certificate_type) {} + + std::string const& ssl_certificate() const { return ssl_certificate_; } + + std::string const& ssl_private_key() const { return ssl_private_key_; } + + SslCertificateType ssl_certificate_type() const { + return ssl_certificate_type_; + } + + static std::string ToString(SslCertificateType type) { + switch (type) { + case SslCertificateType::kPEM: + return "PEM"; + case SslCertificateType::kDER: + return "DER"; + case SslCertificateType::kP12: + return "P12"; + } + return {}; + } + + private: + std::string ssl_certificate_; + std::string ssl_private_key_; + SslCertificateType ssl_certificate_type_ = SslCertificateType::kPEM; +}; + +} // namespace experimental +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END +} // namespace cloud +} // namespace google + +#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_SSL_CERTIFICATE_H diff --git a/google/cloud/storage/tests/CMakeLists.txt b/google/cloud/storage/tests/CMakeLists.txt index 660ff8a264589..80370150ddd23 100644 --- a/google/cloud/storage/tests/CMakeLists.txt +++ b/google/cloud/storage/tests/CMakeLists.txt @@ -72,8 +72,11 @@ set(storage_client_integration_tests tracing_integration_test.cc) set(storage_client_integration_tests_production # cmake-format: sort - alternative_endpoint_integration_test.cc key_file_integration_test.cc - signed_url_integration_test.cc unified_credentials_integration_test.cc) + alternative_endpoint_integration_test.cc + key_file_integration_test.cc + mtls_object_basic_crud_integration_test.cc + signed_url_integration_test.cc + unified_credentials_integration_test.cc) list(APPEND storage_client_integration_tests ${storage_client_integration_tests_production}) list(SORT storage_client_integration_tests) diff --git a/google/cloud/storage/tests/mtls_object_basic_crud_integration_test.cc b/google/cloud/storage/tests/mtls_object_basic_crud_integration_test.cc new file mode 100644 index 0000000000000..f96223fbde7b1 --- /dev/null +++ b/google/cloud/storage/tests/mtls_object_basic_crud_integration_test.cc @@ -0,0 +1,191 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "google/cloud/storage/client.h" +#include "google/cloud/storage/testing/object_integration_test.h" +#include "google/cloud/credentials.h" +#include "google/cloud/internal/getenv.h" +#include "google/cloud/internal/make_status.h" +#include "google/cloud/status_or.h" +#include "google/cloud/testing_util/scoped_environment.h" +#include "google/cloud/testing_util/status_matchers.h" +#include +#include +#include +#include +#include +#include +#include + +namespace google { +namespace cloud { +namespace storage { +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN +namespace { + +using ::google::cloud::internal::GetEnv; +using ::google::cloud::testing_util::ScopedEnvironment; + +using ::testing::Contains; +using ::testing::Not; +using ::testing::UnorderedElementsAreArray; + +struct MtlsObjectBasicCRUDIntegrationTest + : public ::google::cloud::storage::testing::ObjectIntegrationTest { + public: + static StatusOr ReadEnvVarFile(char const* env_var) { + if (!GetEnv(env_var).has_value()) { + return google::cloud::internal::InvalidArgumentError("Missing env var"); + } + auto env = GetEnv(env_var); + std::string key_file = std::move(*env); + std::ifstream is(key_file); + return std::string{std::istreambuf_iterator{is}, {}}; + } + + static Client MakeMtlsClient(std::string const& test_key_file_contents, + std::string const& ssl_cert_blob, + std::string const& ssl_key_blob) { + auto options = Options{} + .set(TestRetryPolicy()) + .set(TestBackoffPolicy()); + + options.set("https://storage.mtls.googleapis.com"); + + experimental::SslCertificate client_ssl_cert{ssl_cert_blob, ssl_key_blob}; + options.set(client_ssl_cert); + + options.set( + MakeServiceAccountCredentials(test_key_file_contents)); + + return Client(std::move(options)); + } +}; + +/// @test Verify the Object CRUD (Create, Get, Update, Delete, List) operations. +TEST_F(MtlsObjectBasicCRUDIntegrationTest, BasicCRUD) { + if (!(GetEnv("GOOGLE_CLOUD_CPP_CLIENT_SSL_CERT_FILE").has_value() && + GetEnv("GOOGLE_CLOUD_CPP_CLIENT_SSL_KEY_FILE").has_value())) { + GTEST_SKIP(); + } + + ScopedEnvironment self_signed_jwt( + "GOOGLE_CLOUD_CPP_EXPERIMENTAL_DISABLE_SELF_SIGNED_JWT", "1"); + auto test_key_file_contents = + ReadEnvVarFile("GOOGLE_CLOUD_CPP_REST_TEST_KEY_FILE_JSON"); + ASSERT_STATUS_OK(test_key_file_contents); + auto client_ssl_cert = + ReadEnvVarFile("GOOGLE_CLOUD_CPP_CLIENT_SSL_CERT_FILE"); + ASSERT_STATUS_OK(client_ssl_cert); + auto client_ssl_key = ReadEnvVarFile("GOOGLE_CLOUD_CPP_CLIENT_SSL_KEY_FILE"); + ASSERT_STATUS_OK(client_ssl_key); + + auto client = MakeMtlsClient(*test_key_file_contents, *client_ssl_cert, + *client_ssl_key); + + auto list_object_names = [&client, this] { + std::vector names; + for (auto o : client.ListObjects(bucket_name_)) { + EXPECT_STATUS_OK(o); + if (!o) break; + names.push_back(o->name()); + } + return names; + }; + + auto object_name = MakeRandomObjectName(); + ASSERT_THAT(list_object_names(), Not(Contains(object_name))) + << "Test aborted. The object <" << object_name << "> already exists." + << "This is unexpected as the test generates a random object name."; + + // Create the object, but only if it does not exist already. + StatusOr insert_meta = + client.InsertObject(bucket_name_, object_name, LoremIpsum(), + IfGenerationMatch(0), Projection("full")); + ASSERT_STATUS_OK(insert_meta); + EXPECT_THAT(list_object_names(), Contains(object_name).Times(1)); + + StatusOr get_meta = client.GetObjectMetadata( + bucket_name_, object_name, Generation(insert_meta->generation()), + Projection("full")); + ASSERT_STATUS_OK(get_meta); + EXPECT_EQ(*get_meta, *insert_meta); + + ObjectMetadata update = *get_meta; + update.mutable_acl().emplace_back( + ObjectAccessControl().set_role("READER").set_entity( + "allAuthenticatedUsers")); + update.set_cache_control("no-cache") + .set_content_disposition("inline") + .set_content_encoding("identity") + .set_content_language("en") + .set_content_type("plain/text"); + update.mutable_metadata().emplace("updated", "true"); + StatusOr updated_meta = client.UpdateObject( + bucket_name_, object_name, update, Projection("full")); + ASSERT_STATUS_OK(updated_meta); + + // Because some ACL field values are not predictable, we convert the values we + // care about to strings and compare those. + { + auto acl_to_string_vector = + [](std::vector const& acl) { + std::vector v; + std::transform(acl.begin(), acl.end(), std::back_inserter(v), + [](ObjectAccessControl const& x) { + return x.entity() + " = " + x.role(); + }); + return v; + }; + auto expected = acl_to_string_vector(update.acl()); + auto actual = acl_to_string_vector(updated_meta->acl()); + EXPECT_THAT(expected, UnorderedElementsAreArray(actual)); + } + EXPECT_EQ(update.cache_control(), updated_meta->cache_control()) + << *updated_meta; + EXPECT_EQ(update.content_disposition(), updated_meta->content_disposition()) + << *updated_meta; + EXPECT_EQ(update.content_encoding(), updated_meta->content_encoding()) + << *updated_meta; + EXPECT_EQ(update.content_language(), updated_meta->content_language()) + << *updated_meta; + EXPECT_EQ(update.content_type(), updated_meta->content_type()) + << *updated_meta; + EXPECT_EQ(update.metadata(), updated_meta->metadata()) << *updated_meta; + + ObjectMetadata desired_patch = *updated_meta; + desired_patch.set_content_language("en"); + desired_patch.mutable_metadata().erase("updated"); + desired_patch.mutable_metadata().emplace("patched", "true"); + StatusOr patched_meta = + client.PatchObject(bucket_name_, object_name, *updated_meta, + desired_patch, PredefinedAcl::Private()); + ASSERT_STATUS_OK(patched_meta); + + EXPECT_EQ(desired_patch.metadata(), patched_meta->metadata()) + << *patched_meta; + EXPECT_EQ(desired_patch.content_language(), patched_meta->content_language()) + << *patched_meta; + + // This is the test for Object CRUD, we cannot rely on `ScheduleForDelete()`. + auto status = client.DeleteObject(bucket_name_, object_name); + ASSERT_STATUS_OK(status); + EXPECT_THAT(list_object_names(), Not(Contains(object_name))); +} + +} // anonymous namespace +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END +} // namespace storage +} // namespace cloud +} // namespace google diff --git a/google/cloud/storage/tests/storage_client_integration_tests.bzl b/google/cloud/storage/tests/storage_client_integration_tests.bzl index 00ee72fb83698..1536bb02c89ef 100644 --- a/google/cloud/storage/tests/storage_client_integration_tests.bzl +++ b/google/cloud/storage/tests/storage_client_integration_tests.bzl @@ -35,6 +35,7 @@ storage_client_integration_tests = [ "grpc_object_metadata_integration_test.cc", "hmac_key_integration_test.cc", "key_file_integration_test.cc", + "mtls_object_basic_crud_integration_test.cc", "notification_integration_test.cc", "object_basic_crud_integration_test.cc", "object_checksum_integration_test.cc", @@ -80,6 +81,7 @@ storage_client_integration_tests = [ storage_client_integration_tests_production = [ "alternative_endpoint_integration_test.cc", "key_file_integration_test.cc", + "mtls_object_basic_crud_integration_test.cc", "signed_url_integration_test.cc", "unified_credentials_integration_test.cc", ]