impl: add experimental ca store in memory (#15068)

Author: scotthart

google/cloud/credentials.h

@@ -19,6 +19,7 @@
 #include "google/cloud/options.h"
 #include "google/cloud/ssl_certificate.h"
 #include "google/cloud/version.h"
+#include "absl/strings/string_view.h"
 #include <chrono>
 #include <memory>
 #include <string>
@@ -45,6 +46,25 @@ namespace experimental {
 struct ClientSslCertificateOption {
   using Type = SslCertificate;
 };
+
+/**
+ * Represents one or more certificates to be added to the CA store in lieu of
+ * using any CA certificates stored on the filesystem.
+ *
+ * @note This option is currently experimental and only works with OpenSSL and
+ *     services using JSON/HTTP transport.
+ *
+ * @note Specifying this option disables reading any certificates that may exist
+ *     on the filesystem.
+ *
+ * @note Requires libcurl v7.10.6 or later.
+ *
+ * @note Not supported on Windows.
+ */
+struct CAInMemoryOption {
+  using Type = std::vector<absl::string_view>;
+};
+
 }  // namespace experimental
 
 /**
@@ -447,11 +467,10 @@ struct CARootsFilePathOption {
 };
 
 /// A list of  options related to authentication.
-using UnifiedCredentialsOptionList =
-    OptionList<AccessTokenLifetimeOption, CARootsFilePathOption,
-               DelegatesOption, ScopesOption, LoggingComponentsOption,
-               UnifiedCredentialsOption,
-               experimental::ClientSslCertificateOption>;
+using UnifiedCredentialsOptionList = OptionList<
+    AccessTokenLifetimeOption, CARootsFilePathOption, DelegatesOption,
+    ScopesOption, LoggingComponentsOption, UnifiedCredentialsOption,
+    experimental::ClientSslCertificateOption, experimental::CAInMemoryOption>;
 
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace cloud

google/cloud/internal/curl_handle_factory.cc

@@ -15,13 +15,93 @@
 #include "google/cloud/internal/curl_handle_factory.h"
 #include "google/cloud/credentials.h"
 #include "google/cloud/internal/curl_options.h"
+#include "google/cloud/internal/make_status.h"
+#include "google/cloud/log.h"
+#include <openssl/err.h>
+#include <openssl/ssl.h>
 #include <algorithm>
 #include <iterator>
 
 namespace google {
 namespace cloud {
 namespace rest_internal {
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
+namespace {
+
+#ifndef _WIN32
+struct BIOPtrCleanup {
+  int operator()(BIO* b) const { return BIO_free(b); }
+};
+
+using BioPtr = std::unique_ptr<BIO, BIOPtrCleanup>;
+
+struct X509InfoPtrCleanup {
+  void operator()(STACK_OF(X509_INFO) * i) const {
+    return sk_X509_INFO_pop_free(i, X509_INFO_free);
+  }
+};
+
+using X509InfoPtr = std::unique_ptr<STACK_OF(X509_INFO), X509InfoPtrCleanup>;
+
+#endif
+
+Status SetCurlCAInMemory(CurlHandleFactory const& factory, SSL_CTX* ssl_ctx) {
+#if _WIN32
+  return internal::InternalError(
+      "SSL callback function currently not supported in windows",
+      GCP_ERROR_INFO());
+#else
+  X509_STORE* cert_store = SSL_CTX_get_cert_store(ssl_ctx);
+  if (!cert_store) {
+    return internal::InternalError("SSL_CTX_get_cert_store returned NULL",
+                                   GCP_ERROR_INFO());
+  }
+
+  // Add each of the provided certs to the store.
+  for (auto const& cert : factory.ca_certs()) {
+    BioPtr buf{BIO_new_mem_buf(cert.data(), static_cast<int>(cert.length()))};
+    if (!buf) {
+      return internal::InternalError("BIO_new_mem_buf returned NULL",
+                                     GCP_ERROR_INFO());
+    }
+    X509InfoPtr info{
+        PEM_X509_INFO_read_bio(buf.get(), nullptr, nullptr, nullptr)};
+    if (!info) {
+      return internal::InternalError("PEM_X509_INFO_read_bio returned NULL",
+                                     GCP_ERROR_INFO());
+    }
+
+    for (decltype(sk_X509_INFO_num(info.get())) i = 0;
+         i < sk_X509_INFO_num(info.get()); ++i) {
+      X509_INFO* value = sk_X509_INFO_value(info.get(), i);
+      if (value->x509) {
+        X509_STORE_add_cert(cert_store, value->x509);
+      }
+      if (value->crl) {
+        X509_STORE_add_crl(cert_store, value->crl);
+      }
+    }
+  }
+
+  return {};
+#endif
+}
+
+}  // namespace
+
+extern "C" {
+
+static CURLcode SslCtxFunction(  // NOLINT(misc-use-anonymous-namespace)
+    CURL*, void* ssl_ctx, void* userdata) {
+  auto* handle_factory = reinterpret_cast<CurlHandleFactory*>(userdata);
+  auto result = SetCurlCAInMemory(*handle_factory, (SSL_CTX*)ssl_ctx);
+  if (!result.ok()) {
+    GCP_LOG(ERROR) << result << "\n";
+    return CURLE_ABORTED_BY_CALLBACK;
+  }
+  return CURLE_OK;
+}
+}
 
 void CurlHandleFactory::SetCurlStringOption(CURL* handle, CURLoption option_tag,
                                             char const* value) {
@@ -36,15 +116,27 @@ std::shared_ptr<CurlHandleFactory> GetDefaultCurlHandleFactory() {
 
 std::shared_ptr<CurlHandleFactory> GetDefaultCurlHandleFactory(
     Options const& options) {
-  if (!options.get<CARootsFilePathOption>().empty()) {
+  if (!options.get<CARootsFilePathOption>().empty() ||
+      !options.get<experimental::CAInMemoryOption>().empty()) {
     return std::make_shared<DefaultCurlHandleFactory>(options);
   }
+
   return GetDefaultCurlHandleFactory();
 }
 
 DefaultCurlHandleFactory::DefaultCurlHandleFactory(Options const& o) {
-  if (o.has<CARootsFilePathOption>()) cainfo_ = o.get<CARootsFilePathOption>();
-  if (o.has<CAPathOption>()) capath_ = o.get<CAPathOption>();
+  if (o.has<experimental::CAInMemoryOption>()) {
+    ca_certs_ = o.get<experimental::CAInMemoryOption>();
+    if (ca_certs_.empty()) {
+      GCP_LOG(FATAL) << internal::InvalidArgumentError(
+          "No CA certificates specified", GCP_ERROR_INFO());
+    }
+  } else {
+    if (o.has<CARootsFilePathOption>()) {
+      cainfo_ = o.get<CARootsFilePathOption>();
+    }
+    if (o.has<CAPathOption>()) capath_ = o.get<CAPathOption>();
+  }
 }
 
 CurlPtr DefaultCurlHandleFactory::CreateHandle() {
@@ -74,17 +166,36 @@ void DefaultCurlHandleFactory::CleanupMultiHandle(CurlMulti m,
 }
 
 void DefaultCurlHandleFactory::SetCurlOptions(CURL* handle) {
-  if (cainfo_) {
-    SetCurlStringOption(handle, CURLOPT_CAINFO, cainfo_->c_str());
-  }
-  if (capath_) {
-    SetCurlStringOption(handle, CURLOPT_CAPATH, capath_->c_str());
+  if (!ca_certs_.empty()) {
+    SetCurlStringOption(handle, CURLOPT_CAINFO, nullptr);
+    SetCurlStringOption(handle, CURLOPT_CAPATH, nullptr);
+    auto result = curl_easy_setopt(handle, CURLOPT_SSL_CTX_DATA, this);
+    if (result != CURLE_OK) {
+      GCP_LOG(FATAL) << internal::InternalError(curl_easy_strerror(result),
+                                                GCP_ERROR_INFO());
+    }
+    result =
+        curl_easy_setopt(handle, CURLOPT_SSL_CTX_FUNCTION, &SslCtxFunction);
+    if (result != CURLE_OK) {
+      GCP_LOG(FATAL) << internal::InternalError(curl_easy_strerror(result),
+                                                GCP_ERROR_INFO());
+    }
+  } else {
+    if (cainfo_) {
+      SetCurlStringOption(handle, CURLOPT_CAINFO, cainfo_->c_str());
+    }
+    if (capath_) {
+      SetCurlStringOption(handle, CURLOPT_CAPATH, capath_->c_str());
+    }
   }
 }
 
 PooledCurlHandleFactory::PooledCurlHandleFactory(std::size_t maximum_size,
                                                  Options const& o)
-    : maximum_size_(maximum_size), cainfo_(CAInfo(o)), capath_(CAPath(o)) {}
+    : maximum_size_(maximum_size),
+      cainfo_(CAInfo(o)),
+      capath_(CAPath(o)),
+      ca_certs_(CACerts(o)) {}
 
 PooledCurlHandleFactory::~PooledCurlHandleFactory() = default;
 
@@ -192,11 +303,27 @@ void PooledCurlHandleFactory::CleanupMultiHandle(CurlMulti m,
 }
 
 void PooledCurlHandleFactory::SetCurlOptions(CURL* handle) {
-  if (cainfo_) {
-    SetCurlStringOption(handle, CURLOPT_CAINFO, cainfo_->c_str());
-  }
-  if (capath_) {
-    SetCurlStringOption(handle, CURLOPT_CAPATH, capath_->c_str());
+  if (!ca_certs_.empty()) {
+    SetCurlStringOption(handle, CURLOPT_CAINFO, nullptr);
+    SetCurlStringOption(handle, CURLOPT_CAPATH, nullptr);
+    auto result = curl_easy_setopt(handle, CURLOPT_SSL_CTX_DATA, this);
+    if (result != CURLE_OK) {
+      GCP_LOG(FATAL) << internal::InternalError(curl_easy_strerror(result),
+                                                GCP_ERROR_INFO());
+    }
+    result =
+        curl_easy_setopt(handle, CURLOPT_SSL_CTX_FUNCTION, &SslCtxFunction);
+    if (result != CURLE_OK) {
+      GCP_LOG(FATAL) << internal::InternalError(curl_easy_strerror(result),
+                                                GCP_ERROR_INFO());
+    }
+  } else {
+    if (cainfo_) {
+      SetCurlStringOption(handle, CURLOPT_CAINFO, cainfo_->c_str());
+    }
+    if (capath_) {
+      SetCurlStringOption(handle, CURLOPT_CAPATH, capath_->c_str());
+    }
   }
 }
 
@@ -210,6 +337,16 @@ absl::optional<std::string> PooledCurlHandleFactory::CAPath(Options const& o) {
   return o.get<CAPathOption>();
 }
 
+std::vector<absl::string_view> PooledCurlHandleFactory::CACerts(
+    Options const& o) {
+  if (!o.has<experimental::CAInMemoryOption>()) return {};
+  if (o.get<experimental::CAInMemoryOption>().empty()) {
+    GCP_LOG(FATAL) << internal::InvalidArgumentError(
+        "No CA certificates specified", GCP_ERROR_INFO());
+  }
+  return o.get<experimental::CAInMemoryOption>();
+}
+
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace rest_internal
 }  // namespace cloud

google/cloud/internal/curl_handle_factory.h

@@ -18,10 +18,12 @@
 #include "google/cloud/internal/curl_wrappers.h"
 #include "google/cloud/options.h"
 #include "google/cloud/version.h"
+#include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 #include <deque>
 #include <mutex>
 #include <string>
+#include <vector>
 
 namespace google {
 namespace cloud {
@@ -54,6 +56,7 @@ class CurlHandleFactory {
   // class is in `internal::`.
   virtual absl::optional<std::string> cainfo() const = 0;
   virtual absl::optional<std::string> capath() const = 0;
+  virtual std::vector<absl::string_view> const& ca_certs() const = 0;
 
  protected:
   // Only virtual for testing purposes.
@@ -90,6 +93,9 @@ class DefaultCurlHandleFactory : public CurlHandleFactory {
 
   absl::optional<std::string> cainfo() const override { return cainfo_; }
   absl::optional<std::string> capath() const override { return capath_; }
+  std::vector<absl::string_view> const& ca_certs() const override {
+    return ca_certs_;
+  }
 
  private:
   void SetCurlOptions(CURL* handle);
@@ -98,6 +104,7 @@ class DefaultCurlHandleFactory : public CurlHandleFactory {
   std::string last_client_ip_address_;
   absl::optional<std::string> cainfo_;
   absl::optional<std::string> capath_;
+  std::vector<absl::string_view> ca_certs_;
 };
 
 /**
@@ -137,16 +144,21 @@ class PooledCurlHandleFactory : public CurlHandleFactory {
 
   absl::optional<std::string> cainfo() const override { return cainfo_; }
   absl::optional<std::string> capath() const override { return capath_; }
+  std::vector<absl::string_view> const& ca_certs() const override {
+    return ca_certs_;
+  }
 
  private:
   void SetCurlOptions(CURL* handle);
   static absl::optional<std::string> CAInfo(Options const& o);
   static absl::optional<std::string> CAPath(Options const& o);
+  static std::vector<absl::string_view> CACerts(Options const& o);
 
   // These are constant after initialization and thus need no locking.
   std::size_t const maximum_size_;
   absl::optional<std::string> const cainfo_;
   absl::optional<std::string> const capath_;
+  std::vector<absl::string_view> ca_certs_;
 
   mutable std::mutex handles_mu_;
   std::deque<CurlPtr> handles_;