feat crypto: use curl blob interface for client certs

Author: nUl1

.mapping.json

@@ -3288,13 +3288,15 @@
   "universal/src/crypto/base64.cpp":"taxi/uservices/userver/universal/src/crypto/base64.cpp",
   "universal/src/crypto/base64_test.cpp":"taxi/uservices/userver/universal/src/crypto/base64_test.cpp",
   "universal/src/crypto/certificate.cpp":"taxi/uservices/userver/universal/src/crypto/certificate.cpp",
+  "universal/src/crypto/certificate_test.cpp":"taxi/uservices/userver/universal/src/crypto/certificate_test.cpp",
   "universal/src/crypto/hash.cpp":"taxi/uservices/userver/universal/src/crypto/hash.cpp",
   "universal/src/crypto/hash_test.cpp":"taxi/uservices/userver/universal/src/crypto/hash_test.cpp",
   "universal/src/crypto/helpers.cpp":"taxi/uservices/userver/universal/src/crypto/helpers.cpp",
   "universal/src/crypto/helpers.hpp":"taxi/uservices/userver/universal/src/crypto/helpers.hpp",
   "universal/src/crypto/openssl.cpp":"taxi/uservices/userver/universal/src/crypto/openssl.cpp",
   "universal/src/crypto/openssl.hpp":"taxi/uservices/userver/universal/src/crypto/openssl.hpp",
   "universal/src/crypto/private_key.cpp":"taxi/uservices/userver/universal/src/crypto/private_key.cpp",
+  "universal/src/crypto/private_key_test.cpp":"taxi/uservices/userver/universal/src/crypto/private_key_test.cpp",
   "universal/src/crypto/public_key.cpp":"taxi/uservices/userver/universal/src/crypto/public_key.cpp",
   "universal/src/crypto/public_key_test.cpp":"taxi/uservices/userver/universal/src/crypto/public_key_test.cpp",
   "universal/src/crypto/random.cpp":"taxi/uservices/userver/universal/src/crypto/random.cpp",

core/src/clients/http/client_crl_test.cpp

@@ -17,10 +17,6 @@
 #include <userver/utest/utest.hpp>
 
 // Tests in this file make sure that certs do not check CRLs by default.
-//
-// Fails on MacOS with Segmentation fault while calling
-// Request::RequestImpl::on_certificate_request, probably because CURL library
-// was misconfigured and uses wrong version of OpenSSL.
 
 USERVER_NAMESPACE_BEGIN
 
@@ -369,7 +365,7 @@ auto InterceptCrlDistribution() {
 
 }  // namespace
 
-UTEST(HttpClient, DISABLED_IN_MAC_OS_TEST_NAME(HttpsWithNoCrl)) {
+UTEST(HttpClient, HttpsWithNoCrl) {
   (void)kCrlFile;
   auto task = InterceptCrlDistribution();
 
@@ -396,7 +392,7 @@ UTEST(HttpClient, DISABLED_IN_MAC_OS_TEST_NAME(HttpsWithNoCrl)) {
   EXPECT_EQ(resp->body_view(), "OK");
 }
 
-UTEST(HttpClient, DISABLED_IN_MAC_OS_TEST_NAME(HttpsWithCrl)) {
+UTEST(HttpClient, HttpsWithCrl) {
   auto tmp_file = fs::blocking::TempFile::Create();
   fs::blocking::RewriteFileContents(tmp_file.GetPath(), kCrlFile);
 
@@ -424,7 +420,7 @@ UTEST(HttpClient, DISABLED_IN_MAC_OS_TEST_NAME(HttpsWithCrl)) {
   UEXPECT_THROW(response_future.Get(), clients::http::SSLException);
 }
 
-UTEST(HttpClient, DISABLED_IN_MAC_OS_TEST_NAME(HttpsWithCrlNoVerify)) {
+UTEST(HttpClient, HttpsWithCrlNoVerify) {
   auto tmp_file = fs::blocking::TempFile::Create();
   fs::blocking::RewriteFileContents(tmp_file.GetPath(), kCrlFile);
 
@@ -452,7 +448,7 @@ UTEST(HttpClient, DISABLED_IN_MAC_OS_TEST_NAME(HttpsWithCrlNoVerify)) {
   UEXPECT_NO_THROW(response_future.Get());
 }
 
-UTEST(HttpClient, DISABLED_IN_MAC_OS_TEST_NAME(HttpsWithNoServerCa)) {
+UTEST(HttpClient, HttpsWithNoServerCa) {
   (void)kCrlFile;
   auto task = InterceptCrlDistribution();
 
@@ -479,7 +475,7 @@ UTEST(HttpClient, DISABLED_IN_MAC_OS_TEST_NAME(HttpsWithNoServerCa)) {
   EXPECT_EQ(resp->body_view(), "OK");
 }
 
-UTEST(HttpClient, DISABLED_IN_MAC_OS_TEST_NAME(HttpsWithNoClientCa)) {
+UTEST(HttpClient, HttpsWithNoClientCa) {
   auto task = InterceptCrlDistribution();
   auto pkey = crypto::PrivateKey::LoadFromString(kRevokedClientPrivateKey, "");
   auto cert = crypto::Certificate::LoadFromString(kClientCertificate);

core/src/clients/http/plugins/yandex_tracing/plugin.hpp

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <userver/clients/http/plugin.hpp>
-#include <userver/dynamic_config/fwd.hpp>
 
 USERVER_NAMESPACE_BEGIN
 

core/src/clients/http/request_state.cpp

@@ -5,6 +5,7 @@
 #include <map>
 #include <string_view>
 
+#include <cryptopp/osrng.h>
 #include <fmt/chrono.h>
 #include <fmt/format.h>
 #include <openssl/ssl.h>
@@ -31,6 +32,11 @@ USERVER_NAMESPACE_BEGIN
 namespace clients::http {
 
 namespace {
+namespace nolint {
+// NOLINT(bugprone-forward-declaration-namespace) due to CryptoPP::Source
+using UnusedConfigSourceFwd = dynamic_config::Source&;
+}  // namespace nolint
+
 /// Default timeout
 constexpr auto kDefaultTimeout = std::chrono::milliseconds{100};
 /// Maximum number of redirects
@@ -156,6 +162,24 @@ class MaybeOwnedUrl final {
   const curl::url* url_ptr_{nullptr};
 };
 
+// Instance-wide password used to avoid passing unencrypted keys
+[[maybe_unused]] const std::string& GetPkeyPassword() {
+  static const std::string password = [] {
+    CryptoPP::DefaultAutoSeededRNG prng;
+    std::string random_bytes;
+    random_bytes.resize(32);
+    // password should not contain '\0' (cURL only accepts C-style strings)
+    while (random_bytes.find('\0') != std::string::npos) {
+      static_assert(sizeof(std::string::value_type) == 1,
+                    "string does not consist of bytes");
+      prng.GenerateBlock(reinterpret_cast<unsigned char*>(random_bytes.data()),
+                         random_bytes.size());
+    }
+    return random_bytes;
+  }();
+  return password;
+}
+
 }  // namespace
 
 RequestState::RequestState(
@@ -208,9 +232,17 @@ void RequestState::ca_info(const std::string& file_path) {
 }
 
 void RequestState::ca(crypto::Certificate cert) {
-  ca_ = std::move(cert);
-  easy().set_ssl_ctx_function(&RequestState::on_certificate_request);
-  easy().set_ssl_ctx_data(this);
+  UINVARIANT(cert, "No certificate");
+  if constexpr (curl::easy::is_set_ca_info_blob_available) {
+    auto cert_pem = cert.GetPemString();
+    UINVARIANT(cert_pem, "Could not serialize certificate");
+    easy().set_ca_info_blob_copy(*cert_pem);
+  } else {
+    // Legacy non-portable way, broken since 7.87.0
+    ca_ = std::move(cert);
+    easy().set_ssl_ctx_function(&RequestState::on_certificate_request);
+    easy().set_ssl_ctx_data(this);
+  }
 }
 
 void RequestState::crl_file(const std::string& file_path) {
@@ -222,40 +254,54 @@ void RequestState::client_key_cert(crypto::PrivateKey pkey,
   UINVARIANT(pkey, "No private key");
   UINVARIANT(cert, "No certificate");
 
-  pkey_ = std::move(pkey);
-  cert_ = std::move(cert);
-
-  // FIXME: until cURL 7.71 there is no sane way to pass TLS keys from memory.
-  // Because of this, we provide our own callback. As a consequence, cURL has
-  // no knowledge of the key used and may reuse this connection for a request
-  // with a different key or without one.
-  // To avoid this until we can upgrade we set the EGD socket option to
-  // an unusable certificate-specific value. This option should have no effect
-  // on systems targeted by userver anyway but it is accounted when checking
-  // cached connection eligibility which is exactly what we need.
-
-  // must be larger than sizeof(sockaddr_un::sun_path)
-  static constexpr size_t kCertIdLength = 255;
-
-  // backwards incompatibility
+  if constexpr (curl::easy::is_set_ssl_cert_blob_available &&
+                curl::easy::is_set_ssl_key_blob_available) {
+    auto cert_pem = cert.GetPemString();
+    UINVARIANT(cert_pem, "Could not serialize certificate");
+    easy().set_ssl_cert_blob_copy(*cert_pem);
+    easy().set_ssl_cert_type("PEM");
+    auto key_pem = pkey.GetPemString(GetPkeyPassword());
+    UINVARIANT(key_pem, "Could not serialize private key");
+    easy().set_ssl_key_blob_copy(*key_pem);
+    easy().set_ssl_key_passwd(GetPkeyPassword());
+    easy().set_ssl_key_type("PEM");
+  } else {
+    // Legacy non-portable way, broken since 7.84.0
+    pkey_ = std::move(pkey);
+    cert_ = std::move(cert);
+
+    // FIXME: until cURL 7.71 there is no sane way to pass TLS keys from memory.
+    // Because of this, we provide our own callback. As a consequence, cURL has
+    // no knowledge of the key used and may reuse this connection for a request
+    // with a different key or without one.
+    // To avoid this until we can upgrade we set the EGD socket option to
+    // an unusable certificate-specific value. This option should have no effect
+    // on systems targeted by userver anyway but it is accounted when checking
+    // cached connection eligibility which is exactly what we need.
+
+    // must be larger than sizeof(sockaddr_un::sun_path)
+    static constexpr size_t kCertIdLength = 255;
+
+    // backwards incompatibility
 #if OPENSSL_VERSION_NUMBER >= 0x010100000L
-  const
+    const
 #endif
-      ASN1_BIT_STRING* cert_sig = nullptr;
-  X509_get0_signature(&cert_sig, nullptr, cert_.GetNative());
-  UINVARIANT(cert_sig, "Cannot get X509 certificate signature");
-
-  std::string cert_id;
-  cert_id.reserve(kCertIdLength);
-  utils::encoding::ToHex(
-      std::string_view{reinterpret_cast<const char*>(cert_sig->data),
-                       std::min<size_t>(cert_sig->length, kCertIdLength / 2)},
-      cert_id);
-  cert_id.resize(kCertIdLength, '=');
-  easy().set_egd_socket(cert_id);
-
-  easy().set_ssl_ctx_function(&RequestState::on_certificate_request);
-  easy().set_ssl_ctx_data(this);
+        ASN1_BIT_STRING* cert_sig = nullptr;
+    X509_get0_signature(&cert_sig, nullptr, cert_.GetNative());
+    UINVARIANT(cert_sig, "Cannot get X509 certificate signature");
+
+    std::string cert_id;
+    cert_id.reserve(kCertIdLength);
+    utils::encoding::ToHex(
+        std::string_view{reinterpret_cast<const char*>(cert_sig->data),
+                         std::min<size_t>(cert_sig->length, kCertIdLength / 2)},
+        cert_id);
+    cert_id.resize(kCertIdLength, '=');
+    easy().set_egd_socket(cert_id);
+
+    easy().set_ssl_ctx_function(&RequestState::on_certificate_request);
+    easy().set_ssl_ctx_data(this);
+  }
 }
 
 void RequestState::http_version(curl::easy::http_version_t version) {

core/src/curl-ev/easy.hpp

@@ -91,6 +91,50 @@ class ThreadControl;
         native::curl_easy_setopt(handle_, OPTION_NAME, str.c_str())));     \
   }
 
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define IMPLEMENT_CURL_OPTION_BLOB(FUNCTION_NAME, OPTION_NAME)                 \
+ private:                                                                      \
+  inline void FUNCTION_NAME##_impl(std::string_view sv, unsigned int flags,    \
+                                   std::error_code& ec) {                      \
+    native::curl_blob blob{};                                                  \
+    /* NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) */                \
+    blob.data = const_cast<void*>(static_cast<const void*>(sv.data()));        \
+    blob.len = sv.size();                                                      \
+    blob.flags = flags;                                                        \
+    ec = std::error_code(static_cast<errc::EasyErrorCode>(                     \
+        native::curl_easy_setopt(handle_, OPTION_NAME, &blob)));               \
+  }                                                                            \
+                                                                               \
+ public:                                                                       \
+  static constexpr bool is_##FUNCTION_NAME##_available = true;                 \
+  inline void FUNCTION_NAME##_copy(std::string_view sv) {                      \
+    std::error_code ec;                                                        \
+    FUNCTION_NAME##_copy(sv, ec);                                              \
+    throw_error(ec, PP_STRINGIZE(FUNCTION_NAME##_copy));                       \
+  }                                                                            \
+  inline void FUNCTION_NAME##_no_copy(std::string_view sv) {                   \
+    std::error_code ec;                                                        \
+    FUNCTION_NAME##_no_copy(sv, ec);                                           \
+    throw_error(ec, PP_STRINGIZE(FUNCTION_NAME##_no_copy));                    \
+  }                                                                            \
+  inline void FUNCTION_NAME##_copy(std::string_view sv, std::error_code& ec) { \
+    FUNCTION_NAME##_impl(sv, CURL_BLOB_COPY, ec);                              \
+  }                                                                            \
+  inline void FUNCTION_NAME##_no_copy(std::string_view sv,                     \
+                                      std::error_code& ec) {                   \
+    FUNCTION_NAME##_impl(sv, CURL_BLOB_NOCOPY, ec);                            \
+  }
+
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define DELETE_CURL_OPTION_BLOB(FUNCTION_NAME)                              \
+  static constexpr bool is_##FUNCTION_NAME##_available = false;             \
+  inline void FUNCTION_NAME##_copy(std::string_view) = delete;              \
+  inline void FUNCTION_NAME##_no_copy(std::string_view) = delete;           \
+  inline void FUNCTION_NAME##_copy(std::string_view, std::error_code&) =    \
+      delete;                                                               \
+  inline void FUNCTION_NAME##_no_copy(std::string_view, std::error_code&) = \
+      delete;
+
 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
 #define IMPLEMENT_CURL_OPTION_GET_STRING_VIEW(FUNCTION_NAME, OPTION_NAME) \
   inline std::string_view FUNCTION_NAME() {                               \
@@ -558,6 +602,13 @@ class easy final : public std::enable_shared_from_this<easy> {
   IMPLEMENT_CURL_OPTION_STRING(set_ssl_engine, native::CURLOPT_SSLENGINE);
   IMPLEMENT_CURL_OPTION_STRING(set_ssl_engine_default,
                                native::CURLOPT_SSLENGINE_DEFAULT);
+#if LIBCURL_VERSION_NUM >= 0x074700
+  IMPLEMENT_CURL_OPTION_BLOB(set_ssl_cert_blob, native::CURLOPT_SSLCERT_BLOB);
+  IMPLEMENT_CURL_OPTION_BLOB(set_ssl_key_blob, native::CURLOPT_SSLKEY_BLOB);
+#else
+  DELETE_CURL_OPTION_BLOB(set_ssl_cert_blob);
+  DELETE_CURL_OPTION_BLOB(set_ssl_key_blob);
+#endif
   enum ssl_version_t {
     ssl_version_default = native::CURL_SSLVERSION_DEFAULT,
     ssl_version_tls_v1 = native::CURL_SSLVERSION_TLSv1,
@@ -569,6 +620,11 @@ class easy final : public std::enable_shared_from_this<easy> {
   IMPLEMENT_CURL_OPTION_BOOLEAN(set_ssl_verify_peer,
                                 native::CURLOPT_SSL_VERIFYPEER);
   IMPLEMENT_CURL_OPTION_STRING(set_ca_info, native::CURLOPT_CAINFO);
+#if LIBCURL_VERSION_NUM >= 0x074D00
+  IMPLEMENT_CURL_OPTION_BLOB(set_ca_info_blob, native::CURLOPT_CAINFO_BLOB);
+#else
+  DELETE_CURL_OPTION_BLOB(set_ca_info_blob);
+#endif
   IMPLEMENT_CURL_OPTION_STRING(set_issuer_cert, native::CURLOPT_ISSUERCERT);
   IMPLEMENT_CURL_OPTION_STRING(set_ca_file, native::CURLOPT_CAPATH);
   IMPLEMENT_CURL_OPTION_STRING(set_crl_file, native::CURLOPT_CRLFILE);

universal/include/userver/crypto/certificate.hpp

@@ -4,6 +4,8 @@
 /// @brief @copybrief crypto::Certificate
 
 #include <memory>
+#include <optional>
+#include <string>
 #include <string_view>
 
 #include <userver/crypto/basic_types.hpp>
@@ -24,6 +26,11 @@ class Certificate {
   NativeType* GetNative() const noexcept { return cert_.get(); }
   explicit operator bool() const noexcept { return !!cert_; }
 
+  /// Returns a PEM-encoded representation of stored certificate.
+  ///
+  /// @throw crypto::SerializationError if serialization fails.
+  std::optional<std::string> GetPemString() const;
+
   /// Accepts a string that contains a certificate, checks that
   /// it's correct, loads it into OpenSSL structures and returns as a
   /// Certificate variable.

universal/include/userver/crypto/exception.hpp

@@ -35,6 +35,12 @@ class KeyParseError : public CryptoException {
   using CryptoException::CryptoException;
 };
 
+/// Serialization error
+class SerializationError : public CryptoException {
+ public:
+  using CryptoException::CryptoException;
+};
+
 }  // namespace crypto
 
 USERVER_NAMESPACE_END

universal/include/userver/crypto/private_key.hpp

@@ -4,6 +4,8 @@
 /// @brief @copybrief crypto::PrivateKey
 
 #include <memory>
+#include <optional>
+#include <string>
 #include <string_view>
 
 #include <userver/crypto/basic_types.hpp>
@@ -24,6 +26,19 @@ class PrivateKey {
   NativeType* GetNative() const noexcept { return pkey_.get(); }
   explicit operator bool() const noexcept { return !!pkey_; }
 
+  /// Returns a PEM-encoded representation of stored private key encrypted by
+  /// the provided password.
+  ///
+  /// @throw crypto::SerializationError if the password is empty or
+  /// serialization fails.
+  std::optional<std::string> GetPemString(std::string_view password) const;
+
+  /// Returns a PEM-encoded representation of stored private key in an
+  /// unencrypted form.
+  ///
+  /// @throw crypto::SerializationError if serialization fails.
+  std::optional<std::string> GetPemStringUnencrypted() const;
+
   /// Accepts a string that contains a private key and a password, checks the
   /// key and password, loads it into OpenSSL structures and returns as a
   /// PrivateKey variable.