Merge pull request #768 from Altinity/backports/24.3/72730

24.3 Backport of ClickHouse#72730 - Fix advanced SSL configuration for Keeper's internal communication

Author: Enmk

contrib/NuRaft

@@ -32,6 +32,7 @@ set(SRCS
     "${LIBRARY_DIR}/src/handle_custom_notification.cxx"
     "${LIBRARY_DIR}/src/handle_vote.cxx"
     "${LIBRARY_DIR}/src/launcher.cxx"
+    "${LIBRARY_DIR}/src/log_entry.cxx"
     "${LIBRARY_DIR}/src/srv_config.cxx"
     "${LIBRARY_DIR}/src/snapshot_sync_req.cxx"
     "${LIBRARY_DIR}/src/snapshot_sync_ctx.cxx"
@@ -50,6 +51,12 @@ else()
     target_compile_definitions(_nuraft PRIVATE USE_BOOST_ASIO=1 BOOST_ASIO_STANDALONE=1)
 endif()
 
+target_link_libraries (_nuraft PRIVATE clickhouse_common_io)
+# We must have it PUBLIC here because some headers which depend on it directly
+# included in clickhouse
+target_compile_definitions(_nuraft PUBLIC USE_CLICKHOUSE_THREADS=1)
+MESSAGE(STATUS "Will use clickhouse threads for NuRaft")
+
 target_include_directories (_nuraft SYSTEM PRIVATE "${LIBRARY_DIR}/include/libnuraft")
 # for some reason include "asio.h" directly without "boost/" prefix.
 target_include_directories (_nuraft SYSTEM PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/boost/boost")

contrib/nuraft-cmake/CMakeLists.txt

@@ -2244,7 +2244,7 @@ uint64_t Changelog::getStartIndex() const
 LogEntryPtr Changelog::getLastEntry() const
 {
     /// This entry treaded in special way by NuRaft
-    static LogEntryPtr fake_entry = nuraft::cs_new<nuraft::log_entry>(0, nuraft::buffer::alloc(sizeof(uint64_t)));
+    static LogEntryPtr fake_entry = nuraft::cs_new<nuraft::log_entry>(0, nuraft::buffer::alloc(0));
 
     auto entry = entry_storage.getEntry(max_log_id);
     if (entry == nullptr)

src/Coordination/Changelog.cpp

@@ -16,7 +16,7 @@ ptr<log_entry> makeClone(const ptr<log_entry> & entry)
 InMemoryLogStore::InMemoryLogStore()
     : start_idx(1)
 {
-    nuraft::ptr<nuraft::buffer> buf = nuraft::buffer::alloc(sizeof(uint64_t));
+    nuraft::ptr<nuraft::buffer> buf = nuraft::buffer::alloc(0);
     logs[0] = nuraft::cs_new<nuraft::log_entry>(0, buf);
 }
 

src/Coordination/InMemoryLogStore.cpp

@@ -56,11 +56,10 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int RAFT_ERROR;
-    extern const int NO_ELEMENTS_IN_CONFIG;
     extern const int SUPPORT_IS_DISABLED;
     extern const int LOGICAL_ERROR;
     extern const int INVALID_CONFIG_PARAMETER;
-    extern const int BAD_ARGUMENTS;
+    extern const int OPENSSL_ERROR;
 }
 
 using namespace std::chrono_literals;
@@ -70,47 +69,38 @@ namespace
 
 #if USE_SSL
 
-int callSetCertificate(SSL * ssl, void * arg)
+auto getSslContextProvider(const Poco::Util::AbstractConfiguration & config, std::string_view key)
 {
-    if (!arg)
-        return -1;
-
-    const CertificateReloader::Data * data = reinterpret_cast<CertificateReloader::Data *>(arg);
-    return setCertificateCallback(ssl, data, getLogger("SSLContext"));
-}
-
-void setSSLParams(nuraft::asio_service::options & asio_opts)
-{
-    const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config();
-    String certificate_file_property = "openSSL.server.certificateFile";
-    String private_key_file_property = "openSSL.server.privateKeyFile";
-    String root_ca_file_property = "openSSL.server.caConfig";
-
-    if (!config.has(certificate_file_property))
-        throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server certificate file is not set.");
-
-    if (!config.has(private_key_file_property))
-        throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server private key file is not set.");
+    String load_default_ca_file_property = fmt::format("openSSL.{}.loadDefaultCAFile", key);
+    String verification_mode_property = fmt::format("openSSL.{}.verificationMode", key);
+    String root_ca_file_property = fmt::format("openSSL.{}.caConfig", key);
+    String private_key_passphrase_property = fmt::format("openSSL.{}.privateKeyPassphraseHandler.options.password", key);
 
     Poco::Net::Context::Params params;
-    params.certificateFile = config.getString(certificate_file_property);
-    if (params.certificateFile.empty())
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server certificate file in config '{}' is empty", certificate_file_property);
+    String certificate_file_property = fmt::format("openSSL.{}.certificateFile", key);
+    String private_key_file_property = fmt::format("openSSL.{}.privateKeyFile", key);
+    if (config.has(certificate_file_property))
+        params.certificateFile = config.getString(certificate_file_property);
 
-    params.privateKeyFile = config.getString(private_key_file_property);
-    if (params.privateKeyFile.empty())
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server key file in config '{}' is empty", private_key_file_property);
+    if (config.has(private_key_file_property))
+        params.privateKeyFile = config.getString(private_key_file_property);
 
-    auto pass_phrase = config.getString("openSSL.server.privateKeyPassphraseHandler.options.password", "");
-    auto certificate_data = std::make_shared<CertificateReloader::Data>(params.certificateFile, params.privateKeyFile, pass_phrase);
+    std::shared_ptr<CertificateReloader::Data> certificate_data;
+    if (config.has(private_key_passphrase_property))
+    {
+        certificate_data = std::make_shared<CertificateReloader::Data>(
+            params.certificateFile, params.privateKeyFile, config.getString(private_key_passphrase_property));
+        params.certificateFile.clear();
+        params.privateKeyFile.clear();
+    }
 
     if (config.has(root_ca_file_property))
         params.caLocation = config.getString(root_ca_file_property);
 
-    params.loadDefaultCAs = config.getBool("openSSL.server.loadDefaultCAFile", false);
-    params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString("openSSL.server.verificationMode", "none"));
+    params.loadDefaultCAs = config.getBool(load_default_ca_file_property, false);
+    params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString(verification_mode_property, "none"));
 
-    std::string disabled_protocols_list = config.getString("openSSL.server.disableProtocols", "");
+    std::string disabled_protocols_list = config.getString(fmt::format("openSSL.{}.disableProtocols", key), "");
     Poco::StringTokenizer dp_tok(disabled_protocols_list, ";,", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
     int disabled_protocols = 0;
     for (const auto & token : dp_tok)
@@ -127,21 +117,54 @@ void setSSLParams(nuraft::asio_service::options & asio_opts)
             disabled_protocols |= Poco::Net::Context::PROTO_TLSV1_2;
     }
 
-    asio_opts.ssl_context_provider_server_ = [params, certificate_data, disabled_protocols]
+    auto prefer_server_cypher = config.getBool(fmt::format("openSSL.{}.preferServerCiphers", key), false);
+    auto cache_sessions = config.getBool(fmt::format("openSSL.{}.cache_sessions", key), false);
+    return [params, disabled_protocols, prefer_server_cypher, cache_sessions, is_server = key == "server", certificate_data]
     {
-        Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_SERVER_USE, params);
+        Poco::Net::Context context(is_server ? Poco::Net::Context::Usage::SERVER_USE : Poco::Net::Context::Usage::CLIENT_USE, params);
         context.disableProtocols(disabled_protocols);
-        SSL_CTX * ssl_ctx = context.takeSslContext();
-        SSL_CTX_set_cert_cb(ssl_ctx, callSetCertificate, reinterpret_cast<void *>(certificate_data.get()));
-        return ssl_ctx;
-    };
 
-    asio_opts.ssl_context_provider_client_ = [ctx_params = std::move(params)]
-    {
-        Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_CLIENT_USE, ctx_params);
+        if (prefer_server_cypher)
+            context.preferServerCiphers();
+
+        if (cache_sessions)
+            context.enableSessionCache();
+
+        auto * ssl_ctx = context.sslContext();
+        if (certificate_data)
+        {
+            if (auto err = SSL_CTX_clear_chain_certs(ssl_ctx); err != 1)
+                throw Exception(ErrorCodes::OPENSSL_ERROR, "Clear certificates {}", Poco::Net::Utility::getLastError());
+
+            if (auto err = SSL_CTX_use_certificate(ssl_ctx, const_cast<X509 *>(certificate_data->certs_chain[0].certificate())); err != 1)
+                throw Exception(ErrorCodes::OPENSSL_ERROR, "Use certificate {}", Poco::Net::Utility::getLastError());
+
+            for (auto cert = certificate_data->certs_chain.begin() + 1; cert != certificate_data->certs_chain.end(); cert++)
+            {
+                if (auto err = SSL_CTX_add1_chain_cert(ssl_ctx, const_cast<X509 *>(cert->certificate())); err != 1)
+                    throw Exception(ErrorCodes::OPENSSL_ERROR, "Add certificate to chain {}", Poco::Net::Utility::getLastError());
+            }
+
+            if (auto err = SSL_CTX_use_PrivateKey(ssl_ctx, const_cast<EVP_PKEY *>(static_cast<const EVP_PKEY *>(certificate_data->key))); err != 1)
+                throw Exception(ErrorCodes::OPENSSL_ERROR, "Use private key {}", Poco::Net::Utility::getLastError());
+
+            if (auto err = SSL_CTX_check_private_key(ssl_ctx); err != 1)
+                throw Exception(ErrorCodes::OPENSSL_ERROR, "Unusable key-pair {}", Poco::Net::Utility::getLastError());
+        }
+
+
         return context.takeSslContext();
     };
 }
+
+void setSSLParams(nuraft::asio_service::options & asio_opts)
+{
+    asio_opts.enable_ssl_ = true;
+
+    const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config();
+    asio_opts.ssl_context_provider_server_ = getSslContextProvider(config, "server");
+    asio_opts.ssl_context_provider_client_ = getSslContextProvider(config, "client");
+}
 #endif
 
 std::string checkAndGetSuperdigest(const String & user_and_digest)
@@ -229,7 +252,7 @@ struct KeeperServer::KeeperRaftServer : public nuraft::raft_server
         }
 
         const size_t voting_members = get_num_voting_members();
-        const auto not_responding_peers = get_not_responding_peers();
+        const auto not_responding_peers = get_not_responding_peers_count();
         const auto quorum_size = voting_members / 2 + 1;
         const auto max_not_responding_peers = voting_members - quorum_size;
 
@@ -266,6 +289,11 @@ struct KeeperServer::KeeperRaftServer : public nuraft::raft_server
         return std::unique_lock(lock_);
     }
 
+    std::unique_lock<std::mutex> lockCommit()
+    {
+        return std::unique_lock(commit_lock_);
+    }
+
     bool isCommitInProgress() const
     {
         return sm_commit_exec_in_progress_;
@@ -441,7 +469,6 @@ void KeeperServer::launchRaftServer(const Poco::Util::AbstractConfiguration & co
         throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSL support for NuRaft is disabled because ClickHouse was built without SSL support.");
 #endif
     }
-
     if (is_recovering)
         enterRecoveryMode(params);
 
@@ -1200,6 +1227,7 @@ Keeper4LWInfo KeeperServer::getPartiallyFilled4LWInfo() const
 
 uint64_t KeeperServer::createSnapshot()
 {
+    auto commit_lock = raft_instance->lockCommit();
     uint64_t log_idx = raft_instance->create_snapshot();
     if (log_idx != 0)
         LOG_INFO(log, "Snapshot creation scheduled with last committed log index {}.", log_idx);

src/Coordination/KeeperServer.cpp

@@ -44,10 +44,19 @@
             <election_timeout_upper_bound_ms>5000</election_timeout_upper_bound_ms>
             <raft_logs_level>information</raft_logs_level>
             <force_sync>false</force_sync>
+            <async_replication>1</async_replication>
             <!-- we want all logs for complex problems investigation -->
             <reserved_log_items>1000000000000000</reserved_log_items>
         </coordination_settings>
 
+        <feature_flags>
+            <filtered_list>1</filtered_list>
+            <multi_read>1</multi_read>
+            <check_not_exists>1</check_not_exists>
+            <create_if_not_exists>1</create_if_not_exists>
+            <remove_recursive>1</remove_recursive>
+        </feature_flags>
+
         <raft_configuration>
             <server>
                 <id>1</id>

tests/config/config.d/database_replicated.xml

@@ -1,20 +1,22 @@
 -----BEGIN CERTIFICATE-----
-MIIDPDCCAiQCFBXNOvsLA+dqmX/TkYG9JXdD5m72MA0GCSqGSIb3DQEBCwUAMFox
-CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
-cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMCmNsaWNraG91c2UwIBcNMjIw
-NDIxMTAzNDU1WhgPMjEyMjAzMjgxMDM0NTVaMFkxCzAJBgNVBAYTAkFVMRMwEQYD
-VQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBM
-dGQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
-AQoCggEBAKaXz596N4NC2zZdIqdwZbSYAtNdBCsBVPt5YT9F640aF5zOogPZyxGP
-ENyOZwABi/7HhwFbH657xyRvi8lTau8dZL+0tbakyoIn1Tw6j+/3GXTjLduJSy6C
-mOf4OzsrFC8mYgU+7p5ijvWVlO9h5NMbLdAPSIB5WSHhmSORH5LgjoK6oMOYdRod
-GmfHqSbwPVwy3Li5SXlniCQmJsM0zl64LFbJ/NU+13qETmhBiDgmh0Svi+wzSzqZ
-q1PIX92T3k44IXNZbvF7lKbUOS9Xb3BoxA4cDqRcTx4x73xRDwodSmqiuQOC99HI
-A0C/tZJ25VNAGjLKukPSHqYscq2PAsUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA
-IDQwjf/ja3TfOXrz+Gn1eErSKnWS3asjRT9rYWQsy3tzVUkMIcszrG+FqTR16g5H
-ZWyuEOi6KIRmda3SYKdLKmtQLrgx6/d/jvH5TQ0LTFZrp6vh0lo3pV+L6fLo1ZRD
-V1i8jW/7HHNyqJamUXOjwA0DpPOMkdtwuyV+rJ+2bTG1ZSK33O4Ae2CY5+dad6zy
-YI6b1c9flWfDznuNEMH7jDDjKgXwjZGeU53FiuuhHiNyRchsr/B9eIBsom8oykiD
-kch4cnAxx2E+m3hLYzupkXHOVQ5CNpVk8PGUCIGcyqDxPt+fOj1WbDQ9laEcfhmV
-kR+vHmzOqWZnHU4QeMqDig==
+MIIDtjCCAp6gAwIBAgIUdOfco+b8/fQZQOafHgghkEYL3YkwDQYJKoZIhvcNAQEL
+BQAwazELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5
+MRAwDgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0
+Y2x1c3RlciBSb290IENBMB4XDTI0MTIwMjE0MjkyNloXDTM0MTEzMDE0MjkyNlow
+ajELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5MRAw
+DgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRswGQYDVQQDDBJjbGlja2hv
+dXNlLWtlZXBlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUXmnn
+Cv7sY9lbrS1Q3c5q7ok9R4XEPq/jWBFkIEnJR7vSjCEnOjLxg+1MdUItjbqODf9N
+5vFbHiiqWQVkGrmg8/CTSme0qyNr7FcmG1hO4bzK/dvIyK1R7YISqZpXoCTVzEnU
+IjU7f+PkX2uAiSypxM4zpNyC7++j6ah8xYNRfR9AS5c7e1dvNKBNMmNipYxVgaEo
+pIke40m12ezIzLOtkL/rGlsnM2Tv/0Wv1xQE+OjHByyQE08vuliatFfweTXLF48m
+4S4NdOq5dh2WX8xLPr8BxRLjXzs08wgKVFpWkIOR2uEInjuVQAGMuZeOqUuQGdar
+GMH4M/3tDl0eJ7mbAgMBAAGjUzBRMB0GA1UdDgQWBBTr9ldBtTB0vatq2yhQgYtt
+zMNhJDAfBgNVHSMEGDAWgBTGSqv6LHbQlKrpPWtYEVoX+/c5cTAPBgNVHRMBAf8E
+BTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBYc03AV8n0D43xm07MxpgqDvNEZC9u
+Q2LnP89UBnBmXD5FwMz4XhA/iupyAeYItZ8R17caIpLHgwOUrh3oHxVW5V144Q0p
+hBBp/im8WQ8NnS3z52CusxE1Zu5AMjoZtxY8FTvgs6vuJZYds/dgtUg5bBawR2LX
+A5FsPLyYpwCjoPTM622uXkuPfRMc5SC5edwHa1RyoG8Poz8B6Y63iKQydOXin9Q1
+9rQ7mqM7D2dCURx4gVoN9y+fLkXgQEzTMBT4wuVJl+CXnxcmKsoROAy7g2mL0RMw
+P3cl+Bod3NrabhjAqG01nHsQzy0uJ/aJHbqoR3OtYo8DdsoKrBRramiG
 -----END CERTIFICATE-----

tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt

@@ -1,30 +1,28 @@
------BEGIN RSA PRIVATE KEY-----
-Proc-Type: 4,ENCRYPTED
-DEK-Info: AES-256-CBC,4E14FF586022476CD22AAFB662BB0E40
-
-dpJZKq5k+fMuC7XECfTSRjPeOEl9wNuVtZkcjEWaHN8ky4umND7ARyRyuU1Nk7cy
-fpCFlFKOqDfCkT5zVK/fB6pF32wqAI7sqeSuYPfQY0+L77yRdwM6L46WslzVKZYE
-lXD1AmqKT/LgF3+eBY5slkAAJo10zYDgKEwnoQVBp31YW2/+6oAGaY/O6x3p7aTG
-dw9CP+SFc0o8lPl1lsSovdNXDUiVCftvClog7hwyDv8AhHyGgynw3UJXX8UlyWu+
-Zz5zpgrvB2gvDLeoZZ6qjMGvtpEwlYBh4de9ZOsvQUpXEEfkQFtJV0j613OCQune
-LoTxKpYV1V/mZX4HPaJ1oC0OJ7XcliAOSS9K49YobTuz7Kg5Wg3bVGo9xRfYDjch
-rVeMP4u5RXSGuHL23FPMfK6EqcldrFyRwLaY/IV1Yl6UNUMKAphn/WMtWVuT3TiT
-QMCI2VRt7ItwZwwFn5RgyDweWdFf5v3AmN/lOhATDBqosahkPxDZ1VZ6OBPoJLPM
-UrDWH/lqrByeEjtAOwr5UsWKwLuJ8qUxQ4TchHwFKOwy6VsrRwMQ3ZWi2govPF9I
-W0sfLj5Ulfjx6zHdqnF48a1Elit4JH6inBEWFuj7bmlOotq+PHoeT61zAwW+gnrG
-3JTo3XnaE2WwRDpqvKYHWLv/J218rq8PtIaq9gjr55odPfIt8lkJ1XzF4WQ21rIJ
-GNWZ3xz4fxpvrKnQyAKGu0ZcdjA1nqs16oiVr+UnJoXmkM5yBCic4fZYwPTSQHYS
-ZxwaTzEjfeGxrSeLrN9CgoweucvogOvUjJOBcW/py80du8vWz0YyzMhg3o0YeGME
-C+Kts/YWxmyfw4DaWt8RtWCKl85hEmz8RODvkMLGtLzvVoSyLQWqp1NhGIlFtzXs
-7sPLulUeyD2avTC/RB/Pu9Nk80c0368BxCoeYbiFWZpaN70SJmCUE5H59J2d0olw
-5v2RVjLBi8wqnzoa0+2L8wnG7IQGadS97dj0eBR+JXNtoJhVrurS80RJ6B0bNxdu
-gX8otfnJYsZyK5hbEhcQqLdnyGhDEE8YHe7Hv9stWwLAFOfOMzyzC06lFS1eNiw4
-FJyXJUhDieb8EqetouAC8dNVXz4Q1zOTlGuAbGoKm5v0U5IhCQap9GUSW5QiUgOQ
-AEMs9aGfd91R+IcDf19mZptsQLYA6MGBN6fm+3O2iZImKIbF+ZZo0S6liFFmn6lm
-M+diTzaoiqgEkiXOuRhdQUMaiGV8BMZxv8qUH6/vyC3gSueoTio0f9PfASDYfvXD
-A3GuI87P6LF1it2UlN6ssFoXTZdfQQZwRmNuqOqw+BJOJHrR6trcXOCZOQ77Qnvd
-M5a348gIzluVUkExAPGCsySQWMx4Of5NBF28jEC3+TAwkRqBV2ZHmfGLWnvwaB+A
-YUeKtpWblfG1lsrDAdwL2dilU95oby+35sExX7M2dCrL9Y2P5oTCW3u12//ZSLeL
-Yhi1Rzol6LAuesZCVF0Zv/YYDhzAckJfT/qXK5B5pz9saswxCUBEpiKlLpVsjOFJ
-2bHm8NgOMD5b3cdh1kvts4wZe+giry7LHsn46f+9VqN+gA6XxeVsPyb4uO1KW3SN
------END RSA PRIVATE KEY-----
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDUXmnnCv7sY9lb
+rS1Q3c5q7ok9R4XEPq/jWBFkIEnJR7vSjCEnOjLxg+1MdUItjbqODf9N5vFbHiiq
+WQVkGrmg8/CTSme0qyNr7FcmG1hO4bzK/dvIyK1R7YISqZpXoCTVzEnUIjU7f+Pk
+X2uAiSypxM4zpNyC7++j6ah8xYNRfR9AS5c7e1dvNKBNMmNipYxVgaEopIke40m1
+2ezIzLOtkL/rGlsnM2Tv/0Wv1xQE+OjHByyQE08vuliatFfweTXLF48m4S4NdOq5
+dh2WX8xLPr8BxRLjXzs08wgKVFpWkIOR2uEInjuVQAGMuZeOqUuQGdarGMH4M/3t
+Dl0eJ7mbAgMBAAECggEAJwzxZlXESI2Xw17VzV/r/Ae+3rDPLSXly+U+1W2Gg+eX
+5wBzfDYcdgKvWPba42uDWWnDf3yu9vVVvvU9o4myhqE0pLDy3ur1SXwdDlnK5D5o
+K9+AUaxtCnqlB29+fQxqmZHGJabgqP88VZsiNnGC7/jLff2buswKAdcOb1sWaZ5F
+p5YBL6TmUzpg3Pdbs2N/OVsZGh5Y+d6m6hJgjsbdcWC4LrFurj9UyQoCGIRruelN
+Ra3ft4QAC3biD0/hEc4WyjzZdlMvvEqaeQUmqK2TJLELwyH33W1Ek2FsLm6LTP9i
+7kW2+9684GWBXo5ge87BXASVwAXnFE+pnXV5QGLMqQKBgQDpo2ZM/R/lWLR8Hpww
+WKZZFfWPpsv0d6DbwVWowdRoojPue62nIby6+LBk08j9UwqkC0TK8pWCjL4hhSfU
+JjLuGuzl+RFfstpt42qh2zgi8aedLtGHjFyjHp1jE3rb9l95YXUpNm7syEMbLR8V
+NR09na41ftFCBPBNYFsdxrzKPwKBgQDosd6qwhrTL+ndtiJIeonRvunjo4yXn22/
+qWqRy3WJmZpDKWpsGWOmlJ4G7+10Q3zOMpb+nUOEjJNI3EdnLrVTeJo8WNuNXHyl
+axgWV3TR8JT6GIG+zavGEI51JjTH3X1eGzm9T4Di8mj34FzyK1af6atiiRj5sIuk
+NG71CUjhpQKBgQCDJ90n3vjm0LMQ8kYPxdQsMm2VZLcd14IPmyqw/45z5opsmDVV
+m1TNSQoMr+8mdlWE3WaS3zcbAFNDkfJX39G7ZJYUS4t7Q3XnNkEH934975Z+YGfz
+RdJDJ86GbcsMa/QQuasBpbMDbTBusxe92gE+M6Q2F6j0/LzBUxQTVRtqFQKBgBFZ
+IXatnf5cthzXdVrd9+RxTVKxYMv1EOOXJ+DSwGKP1xZmwg5pHirPLbDqtlNSrL1a
+vDMjWmNJb7mg4pnou5ALj8QsA8JYQNq8T0FrJ8R3IUQ8C4BEKShNF7HYNVspQi1/
+7iAVC1DgLb89NPDBFmY5r5NbEUecR+zoE9Wk6ZAZAoGBAIHTifNzf7/qDtHI2+Cc
+YGiudMlWWwNqTUr1BjPQx1au1VImpDAB6eaz3DV0oIS0fpREte6SRrcwvtrRqp4M
+AT4uCjiVOaXW/MwybfS6BIivTvuTkyPuNCBIWTH1JBQ3CEcEIIV5YcpFDQs5FQ0M
+GPtHFxbKMUmLmJVW5nbKUUhr
+-----END PRIVATE KEY-----