agents: add option to dump grpc keylog file

By setting the `NSOLID_GRPC_KEYLOG` to a truthy value, a new keylog file
will be generated that should allow us to decrypt the TLS v1.3
connections gRPC Agent uses. Very useful to debug issues on production.
The file will be generated in the binary path with the following format:
`nsolid-tls-keylog-<process_pid>.log`.

Author: santigimeno

agents/grpc/src/grpc_agent.cc

@@ -65,6 +65,7 @@ constexpr size_t span_msg_q_min_size = 1000;
 
 const char* const kNSOLID_GRPC_INSECURE = "NSOLID_GRPC_INSECURE";
 const char* const kNSOLID_GRPC_CERTS = "NSOLID_GRPC_CERTS";
+const char* const kNSOLID_GRPC_KEYLOG = "NSOLID_GRPC_KEYLOG";
 
 const int MAX_AUTH_RETRIES = 20;
 const uint64_t auth_timer_interval = 500;
@@ -460,6 +461,15 @@ GrpcAgent::GrpcAgent(): hooks_init_(false),
       cacert_ += "\n";
     }
   }
+
+  auto keylog = per_process::system_environment->Get(kNSOLID_GRPC_KEYLOG);
+  if (keylog.has_value() && !keylog.value().empty() && keylog.value() != "0") {
+    tls_keylog_file_ = "./nsolid-tls-keylog-" +
+                       std::to_string(uv_os_getpid()) + ".log";
+    uv_fs_t req;
+    uv_fs_unlink(nullptr, &req, tls_keylog_file_.c_str(), nullptr);
+    uv_fs_req_cleanup(&req);
+  }
 }
 
 GrpcAgent::~GrpcAgent() {
@@ -1053,11 +1063,16 @@ int GrpcAgent::config(const json& config) {
         }
       }
 
-      nsolid_service_stub_ = GrpcClient::MakeNSolidServiceStub(opts);
+      nsolid_service_stub_ =
+          GrpcClient::MakeNSolidServiceStub(opts, tls_keylog_file_);
+
       // CommandStream needs to be created before the OTLP client to avoid
       // a race condition with abseil mutexes.
       reset_command_stream();
 
+      // Enable TLS keylog for the OTLP client
+      opts.credentials = GrpcClient::MakeCredentials(opts, tls_keylog_file_);
+
       std::shared_ptr<OtlpGrpcClient> client =
           OtlpGrpcClientFactory::Create(opts);
 

agents/grpc/src/grpc_agent.h

@@ -367,6 +367,7 @@ class GrpcAgent: public std::enable_shared_from_this<GrpcAgent>,
   std::unique_ptr<CommandStream> command_stream_;
   std::string cacert_;
   std::string custom_certs_;
+  std::string tls_keylog_file_;
 
   // For the gRPC server
   nsuv::ns_async command_msg_;

agents/grpc/src/grpc_client.cc

@@ -1,26 +1,66 @@
 #include "grpc_client.h"
 #include "debug_utils-inl.h"
 #include "opentelemetry/exporters/otlp/otlp_grpc_client_options.h"
+#include <grpcpp/security/tls_credentials_options.h>
 
-using grpc::Channel;
-using grpc::ChannelArguments;
-using grpc::ClientContext;
-using grpc::CreateCustomChannel;
-using grpc::InsecureChannelCredentials;
-using grpc::SslCredentials;
-using grpc::SslCredentialsOptions;
+using ::grpc::Channel;
+using ::grpc::ChannelArguments;
+using ::grpc::ClientContext;
+using ::grpc::CreateCustomChannel;
+using ::grpc::InsecureChannelCredentials;
+using ::grpc::SslCredentials;
+using ::grpc::SslCredentialsOptions;
+// The following experimental gRPC TLS APIs are required for TLS session key
+// logging (via set_tls_session_key_log_file_path), which is not currently
+// supported by the stable SslCredentials API.
+// These APIs are subject to change in future gRPC releases. This project
+// currently pins gRPC to version 1.76.0
+// (see deps/grpc/include/grpcpp/version_info.h).
+using ::grpc::experimental::IdentityKeyCertPair;
+using ::grpc::experimental::TlsCredentials;
+using ::grpc::experimental::TlsChannelCredentialsOptions;
+using ::grpc::experimental::StaticDataCertificateProvider;
 using grpcagent::NSolidService;
 using opentelemetry::v1::exporter::otlp::OtlpGrpcClientOptions;
 
 namespace node {
 namespace nsolid {
 namespace grpc {
 
+/**
+  * Create gRPC channel credentials.
+  */
+std::shared_ptr<::grpc::ChannelCredentials>
+    GrpcClient::MakeCredentials(const OtlpGrpcClientOptions& options,
+                                const std::string& tls_keylog_file) {
+  if (!options.use_ssl_credentials) {
+    return InsecureChannelCredentials();
+  }
+
+  if (!tls_keylog_file.empty()) {
+    TlsChannelCredentialsOptions tls_opts;
+    if (!options.ssl_credentials_cacert_as_string.empty()) {
+      auto cert_provider = std::make_shared<StaticDataCertificateProvider>(
+          options.ssl_credentials_cacert_as_string,
+          std::vector<IdentityKeyCertPair>());
+      tls_opts.set_certificate_provider(cert_provider);
+      tls_opts.watch_root_certs();
+    }
+    tls_opts.set_tls_session_key_log_file_path(tls_keylog_file);
+    return TlsCredentials(tls_opts);
+  }
+
+  SslCredentialsOptions ssl_opts;
+  ssl_opts.pem_root_certs = options.ssl_credentials_cacert_as_string;
+  return SslCredentials(ssl_opts);
+}
+
 /**
   * Create gRPC channel.
   */
 std::shared_ptr<Channel>
-    GrpcClient::MakeChannel(const OtlpGrpcClientOptions& options) {
+    GrpcClient::MakeChannel(const OtlpGrpcClientOptions& options,
+                            const std::string& tls_keylog_file) {
   std::shared_ptr<Channel> channel;
   ChannelArguments grpc_arguments;
   // Configure the keepalive of the Client Channel. The keepalive time period is
@@ -32,21 +72,10 @@ std::shared_ptr<Channel>
   grpc_arguments.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, 15 * 1000 /* 15 sec*/);
   grpc_arguments.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, 1);
   grpc_arguments.SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, 0);
-  if (!options.use_ssl_credentials) {
-    channel = CreateCustomChannel(options.endpoint,
-                                  InsecureChannelCredentials(),
-                                  grpc_arguments);
-    return channel;
-  }
-
 
-  SslCredentialsOptions ssl_opts;
-  ssl_opts.pem_root_certs = options.ssl_credentials_cacert_as_string;
-  auto channel_creds = SslCredentials(ssl_opts);
-  channel = CreateCustomChannel(options.endpoint,
-                                channel_creds,
-                                grpc_arguments);
-  return channel;
+  return CreateCustomChannel(options.endpoint,
+                             MakeCredentials(options, tls_keylog_file),
+                             grpc_arguments);
 }
 
 /**
@@ -68,8 +97,9 @@ GrpcClient::MakeClientContext(const std::string& agent_id,
   * Create N|Solid service stub to communicate with the N|Solid Console.
   */
 std::unique_ptr<NSolidService::StubInterface>
-    GrpcClient::MakeNSolidServiceStub(const OtlpGrpcClientOptions& options) {
-  return NSolidService::NewStub(MakeChannel(options));
+    GrpcClient::MakeNSolidServiceStub(const OtlpGrpcClientOptions& options,
+                                      const std::string& tls_keylog_file) {
+  return NSolidService::NewStub(MakeChannel(options, tls_keylog_file));
 }
 
 }  // namespace grpc

agents/grpc/src/grpc_client.h

@@ -60,7 +60,15 @@ class GrpcClient {
    * Create gRPC channel.
    */
   static std::shared_ptr<::grpc::Channel>
-    MakeChannel(const OtlpGrpcClientOptions& options);
+    MakeChannel(const OtlpGrpcClientOptions& options,
+                const std::string& tls_keylog_file = "");
+
+  /**
+   * Create gRPC channel credentials.
+   */
+  static std::shared_ptr<::grpc::ChannelCredentials>
+    MakeCredentials(const OtlpGrpcClientOptions& options,
+                    const std::string& tls_keylog_file);
 
   /**
    * Create gRPC client context to call RPC.
@@ -72,7 +80,8 @@ class GrpcClient {
    * Create N|Solid service stub to communicate with the N|Solid Console.
    */
   static std::unique_ptr<grpcagent::NSolidService::StubInterface>
-    MakeNSolidServiceStub(const OtlpGrpcClientOptions& options);
+    MakeNSolidServiceStub(const OtlpGrpcClientOptions& options,
+                          const std::string& tls_keylog_file);
 
   /**
    * Generic DelegateAsyncExport for any event type.

deps/opentelemetry-cpp/otlp-http-exporter.gyp

@@ -62,6 +62,7 @@
       'defines': [
         'BUILDING_LIBCURL',
         'ENABLE_ASYNC_EXPORT',
+        'ENABLE_OTLP_GRPC_CREDENTIAL_PREVIEW',
         'OPENTELEMETRY_STL_VERSION=2020',
       ],
       'dependencies': [
@@ -74,6 +75,7 @@
       'direct_dependent_settings': {
         'defines': [
           'ENABLE_ASYNC_EXPORT',
+          'ENABLE_OTLP_GRPC_CREDENTIAL_PREVIEW',
           'OPENTELEMETRY_STL_VERSION=2020',
         ],
         'include_dirs': [

test/agents/test-grpc-keylog-file.mjs

@@ -0,0 +1,50 @@
+// Flags: --expose-internals
+import { mustSucceed } from '../common/index.mjs';
+import fixtures from '../common/fixtures.js';
+import fs from 'node:fs';
+import {
+  GRPCServer,
+  TestClient,
+} from '../common/nsolid-grpc-agent/index.js';
+
+async function runTest({ getEnv, nsolidConfig }) {
+  return new Promise((resolve, reject) => {
+    const grpcServer = new GRPCServer({ tls: true });
+    grpcServer.start(mustSucceed(async (port) => {
+      console.log('GRPC server started', port);
+      const env = getEnv(port);
+      const opts = {
+        stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
+        env,
+      };
+      const child = new TestClient([], opts);
+      // Make sure grpc connections are up
+      await child.id();
+      // Check keylog file exists
+      const keylogFile = `./nsolid-tls-keylog-${child.child().pid}.log`;
+      // Should throw if file does not exist
+      fs.unlinkSync(keylogFile);
+      await child.shutdown(0);
+      grpcServer.close();
+      resolve();
+    }));
+  });
+}
+
+const testConfigs = [
+  {
+    getEnv: (port) => {
+      return {
+        NODE_DEBUG_NATIVE: 'nsolid_grpc_agent',
+        NSOLID_GRPC: `localhost:${port}`,
+        NSOLID_GRPC_CERTS: fixtures.path('keys', 'selfsigned-no-keycertsign', 'cert.pem'),
+        NSOLID_GRPC_KEYLOG: '1',
+      };
+    },
+  },
+];
+
+for (const testConfig of testConfigs) {
+  await runTest(testConfig);
+  console.log('run test!');
+}