[EXPORTER] Support handling retry-able errors for OTLP/gRPC

exporters/otlp/CMakeLists.txt

@@ -317,7 +317,7 @@ if(BUILD_TESTING)
     add_executable(otlp_grpc_exporter_test test/otlp_grpc_exporter_test.cc)
     target_link_libraries(
       otlp_grpc_exporter_test ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}
-      ${GMOCK_LIB} opentelemetry_exporter_otlp_grpc)
+      ${GMOCK_LIB} opentelemetry_exporter_otlp_grpc gRPC::grpc++)
     gtest_add_tests(
       TARGET otlp_grpc_exporter_test
       TEST_PREFIX exporter.otlp.

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h

@@ -152,6 +152,22 @@ std::string GetOtlpDefaultTracesCompression();
 std::string GetOtlpDefaultMetricsCompression();
 std::string GetOtlpDefaultLogsCompression();
 
+std::uint32_t GetOtlpDefaultTracesRetryMaxAttempts();
+std::uint32_t GetOtlpDefaultMetricsRetryMaxAttempts();
+std::uint32_t GetOtlpDefaultLogsRetryMaxAttempts();
+
+float GetOtlpDefaultTracesRetryInitialBackoff();
+float GetOtlpDefaultMetricsRetryInitialBackoff();
+float GetOtlpDefaultLogsRetryInitialBackoff();
+
+float GetOtlpDefaultTracesRetryMaxBackoff();
+float GetOtlpDefaultMetricsRetryMaxBackoff();
+float GetOtlpDefaultLogsRetryMaxBackoff();
+
+float GetOtlpDefaultTracesRetryBackoffMultiplier();
+float GetOtlpDefaultMetricsRetryBackoffMultiplier();
+float GetOtlpDefaultLogsRetryBackoffMultiplier();
+
 }  // namespace otlp
 }  // namespace exporter
 OPENTELEMETRY_END_NAMESPACE

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_grpc_client_options.h

@@ -62,6 +62,18 @@ struct OtlpGrpcClientOptions
   // Concurrent requests
   std::size_t max_concurrent_requests;
 #endif
+
+  /** The maximum number of call attempts, including the original attempt. */
+  std::uint32_t retry_policy_max_attempts{};
+
+  /** The initial backoff delay between retry attempts, random between (0, initial_backoff). */
+  float retry_policy_initial_backoff{};
+
+  /** The maximum backoff places an upper limit on exponential backoff growth. */
+  float retry_policy_max_backoff{};
+
+  /** The backoff will be multiplied by this value after each retry attempt. */
+  float retry_policy_backoff_multiplier{};
 };
 
 }  // namespace otlp

exporters/otlp/src/otlp_environment.cc

@@ -80,6 +80,34 @@ static bool GetStringDualEnvVar(const char *signal_name,
   return exists;
 }
 
+static std::uint32_t GetUintEnvVarOrDefault(opentelemetry::nostd::string_view signal_env,
+                                            opentelemetry::nostd::string_view generic_env,
+                                            std::uint32_t default_value)
+{
+  std::string value;
+
+  if (GetStringDualEnvVar(signal_env.data(), generic_env.data(), value))
+  {
+    return static_cast<std::uint32_t>(std::strtoul(value.c_str(), nullptr, 10));
+  }
+
+  return default_value;
+}
+
+static float GetFloatEnvVarOrDefault(opentelemetry::nostd::string_view signal_env,
+                                     opentelemetry::nostd::string_view generic_env,
+                                     float default_value)
+{
+  std::string value;
+
+  if (GetStringDualEnvVar(signal_env.data(), generic_env.data(), value))
+  {
+    return std::strtof(value.c_str(), nullptr);
+  }
+
+  return default_value;
+}
+
 std::string GetOtlpDefaultGrpcTracesEndpoint()
 {
   constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT";
@@ -1125,6 +1153,90 @@ std::string GetOtlpDefaultLogsCompression()
   return std::string{"none"};
 }
 
+std::uint32_t GetOtlpDefaultTracesRetryMaxAttempts()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_TRACES_RETRY_MAX_ATTEMPTS";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS";
+  return GetUintEnvVarOrDefault(kSignalEnv, kGenericEnv, 5U);
+}
+
+std::uint32_t GetOtlpDefaultMetricsRetryMaxAttempts()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_METRICS_RETRY_MAX_ATTEMPTS";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS";
+  return GetUintEnvVarOrDefault(kSignalEnv, kGenericEnv, 5U);
+}
+
+std::uint32_t GetOtlpDefaultLogsRetryMaxAttempts()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_LOGS_RETRY_MAX_ATTEMPTS";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS";
+  return GetUintEnvVarOrDefault(kSignalEnv, kGenericEnv, 5U);
+}
+
+float GetOtlpDefaultTracesRetryInitialBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_TRACES_RETRY_INITIAL_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF";
+  return GetFloatEnvVarOrDefault(kSignalEnv, kGenericEnv, 1.0);
+}
+
+float GetOtlpDefaultMetricsRetryInitialBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_METRICS_RETRY_INITIAL_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF";
+  return GetFloatEnvVarOrDefault(kSignalEnv, kGenericEnv, 1.0);
+}
+
+float GetOtlpDefaultLogsRetryInitialBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_LOGS_RETRY_INITIAL_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF";
+  return GetFloatEnvVarOrDefault(kSignalEnv, kGenericEnv, 1.0);
+}
+
+float GetOtlpDefaultTracesRetryMaxBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_TRACES_RETRY_MAX_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_MAX_BACKOFF";
+  return GetFloatEnvVarOrDefault(kSignalEnv, kGenericEnv, 5.0);
+}
+
+float GetOtlpDefaultMetricsRetryMaxBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_METRICS_RETRY_MAX_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_MAX_BACKOFF";
+  return GetFloatEnvVarOrDefault(kSignalEnv, kGenericEnv, 5.0);
+}
+
+float GetOtlpDefaultLogsRetryMaxBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_LOGS_RETRY_MAX_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_MAX_BACKOFF";
+  return GetFloatEnvVarOrDefault(kSignalEnv, kGenericEnv, 5.0);
+}
+
+float GetOtlpDefaultTracesRetryBackoffMultiplier()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_TRACES_RETRY_BACKOFF_MULTIPLIER";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER";
+  return GetFloatEnvVarOrDefault(kSignalEnv, kGenericEnv, 1.5f);
+}
+
+float GetOtlpDefaultMetricsRetryBackoffMultiplier()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_METRICS_RETRY_BACKOFF_MULTIPLIER";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER";
+  return GetFloatEnvVarOrDefault(kSignalEnv, kGenericEnv, 1.5f);
+}
+
+float GetOtlpDefaultLogsRetryBackoffMultiplier()
+{
+  constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_LOGS_RETRY_BACKOFF_MULTIPLIER";
+  constexpr char kGenericEnv[] = "OTEL_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER";
+  return GetFloatEnvVarOrDefault(kSignalEnv, kGenericEnv, 1.5f);
+}
+
 }  // namespace otlp
 }  // namespace exporter
 OPENTELEMETRY_END_NAMESPACE

exporters/otlp/src/otlp_grpc_client.cc

@@ -12,6 +12,7 @@
 #include <atomic>
 #include <chrono>
 #include <condition_variable>
+#include <cstdio>
 #include <fstream>
 #include <iterator>
 #include <memory>
@@ -23,6 +24,7 @@
 #include "opentelemetry/common/timestamp.h"
 #include "opentelemetry/ext/http/common/url_parser.h"
 #include "opentelemetry/nostd/function_ref.h"
+#include "opentelemetry/nostd/string_view.h"
 #include "opentelemetry/sdk/common/global_log_handler.h"
 
 OPENTELEMETRY_BEGIN_NAMESPACE
@@ -347,6 +349,43 @@ std::shared_ptr<grpc::Channel> OtlpGrpcClient::MakeChannel(const OtlpGrpcClientO
     grpc_arguments.SetCompressionAlgorithm(GRPC_COMPRESS_GZIP);
   }
 
+  if (options.retry_policy_max_attempts > 0U && options.retry_policy_initial_backoff > 0.0f &&
+      options.retry_policy_max_backoff > 0.0f && options.retry_policy_backoff_multiplier > 0.0f)
+  {
+    static const auto kServiceConfigJson = opentelemetry::nostd::string_view{R"(
+    {
+      "methodConfig": [
+        {
+          "name": [{}],
+          "retryPolicy": {
+            "maxAttempts": %0000000000u,
+            "initialBackoff": "%.1fs",
+            "maxBackoff": "%.1fs",
+            "backoffMultiplier": %.1f,
+            "retryableStatusCodes": [
+              "CANCELLED",
+              "DEADLINE_EXCEEDED",
+              "ABORTED",
+              "OUT_OF_RANGE",
+              "DATA_LOSS",
+              "UNAVAILABLE"
+            ]
+          }
+        }
+      ]
+    })"};
+
+    // Allocate string with buffer large enough to hold the formatted json config
+    auto service_config = std::string(kServiceConfigJson.size(), '\0');
+    // Prior to C++17, need to explicitly cast away constness from `data()` buffer
+    std::snprintf(const_cast<decltype(service_config)::value_type *>(service_config.data()),
+                  service_config.size(), kServiceConfigJson.data(),
+                  options.retry_policy_max_attempts, options.retry_policy_initial_backoff,
+                  options.retry_policy_max_backoff, options.retry_policy_backoff_multiplier);
+
+    grpc_arguments.SetServiceConfigJSON(service_config);
+  }
+
   if (options.use_ssl_credentials)
   {
     grpc::SslCredentialsOptions ssl_opts;

exporters/otlp/src/otlp_grpc_exporter.cc

@@ -105,7 +105,7 @@ sdk::common::ExportResult OtlpGrpcExporter::Export(
   google::protobuf::ArenaOptions arena_options;
   // It's easy to allocate datas larger than 1024 when we populate basic resource and attributes
   arena_options.initial_block_size = 1024;
-  // When in batch mode, it's easy to export a large number of spans at once, we can alloc a lager
+  // When in batch mode, it's easy to export a large number of spans at once, we can alloc a larger
   // block to reduce memory fragments.
   arena_options.max_block_size = 65536;
   std::unique_ptr<google::protobuf::Arena> arena{new google::protobuf::Arena{arena_options}};

exporters/otlp/src/otlp_grpc_exporter_options.cc

@@ -36,6 +36,11 @@ OtlpGrpcExporterOptions::OtlpGrpcExporterOptions()
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif
+
+  retry_policy_max_attempts       = GetOtlpDefaultTracesRetryMaxAttempts();
+  retry_policy_initial_backoff    = GetOtlpDefaultTracesRetryInitialBackoff();
+  retry_policy_max_backoff        = GetOtlpDefaultTracesRetryMaxBackoff();
+  retry_policy_backoff_multiplier = GetOtlpDefaultTracesRetryBackoffMultiplier();
 }
 
 OtlpGrpcExporterOptions::~OtlpGrpcExporterOptions() {}

exporters/otlp/src/otlp_grpc_log_record_exporter_options.cc

@@ -34,6 +34,11 @@ OtlpGrpcLogRecordExporterOptions::OtlpGrpcLogRecordExporterOptions()
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif
+
+  retry_policy_max_attempts       = GetOtlpDefaultLogsRetryMaxAttempts();
+  retry_policy_initial_backoff    = GetOtlpDefaultLogsRetryInitialBackoff();
+  retry_policy_max_backoff        = GetOtlpDefaultLogsRetryMaxBackoff();
+  retry_policy_backoff_multiplier = GetOtlpDefaultLogsRetryBackoffMultiplier();
 }
 
 OtlpGrpcLogRecordExporterOptions::~OtlpGrpcLogRecordExporterOptions() {}

exporters/otlp/src/otlp_grpc_metric_exporter_options.cc

@@ -35,6 +35,11 @@ OtlpGrpcMetricExporterOptions::OtlpGrpcMetricExporterOptions()
 #ifdef ENABLE_ASYNC_EXPORT
   max_concurrent_requests = 64;
 #endif
+
+  retry_policy_max_attempts       = GetOtlpDefaultMetricsRetryMaxAttempts();
+  retry_policy_initial_backoff    = GetOtlpDefaultMetricsRetryInitialBackoff();
+  retry_policy_max_backoff        = GetOtlpDefaultMetricsRetryMaxBackoff();
+  retry_policy_backoff_multiplier = GetOtlpDefaultMetricsRetryBackoffMultiplier();
 }
 
 OtlpGrpcMetricExporterOptions::~OtlpGrpcMetricExporterOptions() {}