[EXPORTER] Support handling retry-able errors for OTLP/HTTP

CHANGELOG.md

@@ -24,6 +24,9 @@ Increment the:
 * [EXPORTER] Fix scope attributes missing from otlp traces metrics
   [#3185](https://github.com/open-telemetry/opentelemetry-cpp/pull/3185)
 
+  [EXPORTER] Support handling retry-able errors for OTLP/HTTP
+  [#3223](https://github.com/open-telemetry/opentelemetry-cpp/pull/3223)
+
 ## [1.18 2024-11-25]
 
 * [EXPORTER] Fix crash in ElasticsearchLogRecordExporter

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h

@@ -152,6 +152,22 @@ std::string GetOtlpDefaultTracesCompression();
 std::string GetOtlpDefaultMetricsCompression();
 std::string GetOtlpDefaultLogsCompression();
 
+std::uint32_t GetOtlpDefaultTracesRetryMaxAttempts();
+std::uint32_t GetOtlpDefaultMetricsRetryMaxAttempts();
+std::uint32_t GetOtlpDefaultLogsRetryMaxAttempts();
+
+std::chrono::duration<float> GetOtlpDefaultTracesRetryInitialBackoff();
+std::chrono::duration<float> GetOtlpDefaultMetricsRetryInitialBackoff();
+std::chrono::duration<float> GetOtlpDefaultLogsRetryInitialBackoff();
+
+std::chrono::duration<float> GetOtlpDefaultTracesRetryMaxBackoff();
+std::chrono::duration<float> GetOtlpDefaultMetricsRetryMaxBackoff();
+std::chrono::duration<float> GetOtlpDefaultLogsRetryMaxBackoff();
+
+float GetOtlpDefaultTracesRetryBackoffMultiplier();
+float GetOtlpDefaultMetricsRetryBackoffMultiplier();
+float GetOtlpDefaultLogsRetryBackoffMultiplier();
+
 }  // namespace otlp
 }  // namespace exporter
 OPENTELEMETRY_END_NAMESPACE

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h

@@ -74,6 +74,9 @@ struct OtlpHttpClientOptions
   // Additional HTTP headers
   OtlpHeaders http_headers;
 
+  // Retry policy for select failure codes
+  ext::http::client::RetryPolicy retry_policy;
+
   // Concurrent requests
   std::size_t max_concurrent_requests = 64;
 
@@ -102,6 +105,10 @@ struct OtlpHttpClientOptions
                                bool input_console_debug,
                                std::chrono::system_clock::duration input_timeout,
                                const OtlpHeaders &input_http_headers,
+                               std::uint32_t input_retry_policy_max_attempts,
+                               std::chrono::duration<float> input_retry_policy_initial_backoff,
+                               std::chrono::duration<float> input_retry_policy_max_backoff,
+                               float input_retry_policy_backoff_multiplier,
                                std::size_t input_concurrent_sessions         = 64,
                                std::size_t input_max_requests_per_connection = 8,
                                nostd::string_view input_user_agent = GetOtlpDefaultUserAgent())
@@ -125,6 +132,8 @@ struct OtlpHttpClientOptions
         console_debug(input_console_debug),
         timeout(input_timeout),
         http_headers(input_http_headers),
+        retry_policy{input_retry_policy_max_attempts, input_retry_policy_initial_backoff,
+                     input_retry_policy_max_backoff, input_retry_policy_backoff_multiplier},
         max_concurrent_requests(input_concurrent_sessions),
         max_requests_per_connection(input_max_requests_per_connection),
         user_agent(input_user_agent)

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <chrono>
+#include <cstdint>
 #include <string>
 
 #include "opentelemetry/exporters/otlp/otlp_environment.h"
@@ -101,6 +102,18 @@ struct OPENTELEMETRY_EXPORT OtlpHttpExporterOptions
 
   /** Compression type. */
   std::string compression;
+
+  /** The maximum number of call attempts, including the original attempt. */
+  std::uint32_t retry_policy_max_attempts{};
+
+  /** The initial backoff delay between retry attempts, random between (0, initial_backoff). */
+  std::chrono::duration<float> retry_policy_initial_backoff{};
+
+  /** The maximum backoff places an upper limit on exponential backoff growth. */
+  std::chrono::duration<float> retry_policy_max_backoff{};
+
+  /** The backoff will be multiplied by this value after each retry attempt. */
+  float retry_policy_backoff_multiplier{};
 };
 
 }  // namespace otlp

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <chrono>
+#include <cstdint>
 #include <string>
 
 #include "opentelemetry/exporters/otlp/otlp_environment.h"
@@ -101,6 +102,18 @@ struct OPENTELEMETRY_EXPORT OtlpHttpLogRecordExporterOptions
 
   /** Compression type. */
   std::string compression;
+
+  /** The maximum number of call attempts, including the original attempt. */
+  std::uint32_t retry_policy_max_attempts{};
+
+  /** The initial backoff delay between retry attempts, random between (0, initial_backoff). */
+  std::chrono::duration<float> retry_policy_initial_backoff{};
+
+  /** The maximum backoff places an upper limit on exponential backoff growth. */
+  std::chrono::duration<float> retry_policy_max_backoff{};
+
+  /** The backoff will be multiplied by this value after each retry attempt. */
+  float retry_policy_backoff_multiplier{};
 };
 
 }  // namespace otlp

exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <chrono>
+#include <cstdint>
 #include <string>
 
 #include "opentelemetry/exporters/otlp/otlp_environment.h"
@@ -104,6 +105,18 @@ struct OPENTELEMETRY_EXPORT OtlpHttpMetricExporterOptions
 
   /** Compression type. */
   std::string compression;
+
+  /** The maximum number of call attempts, including the original attempt. */
+  std::uint32_t retry_policy_max_attempts{};
+
+  /** The initial backoff delay between retry attempts, random between (0, initial_backoff). */
+  std::chrono::duration<float> retry_policy_initial_backoff{};
+
+  /** The maximum backoff places an upper limit on exponential backoff growth. */
+  std::chrono::duration<float> retry_policy_max_backoff{};
+
+  /** The backoff will be multiplied by this value after each retry attempt. */
+  float retry_policy_backoff_multiplier{};
 };
 
 }  // namespace otlp

exporters/otlp/src/otlp_environment.cc

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include <chrono>
+#include <cstdint>
 #include <map>
 #include <string>
 #include <unordered_set>
@@ -78,6 +79,38 @@ static bool GetStringDualEnvVar(const char *signal_name,
   return exists;
 }
 
+static bool GetUintDualEnvVar(const char *signal_name,
+                              const char *generic_name,
+                              std::uint32_t &value)
+{
+  bool exists;
+
+  exists = sdk_common::GetUintEnvironmentVariable(signal_name, value);
+  if (exists)
+  {
+    return true;
+  }
+
+  exists = sdk_common::GetUintEnvironmentVariable(generic_name, value);
+
+  return exists;
+}
+
+static bool GetFloatDualEnvVar(const char *signal_name, const char *generic_name, float &value)
+{
+  bool exists;
+
+  exists = sdk_common::GetFloatEnvironmentVariable(signal_name, value);
+  if (exists)
+  {
+    return true;
+  }
+
+  exists = sdk_common::GetFloatEnvironmentVariable(generic_name, value);
+
+  return exists;
+}
+
 std::string GetOtlpDefaultGrpcTracesEndpoint()
 {
   constexpr char kSignalEnv[]  = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT";
@@ -1123,6 +1156,174 @@ std::string GetOtlpDefaultLogsCompression()
   return std::string{"none"};
 }
 
+std::uint32_t GetOtlpDefaultTracesRetryMaxAttempts()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_MAX_ATTEMPTS";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS";
+  std::uint32_t value{};
+
+  if (GetUintDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return value;
+  }
+
+  return 5U;
+}
+
+std::uint32_t GetOtlpDefaultMetricsRetryMaxAttempts()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_MAX_ATTEMPTS";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS";
+  std::uint32_t value{};
+
+  if (GetUintDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return value;
+  }
+
+  return 5U;
+}
+
+std::uint32_t GetOtlpDefaultLogsRetryMaxAttempts()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_MAX_ATTEMPTS";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS";
+  std::uint32_t value{};
+
+  if (GetUintDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return value;
+  }
+
+  return 5U;
+}
+
+std::chrono::duration<float> GetOtlpDefaultTracesRetryInitialBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_INITIAL_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF";
+  float value{};
+
+  if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return std::chrono::duration<float>{value};
+  }
+
+  return std::chrono::duration<float>{1.0f};
+}
+
+std::chrono::duration<float> GetOtlpDefaultMetricsRetryInitialBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_INITIAL_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF";
+  float value{};
+
+  if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return std::chrono::duration<float>{value};
+  }
+
+  return std::chrono::duration<float>{1.0f};
+}
+
+std::chrono::duration<float> GetOtlpDefaultLogsRetryInitialBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_INITIAL_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF";
+  float value{};
+
+  if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return std::chrono::duration<float>{value};
+  }
+
+  return std::chrono::duration<float>{1.0f};
+}
+
+std::chrono::duration<float> GetOtlpDefaultTracesRetryMaxBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_MAX_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF";
+  float value{};
+
+  if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return std::chrono::duration<float>{value};
+  }
+
+  return std::chrono::duration<float>{5.0f};
+}
+
+std::chrono::duration<float> GetOtlpDefaultMetricsRetryMaxBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_MAX_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF";
+  float value{};
+
+  if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return std::chrono::duration<float>{value};
+  }
+
+  return std::chrono::duration<float>{5.0f};
+}
+
+std::chrono::duration<float> GetOtlpDefaultLogsRetryMaxBackoff()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_MAX_BACKOFF";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF";
+  float value{};
+
+  if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return std::chrono::duration<float>{value};
+  }
+
+  return std::chrono::duration<float>{5.0f};
+}
+
+float GetOtlpDefaultTracesRetryBackoffMultiplier()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_BACKOFF_MULTIPLIER";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER";
+  float value{};
+
+  if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return value;
+  }
+
+  return 1.5f;
+}
+
+float GetOtlpDefaultMetricsRetryBackoffMultiplier()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_BACKOFF_MULTIPLIER";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER";
+  float value{};
+
+  if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return value;
+  }
+
+  return 1.5f;
+}
+
+float GetOtlpDefaultLogsRetryBackoffMultiplier()
+{
+  constexpr char kSignalEnv[]  = "OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_BACKOFF_MULTIPLIER";
+  constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER";
+  float value{};
+
+  if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value))
+  {
+    return value;
+  }
+
+  return 1.5f;
+}
+
 }  // namespace otlp
 }  // namespace exporter
 OPENTELEMETRY_END_NAMESPACE

exporters/otlp/src/otlp_http_client.cc

@@ -992,6 +992,7 @@ OtlpHttpClient::createSession(
   request->ReplaceHeader("Content-Type", content_type);
   request->ReplaceHeader("User-Agent", options_.user_agent);
   request->EnableLogging(options_.console_debug);
+  request->SetRetryPolicy(options_.retry_policy);
 
   if (options_.compression == "gzip")
   {