diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aa23093bb..81726cd493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,9 @@ Increment the: * [SDK] Enable deriving from ResourceDetector to create a Resource [#3247](https://github.com/open-telemetry/opentelemetry-cpp/pull/3247) +* [EXPORTER] Support handling retry-able errors for OTLP/HTTP + [#3223](https://github.com/open-telemetry/opentelemetry-cpp/pull/3223) + New features: * [SDK] Better control of threads executed by opentelemetry-cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index aa76c997bc..88709cf564 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -212,6 +212,9 @@ if(NOT WITH_STL STREQUAL "OFF") endif() endif() +option(WITH_OTLP_RETRY_PREVIEW + "Whether to enable experimental retry functionality" OFF) + option(WITH_OTLP_GRPC_SSL_MTLS_PREVIEW "Whether to enable mTLS support fro gRPC" OFF) diff --git a/api/CMakeLists.txt b/api/CMakeLists.txt index b3c97808f6..78b924681f 100644 --- a/api/CMakeLists.txt +++ b/api/CMakeLists.txt @@ -116,6 +116,11 @@ target_compile_definitions( opentelemetry_api INTERFACE OPENTELEMETRY_ABI_VERSION_NO=${OPENTELEMETRY_ABI_VERSION_NO}) +if(WITH_OTLP_RETRY_PREVIEW) + target_compile_definitions(opentelemetry_api + INTERFACE ENABLE_OTLP_RETRY_PREVIEW) +endif() + if(WITH_OTLP_GRPC_SSL_MTLS_PREVIEW) target_compile_definitions(opentelemetry_api INTERFACE ENABLE_OTLP_GRPC_SSL_MTLS_PREVIEW) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 1fd1a37f80..7c35a88de1 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -120,6 +120,7 @@ elif [[ "$1" == "cmake.maintainer.sync.test" ]]; then -DOTELCPP_MAINTAINER_MODE=ON \ -DWITH_NO_DEPRECATED_CODE=ON \ -DWITH_OTLP_HTTP_COMPRESSION=ON \ + -DWITH_OTLP_RETRY_PREVIEW=ON \ -DWITH_THREAD_INSTRUMENTATION_PREVIEW=ON \ "${SRC_DIR}" eval "$MAKE_COMMAND" @@ -142,6 +143,7 @@ elif [[ "$1" == "cmake.maintainer.async.test" ]]; then -DOTELCPP_MAINTAINER_MODE=ON \ -DWITH_NO_DEPRECATED_CODE=ON \ -DWITH_OTLP_HTTP_COMPRESSION=ON \ + -DWITH_OTLP_RETRY_PREVIEW=ON \ -DWITH_THREAD_INSTRUMENTATION_PREVIEW=ON \ "${SRC_DIR}" eval "$MAKE_COMMAND" @@ -165,6 +167,7 @@ elif [[ "$1" == "cmake.maintainer.cpp11.async.test" ]]; then -DOTELCPP_MAINTAINER_MODE=ON \ -DWITH_NO_DEPRECATED_CODE=ON \ -DWITH_OTLP_HTTP_COMPRESSION=ON \ + -DWITH_OTLP_RETRY_PREVIEW=ON \ -DWITH_THREAD_INSTRUMENTATION_PREVIEW=ON \ "${SRC_DIR}" make -k -j $(nproc) @@ -189,6 +192,7 @@ elif [[ "$1" == "cmake.maintainer.abiv2.test" ]]; then -DWITH_ABI_VERSION_1=OFF \ -DWITH_ABI_VERSION_2=ON \ -DWITH_OTLP_HTTP_COMPRESSION=ON \ + -DWITH_OTLP_RETRY_PREVIEW=ON \ -DWITH_THREAD_INSTRUMENTATION_PREVIEW=ON \ "${SRC_DIR}" eval "$MAKE_COMMAND" diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h index 74a222a9d0..f6ea55d32d 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_environment.h @@ -152,6 +152,22 @@ std::string GetOtlpDefaultTracesCompression(); std::string GetOtlpDefaultMetricsCompression(); std::string GetOtlpDefaultLogsCompression(); +std::uint32_t GetOtlpDefaultTracesRetryMaxAttempts(); +std::uint32_t GetOtlpDefaultMetricsRetryMaxAttempts(); +std::uint32_t GetOtlpDefaultLogsRetryMaxAttempts(); + +std::chrono::duration GetOtlpDefaultTracesRetryInitialBackoff(); +std::chrono::duration GetOtlpDefaultMetricsRetryInitialBackoff(); +std::chrono::duration GetOtlpDefaultLogsRetryInitialBackoff(); + +std::chrono::duration GetOtlpDefaultTracesRetryMaxBackoff(); +std::chrono::duration GetOtlpDefaultMetricsRetryMaxBackoff(); +std::chrono::duration GetOtlpDefaultLogsRetryMaxBackoff(); + +float GetOtlpDefaultTracesRetryBackoffMultiplier(); +float GetOtlpDefaultMetricsRetryBackoffMultiplier(); +float GetOtlpDefaultLogsRetryBackoffMultiplier(); + } // namespace otlp } // namespace exporter OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h index 1668f0381d..5cb5fdfbc6 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h @@ -75,6 +75,9 @@ struct OtlpHttpClientOptions // Additional HTTP headers OtlpHeaders http_headers; + // Retry policy for select failure codes + ext::http::client::RetryPolicy retry_policy; + // Concurrent requests std::size_t max_concurrent_requests = 64; @@ -107,6 +110,10 @@ struct OtlpHttpClientOptions bool input_console_debug, std::chrono::system_clock::duration input_timeout, const OtlpHeaders &input_http_headers, + std::uint32_t input_retry_policy_max_attempts, + std::chrono::duration input_retry_policy_initial_backoff, + std::chrono::duration input_retry_policy_max_backoff, + float input_retry_policy_backoff_multiplier, const std::shared_ptr &input_thread_instrumentation, std::size_t input_concurrent_sessions = 64, std::size_t input_max_requests_per_connection = 8, @@ -131,6 +138,8 @@ struct OtlpHttpClientOptions console_debug(input_console_debug), timeout(input_timeout), http_headers(input_http_headers), + retry_policy{input_retry_policy_max_attempts, input_retry_policy_initial_backoff, + input_retry_policy_max_backoff, input_retry_policy_backoff_multiplier}, max_concurrent_requests(input_concurrent_sessions), max_requests_per_connection(input_max_requests_per_connection), user_agent(input_user_agent), diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h index 7f6d5a1b35..314854f292 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter_options.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include "opentelemetry/exporters/otlp/otlp_environment.h" @@ -101,6 +102,18 @@ struct OPENTELEMETRY_EXPORT OtlpHttpExporterOptions /** Compression type. */ std::string compression; + + /** The maximum number of call attempts, including the original attempt. */ + std::uint32_t retry_policy_max_attempts{}; + + /** The initial backoff delay between retry attempts, random between (0, initial_backoff). */ + std::chrono::duration retry_policy_initial_backoff{}; + + /** The maximum backoff places an upper limit on exponential backoff growth. */ + std::chrono::duration retry_policy_max_backoff{}; + + /** The backoff will be multiplied by this value after each retry attempt. */ + float retry_policy_backoff_multiplier{}; }; } // namespace otlp diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h index 60f674a3a7..e7c47a5564 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_record_exporter_options.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include "opentelemetry/exporters/otlp/otlp_environment.h" @@ -101,6 +102,18 @@ struct OPENTELEMETRY_EXPORT OtlpHttpLogRecordExporterOptions /** Compression type. */ std::string compression; + + /** The maximum number of call attempts, including the original attempt. */ + std::uint32_t retry_policy_max_attempts{}; + + /** The initial backoff delay between retry attempts, random between (0, initial_backoff). */ + std::chrono::duration retry_policy_initial_backoff{}; + + /** The maximum backoff places an upper limit on exponential backoff growth. */ + std::chrono::duration retry_policy_max_backoff{}; + + /** The backoff will be multiplied by this value after each retry attempt. */ + float retry_policy_backoff_multiplier{}; }; } // namespace otlp diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h index 82c91aa51f..5ff1b28321 100644 --- a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h +++ b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include "opentelemetry/exporters/otlp/otlp_environment.h" @@ -104,6 +105,18 @@ struct OPENTELEMETRY_EXPORT OtlpHttpMetricExporterOptions /** Compression type. */ std::string compression; + + /** The maximum number of call attempts, including the original attempt. */ + std::uint32_t retry_policy_max_attempts{}; + + /** The initial backoff delay between retry attempts, random between (0, initial_backoff). */ + std::chrono::duration retry_policy_initial_backoff{}; + + /** The maximum backoff places an upper limit on exponential backoff growth. */ + std::chrono::duration retry_policy_max_backoff{}; + + /** The backoff will be multiplied by this value after each retry attempt. */ + float retry_policy_backoff_multiplier{}; }; } // namespace otlp diff --git a/exporters/otlp/src/otlp_environment.cc b/exporters/otlp/src/otlp_environment.cc index b95b7ae7c2..a7bc95b321 100644 --- a/exporters/otlp/src/otlp_environment.cc +++ b/exporters/otlp/src/otlp_environment.cc @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 #include +#include #include #include #include @@ -78,6 +79,38 @@ static bool GetStringDualEnvVar(const char *signal_name, return exists; } +static bool GetUintDualEnvVar(const char *signal_name, + const char *generic_name, + std::uint32_t &value) +{ + bool exists; + + exists = sdk_common::GetUintEnvironmentVariable(signal_name, value); + if (exists) + { + return true; + } + + exists = sdk_common::GetUintEnvironmentVariable(generic_name, value); + + return exists; +} + +static bool GetFloatDualEnvVar(const char *signal_name, const char *generic_name, float &value) +{ + bool exists; + + exists = sdk_common::GetFloatEnvironmentVariable(signal_name, value); + if (exists) + { + return true; + } + + exists = sdk_common::GetFloatEnvironmentVariable(generic_name, value); + + return exists; +} + std::string GetOtlpDefaultGrpcTracesEndpoint() { constexpr char kSignalEnv[] = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"; @@ -1123,6 +1156,174 @@ std::string GetOtlpDefaultLogsCompression() return std::string{"none"}; } +std::uint32_t GetOtlpDefaultTracesRetryMaxAttempts() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_MAX_ATTEMPTS"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS"; + std::uint32_t value{}; + + if (GetUintDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return value; + } + + return 5U; +} + +std::uint32_t GetOtlpDefaultMetricsRetryMaxAttempts() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_MAX_ATTEMPTS"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS"; + std::uint32_t value{}; + + if (GetUintDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return value; + } + + return 5U; +} + +std::uint32_t GetOtlpDefaultLogsRetryMaxAttempts() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_MAX_ATTEMPTS"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS"; + std::uint32_t value{}; + + if (GetUintDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return value; + } + + return 5U; +} + +std::chrono::duration GetOtlpDefaultTracesRetryInitialBackoff() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_INITIAL_BACKOFF"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF"; + float value{}; + + if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return std::chrono::duration{value}; + } + + return std::chrono::duration{1.0f}; +} + +std::chrono::duration GetOtlpDefaultMetricsRetryInitialBackoff() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_INITIAL_BACKOFF"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF"; + float value{}; + + if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return std::chrono::duration{value}; + } + + return std::chrono::duration{1.0f}; +} + +std::chrono::duration GetOtlpDefaultLogsRetryInitialBackoff() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_INITIAL_BACKOFF"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF"; + float value{}; + + if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return std::chrono::duration{value}; + } + + return std::chrono::duration{1.0f}; +} + +std::chrono::duration GetOtlpDefaultTracesRetryMaxBackoff() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_MAX_BACKOFF"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF"; + float value{}; + + if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return std::chrono::duration{value}; + } + + return std::chrono::duration{5.0f}; +} + +std::chrono::duration GetOtlpDefaultMetricsRetryMaxBackoff() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_MAX_BACKOFF"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF"; + float value{}; + + if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return std::chrono::duration{value}; + } + + return std::chrono::duration{5.0f}; +} + +std::chrono::duration GetOtlpDefaultLogsRetryMaxBackoff() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_MAX_BACKOFF"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF"; + float value{}; + + if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return std::chrono::duration{value}; + } + + return std::chrono::duration{5.0f}; +} + +float GetOtlpDefaultTracesRetryBackoffMultiplier() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_BACKOFF_MULTIPLIER"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER"; + float value{}; + + if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return value; + } + + return 1.5f; +} + +float GetOtlpDefaultMetricsRetryBackoffMultiplier() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_BACKOFF_MULTIPLIER"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER"; + float value{}; + + if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return value; + } + + return 1.5f; +} + +float GetOtlpDefaultLogsRetryBackoffMultiplier() +{ + constexpr char kSignalEnv[] = "OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_BACKOFF_MULTIPLIER"; + constexpr char kGenericEnv[] = "OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER"; + float value{}; + + if (GetFloatDualEnvVar(kSignalEnv, kGenericEnv, value)) + { + return value; + } + + return 1.5f; +} + } // namespace otlp } // namespace exporter OPENTELEMETRY_END_NAMESPACE diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc index 7499574bcb..a70d249f5e 100644 --- a/exporters/otlp/src/otlp_http_client.cc +++ b/exporters/otlp/src/otlp_http_client.cc @@ -992,6 +992,7 @@ OtlpHttpClient::createSession( request->ReplaceHeader("Content-Type", content_type); request->ReplaceHeader("User-Agent", options_.user_agent); request->EnableLogging(options_.console_debug); + request->SetRetryPolicy(options_.retry_policy); if (options_.compression == "gzip") { diff --git a/exporters/otlp/src/otlp_http_exporter.cc b/exporters/otlp/src/otlp_http_exporter.cc index 95383d164a..2ebb487afd 100644 --- a/exporters/otlp/src/otlp_http_exporter.cc +++ b/exporters/otlp/src/otlp_http_exporter.cc @@ -15,6 +15,7 @@ #include "opentelemetry/exporters/otlp/otlp_http_exporter_runtime_options.h" #include "opentelemetry/exporters/otlp/otlp_recordable.h" #include "opentelemetry/exporters/otlp/otlp_recordable_utils.h" +#include "opentelemetry/ext/http/client/http_client.h" #include "opentelemetry/nostd/span.h" #include "opentelemetry/nostd/string_view.h" #include "opentelemetry/sdk/common/exporter_utils.h" @@ -61,6 +62,10 @@ OtlpHttpExporter::OtlpHttpExporter(const OtlpHttpExporterOptions &options) options.console_debug, options.timeout, options.http_headers, + options.retry_policy_max_attempts, + options.retry_policy_initial_backoff, + options.retry_policy_max_backoff, + options.retry_policy_backoff_multiplier, std::shared_ptr{nullptr} #ifdef ENABLE_ASYNC_EXPORT , @@ -93,6 +98,10 @@ OtlpHttpExporter::OtlpHttpExporter(const OtlpHttpExporterOptions &options, options.console_debug, options.timeout, options.http_headers, + options.retry_policy_max_attempts, + options.retry_policy_initial_backoff, + options.retry_policy_max_backoff, + options.retry_policy_backoff_multiplier, runtime_options.thread_instrumentation #ifdef ENABLE_ASYNC_EXPORT , @@ -105,13 +114,18 @@ OtlpHttpExporter::OtlpHttpExporter(const OtlpHttpExporterOptions &options, OtlpHttpExporter::OtlpHttpExporter(std::unique_ptr http_client) : options_(OtlpHttpExporterOptions()), http_client_(std::move(http_client)) { - options_.url = http_client_->GetOptions().url; - options_.content_type = http_client_->GetOptions().content_type; - options_.json_bytes_mapping = http_client_->GetOptions().json_bytes_mapping; - options_.use_json_name = http_client_->GetOptions().use_json_name; - options_.console_debug = http_client_->GetOptions().console_debug; - options_.timeout = http_client_->GetOptions().timeout; - options_.http_headers = http_client_->GetOptions().http_headers; + options_.url = http_client_->GetOptions().url; + options_.content_type = http_client_->GetOptions().content_type; + options_.json_bytes_mapping = http_client_->GetOptions().json_bytes_mapping; + options_.use_json_name = http_client_->GetOptions().use_json_name; + options_.console_debug = http_client_->GetOptions().console_debug; + options_.timeout = http_client_->GetOptions().timeout; + options_.http_headers = http_client_->GetOptions().http_headers; + options_.retry_policy_max_attempts = http_client_->GetOptions().retry_policy.max_attempts; + options_.retry_policy_initial_backoff = http_client_->GetOptions().retry_policy.initial_backoff; + options_.retry_policy_max_backoff = http_client_->GetOptions().retry_policy.max_backoff; + options_.retry_policy_backoff_multiplier = + http_client_->GetOptions().retry_policy.backoff_multiplier; #ifdef ENABLE_ASYNC_EXPORT options_.max_concurrent_requests = http_client_->GetOptions().max_concurrent_requests; options_.max_requests_per_connection = http_client_->GetOptions().max_requests_per_connection; diff --git a/exporters/otlp/src/otlp_http_exporter_options.cc b/exporters/otlp/src/otlp_http_exporter_options.cc index f6f7027f50..34ba5e173a 100644 --- a/exporters/otlp/src/otlp_http_exporter_options.cc +++ b/exporters/otlp/src/otlp_http_exporter_options.cc @@ -31,7 +31,11 @@ OtlpHttpExporterOptions::OtlpHttpExporterOptions() ssl_max_tls(GetOtlpDefaultTracesSslTlsMaxVersion()), ssl_cipher(GetOtlpDefaultTracesSslTlsCipher()), ssl_cipher_suite(GetOtlpDefaultTracesSslTlsCipherSuite()), - compression(GetOtlpDefaultTracesCompression()) + compression(GetOtlpDefaultTracesCompression()), + retry_policy_max_attempts(GetOtlpDefaultTracesRetryMaxAttempts()), + retry_policy_initial_backoff(GetOtlpDefaultTracesRetryInitialBackoff()), + retry_policy_max_backoff(GetOtlpDefaultTracesRetryMaxBackoff()), + retry_policy_backoff_multiplier(GetOtlpDefaultTracesRetryBackoffMultiplier()) { #ifdef ENABLE_ASYNC_EXPORT max_concurrent_requests = 64; diff --git a/exporters/otlp/src/otlp_http_log_record_exporter.cc b/exporters/otlp/src/otlp_http_log_record_exporter.cc index 98865fdfcd..cd9852e05a 100644 --- a/exporters/otlp/src/otlp_http_log_record_exporter.cc +++ b/exporters/otlp/src/otlp_http_log_record_exporter.cc @@ -15,6 +15,7 @@ #include "opentelemetry/exporters/otlp/otlp_http_log_record_exporter_runtime_options.h" #include "opentelemetry/exporters/otlp/otlp_log_recordable.h" #include "opentelemetry/exporters/otlp/otlp_recordable_utils.h" +#include "opentelemetry/ext/http/client/http_client.h" #include "opentelemetry/nostd/span.h" #include "opentelemetry/nostd/string_view.h" #include "opentelemetry/sdk/common/exporter_utils.h" @@ -64,6 +65,10 @@ OtlpHttpLogRecordExporter::OtlpHttpLogRecordExporter( options.console_debug, options.timeout, options.http_headers, + options.retry_policy_max_attempts, + options.retry_policy_initial_backoff, + options.retry_policy_max_backoff, + options.retry_policy_backoff_multiplier, std::shared_ptr{nullptr} #ifdef ENABLE_ASYNC_EXPORT , @@ -97,6 +102,10 @@ OtlpHttpLogRecordExporter::OtlpHttpLogRecordExporter( options.console_debug, options.timeout, options.http_headers, + options.retry_policy_max_attempts, + options.retry_policy_initial_backoff, + options.retry_policy_max_backoff, + options.retry_policy_backoff_multiplier, runtime_options.thread_instrumentation #ifdef ENABLE_ASYNC_EXPORT , @@ -109,13 +118,18 @@ OtlpHttpLogRecordExporter::OtlpHttpLogRecordExporter( OtlpHttpLogRecordExporter::OtlpHttpLogRecordExporter(std::unique_ptr http_client) : options_(OtlpHttpLogRecordExporterOptions()), http_client_(std::move(http_client)) { - options_.url = http_client_->GetOptions().url; - options_.content_type = http_client_->GetOptions().content_type; - options_.json_bytes_mapping = http_client_->GetOptions().json_bytes_mapping; - options_.use_json_name = http_client_->GetOptions().use_json_name; - options_.console_debug = http_client_->GetOptions().console_debug; - options_.timeout = http_client_->GetOptions().timeout; - options_.http_headers = http_client_->GetOptions().http_headers; + options_.url = http_client_->GetOptions().url; + options_.content_type = http_client_->GetOptions().content_type; + options_.json_bytes_mapping = http_client_->GetOptions().json_bytes_mapping; + options_.use_json_name = http_client_->GetOptions().use_json_name; + options_.console_debug = http_client_->GetOptions().console_debug; + options_.timeout = http_client_->GetOptions().timeout; + options_.http_headers = http_client_->GetOptions().http_headers; + options_.retry_policy_max_attempts = http_client_->GetOptions().retry_policy.max_attempts; + options_.retry_policy_initial_backoff = http_client_->GetOptions().retry_policy.initial_backoff; + options_.retry_policy_max_backoff = http_client_->GetOptions().retry_policy.max_backoff; + options_.retry_policy_backoff_multiplier = + http_client_->GetOptions().retry_policy.backoff_multiplier; #ifdef ENABLE_ASYNC_EXPORT options_.max_concurrent_requests = http_client_->GetOptions().max_concurrent_requests; options_.max_requests_per_connection = http_client_->GetOptions().max_requests_per_connection; diff --git a/exporters/otlp/src/otlp_http_log_record_exporter_options.cc b/exporters/otlp/src/otlp_http_log_record_exporter_options.cc index b38a292476..d3c6742cb2 100644 --- a/exporters/otlp/src/otlp_http_log_record_exporter_options.cc +++ b/exporters/otlp/src/otlp_http_log_record_exporter_options.cc @@ -31,7 +31,11 @@ OtlpHttpLogRecordExporterOptions::OtlpHttpLogRecordExporterOptions() ssl_max_tls(GetOtlpDefaultLogsSslTlsMaxVersion()), ssl_cipher(GetOtlpDefaultLogsSslTlsCipher()), ssl_cipher_suite(GetOtlpDefaultLogsSslTlsCipherSuite()), - compression(GetOtlpDefaultLogsCompression()) + compression(GetOtlpDefaultLogsCompression()), + retry_policy_max_attempts(GetOtlpDefaultLogsRetryMaxAttempts()), + retry_policy_initial_backoff(GetOtlpDefaultLogsRetryInitialBackoff()), + retry_policy_max_backoff(GetOtlpDefaultLogsRetryMaxBackoff()), + retry_policy_backoff_multiplier(GetOtlpDefaultLogsRetryBackoffMultiplier()) { #ifdef ENABLE_ASYNC_EXPORT max_concurrent_requests = 64; diff --git a/exporters/otlp/src/otlp_http_metric_exporter.cc b/exporters/otlp/src/otlp_http_metric_exporter.cc index f59138f981..5c2bc76834 100644 --- a/exporters/otlp/src/otlp_http_metric_exporter.cc +++ b/exporters/otlp/src/otlp_http_metric_exporter.cc @@ -15,6 +15,7 @@ #include "opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h" #include "opentelemetry/exporters/otlp/otlp_http_metric_exporter_runtime_options.h" #include "opentelemetry/exporters/otlp/otlp_metric_utils.h" +#include "opentelemetry/ext/http/client/http_client.h" #include "opentelemetry/nostd/string_view.h" #include "opentelemetry/sdk/common/exporter_utils.h" #include "opentelemetry/sdk/common/global_log_handler.h" @@ -65,6 +66,10 @@ OtlpHttpMetricExporter::OtlpHttpMetricExporter(const OtlpHttpMetricExporterOptio options.console_debug, options.timeout, options.http_headers, + options.retry_policy_max_attempts, + options.retry_policy_initial_backoff, + options.retry_policy_max_backoff, + options.retry_policy_backoff_multiplier, std::shared_ptr{nullptr} #ifdef ENABLE_ASYNC_EXPORT , @@ -100,6 +105,10 @@ OtlpHttpMetricExporter::OtlpHttpMetricExporter( options.console_debug, options.timeout, options.http_headers, + options.retry_policy_max_attempts, + options.retry_policy_initial_backoff, + options.retry_policy_max_backoff, + options.retry_policy_backoff_multiplier, runtime_options.thread_instrumentation #ifdef ENABLE_ASYNC_EXPORT , @@ -115,13 +124,18 @@ OtlpHttpMetricExporter::OtlpHttpMetricExporter(std::unique_ptr h OtlpMetricUtils::ChooseTemporalitySelector(options_.aggregation_temporality)}, http_client_(std::move(http_client)) { - options_.url = http_client_->GetOptions().url; - options_.content_type = http_client_->GetOptions().content_type; - options_.json_bytes_mapping = http_client_->GetOptions().json_bytes_mapping; - options_.use_json_name = http_client_->GetOptions().use_json_name; - options_.console_debug = http_client_->GetOptions().console_debug; - options_.timeout = http_client_->GetOptions().timeout; - options_.http_headers = http_client_->GetOptions().http_headers; + options_.url = http_client_->GetOptions().url; + options_.content_type = http_client_->GetOptions().content_type; + options_.json_bytes_mapping = http_client_->GetOptions().json_bytes_mapping; + options_.use_json_name = http_client_->GetOptions().use_json_name; + options_.console_debug = http_client_->GetOptions().console_debug; + options_.timeout = http_client_->GetOptions().timeout; + options_.http_headers = http_client_->GetOptions().http_headers; + options_.retry_policy_max_attempts = http_client_->GetOptions().retry_policy.max_attempts; + options_.retry_policy_initial_backoff = http_client_->GetOptions().retry_policy.initial_backoff; + options_.retry_policy_max_backoff = http_client_->GetOptions().retry_policy.max_backoff; + options_.retry_policy_backoff_multiplier = + http_client_->GetOptions().retry_policy.backoff_multiplier; #ifdef ENABLE_ASYNC_EXPORT options_.max_concurrent_requests = http_client_->GetOptions().max_concurrent_requests; options_.max_requests_per_connection = http_client_->GetOptions().max_requests_per_connection; diff --git a/exporters/otlp/src/otlp_http_metric_exporter_options.cc b/exporters/otlp/src/otlp_http_metric_exporter_options.cc index b5fb83b090..eb11fe2a88 100644 --- a/exporters/otlp/src/otlp_http_metric_exporter_options.cc +++ b/exporters/otlp/src/otlp_http_metric_exporter_options.cc @@ -33,7 +33,11 @@ OtlpHttpMetricExporterOptions::OtlpHttpMetricExporterOptions() ssl_max_tls(GetOtlpDefaultMetricsSslTlsMaxVersion()), ssl_cipher(GetOtlpDefaultMetricsSslTlsCipher()), ssl_cipher_suite(GetOtlpDefaultMetricsSslTlsCipherSuite()), - compression(GetOtlpDefaultMetricsCompression()) + compression(GetOtlpDefaultMetricsCompression()), + retry_policy_max_attempts(GetOtlpDefaultMetricsRetryMaxAttempts()), + retry_policy_initial_backoff(GetOtlpDefaultMetricsRetryInitialBackoff()), + retry_policy_max_backoff(GetOtlpDefaultMetricsRetryMaxBackoff()), + retry_policy_backoff_multiplier(GetOtlpDefaultMetricsRetryBackoffMultiplier()) { #ifdef ENABLE_ASYNC_EXPORT max_concurrent_requests = 64; diff --git a/exporters/otlp/test/otlp_http_exporter_test.cc b/exporters/otlp/test/otlp_http_exporter_test.cc index 0824a59a06..3cb081ba98 100644 --- a/exporters/otlp/test/otlp_http_exporter_test.cc +++ b/exporters/otlp/test/otlp_http_exporter_test.cc @@ -7,6 +7,7 @@ # include # include "opentelemetry/exporters/otlp/otlp_http_exporter.h" +# include "opentelemetry/exporters/otlp/otlp_http_exporter_factory.h" # include "opentelemetry/exporters/otlp/protobuf_include_prefix.h" @@ -18,10 +19,14 @@ # include "opentelemetry/ext/http/server/http_server.h" # include "opentelemetry/sdk/trace/batch_span_processor.h" # include "opentelemetry/sdk/trace/batch_span_processor_options.h" +# include "opentelemetry/sdk/trace/simple_processor.h" +# include "opentelemetry/sdk/trace/simple_processor_factory.h" # include "opentelemetry/sdk/trace/tracer_provider.h" +# include "opentelemetry/sdk/trace/tracer_provider_factory.h" # include "opentelemetry/test_common/ext/http/client/http_client_test_factory.h" # include "opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h" # include "opentelemetry/trace/provider.h" +# include "opentelemetry/trace/tracer_provider.h" # include # include @@ -60,18 +65,26 @@ OtlpHttpClientOptions MakeOtlpHttpClientOptions(HttpRequestContentType content_t options.console_debug = true; options.timeout = std::chrono::system_clock::duration::zero(); options.http_headers.insert(std::make_pair("Custom-Header-Key", "Custom-Header-Value")); + options.retry_policy_max_attempts = 0U; + options.retry_policy_initial_backoff = std::chrono::duration::zero(); + options.retry_policy_max_backoff = std::chrono::duration::zero(); + options.retry_policy_backoff_multiplier = 0.0f; OtlpHttpClientOptions otlp_http_client_options( - options.url, false, /* ssl_insecure_skip_verify */ - "", /* ssl_ca_cert_path */ "", /* ssl_ca_cert_string */ - "", /* ssl_client_key_path */ - "", /* ssl_client_key_string */ "", /* ssl_client_cert_path */ - "", /* ssl_client_cert_string */ - "", /* ssl_min_tls */ - "", /* ssl_max_tls */ - "", /* ssl_cipher */ - "", /* ssl_cipher_suite */ + options.url, false, /* ssl_insecure_skip_verify */ + "", /* ssl_ca_cert_path */ + "", /* ssl_ca_cert_string */ + "", /* ssl_client_key_path */ + "", /* ssl_client_key_string */ + "", /* ssl_client_cert_path */ + "", /* ssl_client_cert_string */ + "", /* ssl_min_tls */ + "", /* ssl_max_tls */ + "", /* ssl_cipher */ + "", /* ssl_cipher_suite */ options.content_type, options.json_bytes_mapping, options.compression, options.use_json_name, - options.console_debug, options.timeout, options.http_headers, not_instrumented); + options.console_debug, options.timeout, options.http_headers, + options.retry_policy_max_attempts, options.retry_policy_initial_backoff, + options.retry_policy_max_backoff, options.retry_policy_backoff_multiplier, not_instrumented); if (!async_mode) { otlp_http_client_options.max_concurrent_requests = 0; @@ -621,7 +634,162 @@ TEST_F(OtlpHttpExporterTestPeer, ConfigFromTracesEnv) unsetenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS"); unsetenv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"); } -# endif + +TEST_F(OtlpHttpExporterTestPeer, ConfigRetryDefaultValues) +{ + std::unique_ptr exporter(new OtlpHttpExporter()); + const auto options = GetOptions(exporter); + ASSERT_EQ(options.retry_policy_max_attempts, 5); + ASSERT_FLOAT_EQ(options.retry_policy_initial_backoff.count(), 1.0); + ASSERT_FLOAT_EQ(options.retry_policy_max_backoff.count(), 5.0); + ASSERT_FLOAT_EQ(options.retry_policy_backoff_multiplier, 1.5); +} + +TEST_F(OtlpHttpExporterTestPeer, ConfigRetryValuesFromEnv) +{ + setenv("OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_MAX_ATTEMPTS", "123", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_INITIAL_BACKOFF", "4.5", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_MAX_BACKOFF", "6.7", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_BACKOFF_MULTIPLIER", "8.9", 1); + + std::unique_ptr exporter(new OtlpHttpExporter()); + const auto options = GetOptions(exporter); + ASSERT_EQ(options.retry_policy_max_attempts, 123); + ASSERT_FLOAT_EQ(options.retry_policy_initial_backoff.count(), 4.5); + ASSERT_FLOAT_EQ(options.retry_policy_max_backoff.count(), 6.7); + ASSERT_FLOAT_EQ(options.retry_policy_backoff_multiplier, 8.9); + + unsetenv("OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_MAX_ATTEMPTS"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_INITIAL_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_MAX_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_TRACES_RETRY_BACKOFF_MULTIPLIER"); +} + +TEST_F(OtlpHttpExporterTestPeer, ConfigRetryGenericValuesFromEnv) +{ + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS", "321", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF", "5.4", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF", "7.6", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER", "9.8", 1); + + std::unique_ptr exporter(new OtlpHttpExporter()); + const auto options = GetOptions(exporter); + ASSERT_EQ(options.retry_policy_max_attempts, 321); + ASSERT_FLOAT_EQ(options.retry_policy_initial_backoff.count(), 5.4); + ASSERT_FLOAT_EQ(options.retry_policy_max_backoff.count(), 7.6); + ASSERT_FLOAT_EQ(options.retry_policy_backoff_multiplier, 9.8); + + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER"); +} +# endif // NO_GETENV + +# ifdef ENABLE_OTLP_RETRY_PREVIEW +using StatusCodeVector = std::vector; + +class OtlpHttpExporterRetryIntegrationTests + : public ::testing::TestWithParam> +{}; + +INSTANTIATE_TEST_SUITE_P(StatusCodes, + OtlpHttpExporterRetryIntegrationTests, + testing::Values( + // With retry policy enabled + std::make_tuple(true, StatusCodeVector{100}, 1), + std::make_tuple(true, StatusCodeVector{200}, 1), + std::make_tuple(true, StatusCodeVector{201}, 1), + std::make_tuple(true, StatusCodeVector{202}, 1), + std::make_tuple(true, StatusCodeVector{204}, 1), + std::make_tuple(true, StatusCodeVector{302}, 1), + std::make_tuple(true, StatusCodeVector{400}, 1), + std::make_tuple(true, StatusCodeVector{401}, 1), + std::make_tuple(true, StatusCodeVector{403}, 1), + std::make_tuple(true, StatusCodeVector{404}, 1), + std::make_tuple(true, StatusCodeVector{405}, 1), + std::make_tuple(true, StatusCodeVector{429}, 5), + std::make_tuple(true, StatusCodeVector{500}, 1), + std::make_tuple(true, StatusCodeVector{501}, 1), + std::make_tuple(true, StatusCodeVector{502}, 5), + std::make_tuple(true, StatusCodeVector{503}, 5), + std::make_tuple(true, StatusCodeVector{504}, 5), + std::make_tuple(true, StatusCodeVector{429, 502, 503, 504}, 5), + std::make_tuple(true, StatusCodeVector{503, 503, 503, 200}, 4), + std::make_tuple(true, StatusCodeVector{429, 503, 504, 200}, 4), + // With retry policy disabled + std::make_tuple(false, StatusCodeVector{100}, 1), + std::make_tuple(false, StatusCodeVector{200}, 1), + std::make_tuple(false, StatusCodeVector{201}, 1), + std::make_tuple(false, StatusCodeVector{202}, 1), + std::make_tuple(false, StatusCodeVector{204}, 1), + std::make_tuple(false, StatusCodeVector{302}, 1), + std::make_tuple(false, StatusCodeVector{400}, 1), + std::make_tuple(false, StatusCodeVector{401}, 1), + std::make_tuple(false, StatusCodeVector{403}, 1), + std::make_tuple(false, StatusCodeVector{404}, 1), + std::make_tuple(false, StatusCodeVector{405}, 1), + std::make_tuple(false, StatusCodeVector{429}, 1), + std::make_tuple(false, StatusCodeVector{500}, 1), + std::make_tuple(false, StatusCodeVector{501}, 1), + std::make_tuple(false, StatusCodeVector{502}, 1), + std::make_tuple(false, StatusCodeVector{503}, 1), + std::make_tuple(false, StatusCodeVector{504}, 1), + std::make_tuple(false, StatusCodeVector{429, 502, 503, 504}, 1), + std::make_tuple(false, StatusCodeVector{503, 503, 503, 200}, 1), + std::make_tuple(false, StatusCodeVector{429, 503, 504, 200}, 1))); + +TEST_P(OtlpHttpExporterRetryIntegrationTests, StatusCodes) +{ + namespace otlp = opentelemetry::exporter::otlp; + namespace trace_sdk = opentelemetry::sdk::trace; + + const auto is_retry_enabled = std::get<0>(GetParam()); + const auto status_codes = std::get<1>(GetParam()); + const auto expected_attempts = std::get<2>(GetParam()); + + size_t request_count = 0UL; + HTTP_SERVER_NS::HttpRequestCallback request_handler{ + [&request_count, &status_codes](HTTP_SERVER_NS::HttpRequest const & /* request */, + HTTP_SERVER_NS::HttpResponse &response) { + response.body = "TEST!"; + response.code = status_codes.at(request_count++ % status_codes.size()); + return response.code; + }}; + HTTP_SERVER_NS::HttpServer server; + server.setKeepalive(true); + server.setServerName("test_server"); + server.addHandler("/v1/traces", request_handler); + ASSERT_EQ(server.addListeningPort(4318), 4318); + server.start(); + + otlp::OtlpHttpExporterOptions opts{}; + + if (is_retry_enabled) + { + opts.retry_policy_max_attempts = 5; + opts.retry_policy_initial_backoff = std::chrono::duration{0.1f}; + opts.retry_policy_max_backoff = std::chrono::duration{5.0f}; + opts.retry_policy_backoff_multiplier = 1.0f; + } + else + { + opts.retry_policy_max_attempts = 0; + opts.retry_policy_initial_backoff = std::chrono::duration::zero(); + opts.retry_policy_max_backoff = std::chrono::duration::zero(); + opts.retry_policy_backoff_multiplier = 0.0f; + } + + auto exporter = otlp::OtlpHttpExporterFactory::Create(opts); + auto processor = trace_sdk::SimpleSpanProcessorFactory::Create(std::move(exporter)); + auto provider = trace_sdk::TracerProviderFactory::Create(std::move(processor)); + provider->GetTracer("Test tracer")->StartSpan("Test span")->End(); + provider->ForceFlush(); + server.stop(); + + ASSERT_EQ(expected_attempts, request_count); +} +# endif // ENABLE_OTLP_RETRY_PREVIEW } // namespace otlp } // namespace exporter diff --git a/exporters/otlp/test/otlp_http_log_record_exporter_test.cc b/exporters/otlp/test/otlp_http_log_record_exporter_test.cc index 606f162024..20e2aef71c 100644 --- a/exporters/otlp/test/otlp_http_log_record_exporter_test.cc +++ b/exporters/otlp/test/otlp_http_log_record_exporter_test.cc @@ -59,18 +59,26 @@ OtlpHttpClientOptions MakeOtlpHttpClientOptions(HttpRequestContentType content_t options.content_type = content_type; options.console_debug = true; options.http_headers.insert(std::make_pair("Custom-Header-Key", "Custom-Header-Value")); + options.retry_policy_max_attempts = 0U; + options.retry_policy_initial_backoff = std::chrono::duration::zero(); + options.retry_policy_max_backoff = std::chrono::duration::zero(); + options.retry_policy_backoff_multiplier = 0.0f; OtlpHttpClientOptions otlp_http_client_options( - options.url, false, /* ssl_insecure_skip_verify */ - "", /* ssl_ca_cert_path */ "", /* ssl_ca_cert_string */ - "", /* ssl_client_key_path */ - "", /* ssl_client_key_string */ "", /* ssl_client_cert_path */ - "", /* ssl_client_cert_string */ - "", /* ssl_min_tls */ - "", /* ssl_max_tls */ - "", /* ssl_cipher */ - "", /* ssl_cipher_suite */ + options.url, false, /* ssl_insecure_skip_verify */ + "", /* ssl_ca_cert_path */ + "", /* ssl_ca_cert_string */ + "", /* ssl_client_key_path */ + "", /* ssl_client_key_string */ + "", /* ssl_client_cert_path */ + "", /* ssl_client_cert_string */ + "", /* ssl_min_tls */ + "", /* ssl_max_tls */ + "", /* ssl_cipher */ + "", /* ssl_cipher_suite */ options.content_type, options.json_bytes_mapping, options.compression, options.use_json_name, - options.console_debug, options.timeout, options.http_headers, not_instrumented); + options.console_debug, options.timeout, options.http_headers, + options.retry_policy_max_attempts, options.retry_policy_initial_backoff, + options.retry_policy_max_backoff, options.retry_policy_backoff_multiplier, not_instrumented); if (!async_mode) { otlp_http_client_options.max_concurrent_requests = 0; @@ -749,7 +757,56 @@ TEST_F(OtlpHttpLogRecordExporterTestPeer, DefaultEndpoint) EXPECT_EQ("http://localhost:4317", GetOtlpDefaultGrpcEndpoint()); } -# endif +TEST_F(OtlpHttpLogRecordExporterTestPeer, ConfigRetryDefaultValues) +{ + std::unique_ptr exporter(new OtlpHttpLogRecordExporter()); + const auto options = GetOptions(exporter); + ASSERT_EQ(options.retry_policy_max_attempts, 5); + ASSERT_FLOAT_EQ(options.retry_policy_initial_backoff.count(), 1.0); + ASSERT_FLOAT_EQ(options.retry_policy_max_backoff.count(), 5.0); + ASSERT_FLOAT_EQ(options.retry_policy_backoff_multiplier, 1.5); +} + +TEST_F(OtlpHttpLogRecordExporterTestPeer, ConfigRetryValuesFromEnv) +{ + setenv("OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_MAX_ATTEMPTS", "123", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_INITIAL_BACKOFF", "4.5", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_MAX_BACKOFF", "6.7", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_BACKOFF_MULTIPLIER", "8.9", 1); + + std::unique_ptr exporter(new OtlpHttpLogRecordExporter()); + const auto options = GetOptions(exporter); + ASSERT_EQ(options.retry_policy_max_attempts, 123); + ASSERT_FLOAT_EQ(options.retry_policy_initial_backoff.count(), 4.5); + ASSERT_FLOAT_EQ(options.retry_policy_max_backoff.count(), 6.7); + ASSERT_FLOAT_EQ(options.retry_policy_backoff_multiplier, 8.9); + + unsetenv("OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_MAX_ATTEMPTS"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_INITIAL_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_MAX_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_LOGS_RETRY_BACKOFF_MULTIPLIER"); +} + +TEST_F(OtlpHttpLogRecordExporterTestPeer, ConfigRetryGenericValuesFromEnv) +{ + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS", "321", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF", "5.4", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF", "7.6", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER", "9.8", 1); + + std::unique_ptr exporter(new OtlpHttpLogRecordExporter()); + const auto options = GetOptions(exporter); + ASSERT_EQ(options.retry_policy_max_attempts, 321); + ASSERT_FLOAT_EQ(options.retry_policy_initial_backoff.count(), 5.4); + ASSERT_FLOAT_EQ(options.retry_policy_max_backoff.count(), 7.6); + ASSERT_FLOAT_EQ(options.retry_policy_backoff_multiplier, 9.8); + + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER"); +} +# endif // NO_GETENV } // namespace otlp } // namespace exporter diff --git a/exporters/otlp/test/otlp_http_metric_exporter_test.cc b/exporters/otlp/test/otlp_http_metric_exporter_test.cc index 420946ff70..237f59ccba 100644 --- a/exporters/otlp/test/otlp_http_metric_exporter_test.cc +++ b/exporters/otlp/test/otlp_http_metric_exporter_test.cc @@ -80,18 +80,26 @@ OtlpHttpClientOptions MakeOtlpHttpClientOptions(HttpRequestContentType content_t options.content_type = content_type; options.console_debug = true; options.http_headers.insert(std::make_pair("Custom-Header-Key", "Custom-Header-Value")); + options.retry_policy_max_attempts = 0U; + options.retry_policy_initial_backoff = std::chrono::duration::zero(); + options.retry_policy_max_backoff = std::chrono::duration::zero(); + options.retry_policy_backoff_multiplier = 0.0f; OtlpHttpClientOptions otlp_http_client_options( - options.url, false, /* ssl_insecure_skip_verify */ - "", /* ssl_ca_cert_path */ "", /* ssl_ca_cert_string */ - "", /* ssl_client_key_path */ - "", /* ssl_client_key_string */ "", /* ssl_client_cert_path */ - "", /* ssl_client_cert_string */ - "", /* ssl_min_tls */ - "", /* ssl_max_tls */ - "", /* ssl_cipher */ - "", /* ssl_cipher_suite */ + options.url, false, /* ssl_insecure_skip_verify */ + "", /* ssl_ca_cert_path */ + "", /* ssl_ca_cert_string */ + "", /* ssl_client_key_path */ + "", /* ssl_client_key_string */ + "", /* ssl_client_cert_path */ + "", /* ssl_client_cert_string */ + "", /* ssl_min_tls */ + "", /* ssl_max_tls */ + "", /* ssl_cipher */ + "", /* ssl_cipher_suite */ options.content_type, options.json_bytes_mapping, options.compression, options.use_json_name, - options.console_debug, options.timeout, options.http_headers, not_instrumented); + options.console_debug, options.timeout, options.http_headers, + options.retry_policy_max_attempts, options.retry_policy_initial_backoff, + options.retry_policy_max_backoff, options.retry_policy_backoff_multiplier, not_instrumented); if (!async_mode) { otlp_http_client_options.max_concurrent_requests = 0; @@ -1003,7 +1011,57 @@ TEST_F(OtlpHttpMetricExporterTestPeer, CheckDefaultTemporality) exporter->GetAggregationTemporality( opentelemetry::sdk::metrics::InstrumentType::kObservableUpDownCounter)); } -#endif + +TEST_F(OtlpHttpMetricExporterTestPeer, ConfigRetryDefaultValues) +{ + std::unique_ptr exporter(new OtlpHttpMetricExporter()); + const auto options = GetOptions(exporter); + ASSERT_EQ(options.retry_policy_max_attempts, 5); + ASSERT_FLOAT_EQ(options.retry_policy_initial_backoff.count(), 1.0); + ASSERT_FLOAT_EQ(options.retry_policy_max_backoff.count(), 5.0); + ASSERT_FLOAT_EQ(options.retry_policy_backoff_multiplier, 1.5); +} + +TEST_F(OtlpHttpMetricExporterTestPeer, ConfigRetryValuesFromEnv) +{ + setenv("OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_MAX_ATTEMPTS", "123", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_INITIAL_BACKOFF", "4.5", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_MAX_BACKOFF", "6.7", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_BACKOFF_MULTIPLIER", "8.9", 1); + + std::unique_ptr exporter(new OtlpHttpMetricExporter()); + const auto options = GetOptions(exporter); + ASSERT_EQ(options.retry_policy_max_attempts, 123); + ASSERT_FLOAT_EQ(options.retry_policy_initial_backoff.count(), 4.5); + ASSERT_FLOAT_EQ(options.retry_policy_max_backoff.count(), 6.7); + ASSERT_FLOAT_EQ(options.retry_policy_backoff_multiplier, 8.9); + + unsetenv("OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_MAX_ATTEMPTS"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_INITIAL_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_MAX_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_METRICS_RETRY_BACKOFF_MULTIPLIER"); +} + +TEST_F(OtlpHttpMetricExporterTestPeer, ConfigRetryGenericValuesFromEnv) +{ + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS", "321", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF", "5.4", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF", "7.6", 1); + setenv("OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER", "9.8", 1); + + std::unique_ptr exporter(new OtlpHttpMetricExporter()); + const auto options = GetOptions(exporter); + ASSERT_EQ(options.retry_policy_max_attempts, 321); + ASSERT_FLOAT_EQ(options.retry_policy_initial_backoff.count(), 5.4); + ASSERT_FLOAT_EQ(options.retry_policy_max_backoff.count(), 7.6); + ASSERT_FLOAT_EQ(options.retry_policy_backoff_multiplier, 9.8); + + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_ATTEMPTS"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_INITIAL_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_MAX_BACKOFF"); + unsetenv("OTEL_CPP_EXPORTER_OTLP_RETRY_BACKOFF_MULTIPLIER"); +} +#endif // NO_GETENV // Test Preferred aggregtion temporality selection TEST_F(OtlpHttpMetricExporterTestPeer, PreferredAggergationTemporality) diff --git a/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h b/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h index cd88c9f7c0..77bd6bf89c 100644 --- a/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h +++ b/ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -105,6 +106,12 @@ class Request : public opentelemetry::ext::http::client::Request void EnableLogging(bool is_log_enabled) noexcept override { is_log_enabled_ = is_log_enabled; } + void SetRetryPolicy( + const opentelemetry::ext::http::client::RetryPolicy &retry_policy) noexcept override + { + retry_policy_ = retry_policy; + } + public: opentelemetry::ext::http::client::Method method_; opentelemetry::ext::http::client::HttpSslOptions ssl_options_; @@ -115,6 +122,7 @@ class Request : public opentelemetry::ext::http::client::Request opentelemetry::ext::http::client::Compression compression_{ opentelemetry::ext::http::client::Compression::kNone}; bool is_log_enabled_{false}; + opentelemetry::ext::http::client::RetryPolicy retry_policy_; }; class Response : public opentelemetry::ext::http::client::Response @@ -340,6 +348,7 @@ class HttpClient : public opentelemetry::ext::http::client::HttpClient bool doAddSessions(); bool doAbortSessions(); bool doRemoveSessions(); + bool doRetrySessions(bool report_all); void resetMultiHandle(); std::mutex multi_handle_m_; @@ -354,6 +363,7 @@ class HttpClient : public opentelemetry::ext::http::client::HttpClient std::unordered_map> pending_to_abort_sessions_; std::unordered_map pending_to_remove_session_handles_; std::list> pending_to_remove_sessions_; + std::deque> pending_to_retry_sessions_; std::mutex background_thread_m_; std::unique_ptr background_thread_; diff --git a/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h b/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h index b94c53b2d0..2c3565177c 100644 --- a/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h +++ b/ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h @@ -13,6 +13,7 @@ # include #endif +#include #include #include #include @@ -38,9 +39,9 @@ namespace client { namespace curl { -const std::chrono::milliseconds default_http_conn_timeout(5000); // ms -const std::string http_status_regexp = "HTTP\\/\\d\\.\\d (\\d+)\\ .*"; -const std::string http_header_regexp = "(.*)\\: (.*)\\n*"; +const std::chrono::milliseconds kDefaultHttpConnTimeout(5000); // ms +const std::string kHttpStatusRegexp = "HTTP\\/\\d\\.\\d (\\d+)\\ .*"; +const std::string kHttpHeaderRegexp = "(.*)\\: (.*)\\n*"; class HttpClient; class Session; @@ -135,14 +136,17 @@ class HttpOperation /** * Create local CURL instance for url and body - * @param method // HTTP Method - * @param url // HTTP URL + * @param method HTTP Method + * @param url HTTP URL * @param callback - * @param request_mode // sync or async - * @param request Request Headers - * @param body Reques Body - * @param raw_response whether to parse the response - * @param httpConnTimeout HTTP connection timeout in seconds + * @param request_mode Sync or async + * @param request Request Headers + * @param body Request Body + * @param raw_response Whether to parse the response + * @param http_conn_timeout HTTP connection timeout in seconds + * @param reuse_connection Whether connection should be reused or closed + * @param is_log_enabled To intercept some information from cURL request + * @param retry_policy Retry policy for select failure status codes */ HttpOperation(opentelemetry::ext::http::client::Method method, std::string url, @@ -157,9 +161,10 @@ class HttpOperation opentelemetry::ext::http::client::Compression::kNone, // Default connectivity and response size options bool is_raw_response = false, - std::chrono::milliseconds http_conn_timeout = default_http_conn_timeout, + std::chrono::milliseconds http_conn_timeout = kDefaultHttpConnTimeout, bool reuse_connection = false, - bool is_log_enabled = false); + bool is_log_enabled = false, + const opentelemetry::ext::http::client::RetryPolicy &retry_policy = {}); /** * Destroy CURL instance @@ -176,6 +181,16 @@ class HttpOperation */ void Cleanup(); + /** + * Determine if operation is retryable + */ + bool IsRetryable(); + + /** + * Calculate next time to retry request + */ + std::chrono::system_clock::time_point NextRetryTime(); + /** * Setup request */ @@ -216,7 +231,7 @@ class HttpOperation bool WasAborted() { return is_aborted_.load(std::memory_order_acquire); } /** - * Return a copy of resposne headers + * Return a copy of response headers * * @return */ @@ -309,6 +324,10 @@ class HttpOperation const bool is_log_enabled_; + const RetryPolicy retry_policy_; + decltype(RetryPolicy::max_attempts) retry_attempts_; + std::chrono::system_clock::time_point last_attempt_time_; + // Processed response headers and body long response_code_; std::vector response_headers_; diff --git a/ext/include/opentelemetry/ext/http/client/http_client.h b/ext/include/opentelemetry/ext/http/client/http_client.h index e467f9ef63..36f4e2d0a6 100644 --- a/ext/include/opentelemetry/ext/http/client/http_client.h +++ b/ext/include/opentelemetry/ext/http/client/http_client.h @@ -226,6 +226,28 @@ struct HttpSslOptions std::string ssl_cipher_suite{}; }; +using SecondsDecimal = std::chrono::duration>; + +struct RetryPolicy +{ + RetryPolicy() = default; + + RetryPolicy(std::uint32_t input_max_attempts, + SecondsDecimal input_initial_backoff, + SecondsDecimal input_max_backoff, + float input_backoff_multiplier) + : max_attempts(input_max_attempts), + initial_backoff(input_initial_backoff), + max_backoff(input_max_backoff), + backoff_multiplier(input_backoff_multiplier) + {} + + std::uint32_t max_attempts{}; + SecondsDecimal initial_backoff{}; + SecondsDecimal max_backoff{}; + float backoff_multiplier{}; +}; + class Request { public: @@ -247,6 +269,8 @@ class Request virtual void EnableLogging(bool is_log_enabled) noexcept = 0; + virtual void SetRetryPolicy(const RetryPolicy &retry_policy) noexcept = 0; + virtual ~Request() = default; }; diff --git a/ext/src/http/client/curl/http_client_curl.cc b/ext/src/http/client/curl/http_client_curl.cc index b9f46f30bd..a65410d058 100644 --- a/ext/src/http/client/curl/http_client_curl.cc +++ b/ext/src/http/client/curl/http_client_curl.cc @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -198,10 +199,11 @@ void Session::SendRequest( #endif // ENABLE_OTLP_COMPRESSION_PREVIEW } - curl_operation_.reset(new HttpOperation( - http_request_->method_, url, http_request_->ssl_options_, callback_ptr, - http_request_->headers_, http_request_->body_, http_request_->compression_, false, - http_request_->timeout_ms_, reuse_connection, http_request_->is_log_enabled_)); + curl_operation_.reset( + new HttpOperation(http_request_->method_, url, http_request_->ssl_options_, callback_ptr, + http_request_->headers_, http_request_->body_, http_request_->compression_, + false, http_request_->timeout_ms_, reuse_connection, + http_request_->is_log_enabled_, http_request_->retry_policy_)); bool success = CURLE_OK == curl_operation_->SendAsync(this, [this, callback](HttpOperation &operation) { if (operation.WasAborted()) @@ -224,8 +226,8 @@ void Session::SendRequest( if (success) { - // We will try to create a background to poll events.But when the background is running, we will - // reuse it instead of creating a new one. + // We will try to create a background to poll events. But when the background is running, we + // will reuse it instead of creating a new one. http_client_.MaybeSpawnBackgroundThread(); } else @@ -333,7 +335,7 @@ std::shared_ptr HttpClient::CreateSes std::lock_guard lock_guard{sessions_m_}; sessions_.insert({session_id, session}); - // FIXME: Session may leak if it do not call SendRequest + // FIXME: Session may leak if it does not call SendRequest return session; } @@ -449,16 +451,14 @@ bool HttpClient::MaybeSpawnBackgroundThread() } #endif /* ENABLE_THREAD_INSTRUMENTATION_PREVIEW */ - int still_running = 1; - std::chrono::system_clock::time_point last_free_job_timepoint = - std::chrono::system_clock::now(); - bool need_wait_more = false; + auto still_running = 1; + auto last_free_job_timepoint = std::chrono::system_clock::now(); + auto need_wait_more = false; while (true) { - CURLMsg *msg; - int queued; + CURLMsg *msg = nullptr; + int queued = 0; CURLMcode mc = curl_multi_perform(self->multi_handle_, &still_running); - // According to https://curl.se/libcurl/c/curl_multi_perform.html, when mc is not OK, we // can not curl_multi_perform it again if (mc != CURLM_OK) @@ -474,7 +474,7 @@ bool HttpClient::MaybeSpawnBackgroundThread() } #endif /* ENABLE_THREAD_INSTRUMENTATION_PREVIEW */ - // curl_multi_poll is added from libcurl 7.66.0, before 7.68.0, we can only wait util + // curl_multi_poll is added from libcurl 7.66.0, before 7.68.0, we can only wait until // timeout to do the rest jobs #if LIBCURL_VERSION_NUM >= 0x074200 /* wait for activity, timeout or "nothing" */ @@ -509,13 +509,20 @@ bool HttpClient::MaybeSpawnBackgroundThread() CURLcode result = msg->data.result; Session *session = nullptr; curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &session); - // If it's already moved into pending_to_remove_session_handles_, we just ingore this + const auto operation = (nullptr != session) ? session->GetOperation().get() : nullptr; + + // If it's already moved into pending_to_remove_session_handles_, we just ignore this // message. - if (nullptr != session && session->GetOperation()) + if (operation) { // Session can not be destroyed when calling PerformCurlMessage auto hold_session = session->shared_from_this(); - session->GetOperation()->PerformCurlMessage(result); + operation->PerformCurlMessage(result); + + if (operation->IsRetryable()) + { + self->pending_to_retry_sessions_.push_back(hold_session); + } } } } while (true); @@ -538,6 +545,12 @@ bool HttpClient::MaybeSpawnBackgroundThread() still_running = 1; } + // Check if pending easy handles can be retried + if (self->doRetrySessions(false)) + { + still_running = 1; + } + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); if (still_running > 0) { @@ -588,6 +601,12 @@ bool HttpClient::MaybeSpawnBackgroundThread() still_running = 1; } + // Check if pending easy handles can be retried + if (self->doRetrySessions(true)) + { + still_running = 1; + } + // If there is no pending jobs, we can stop the background thread. if (still_running == 0) { @@ -811,6 +830,50 @@ bool HttpClient::doRemoveSessions() return has_data; } +#ifdef ENABLE_OTLP_RETRY_PREVIEW +bool HttpClient::doRetrySessions(bool report_all) +{ + const auto now = std::chrono::system_clock::now(); + auto has_data = false; + + // Assumptions: + // - This is a FIFO list so older sessions, pushed at the back, always end up at the front + // - Locking not required because only the background thread would be pushing to this container + // - Retry policy is not changed once HTTP client is initialized, so same settings for everyone + for (auto retry_it = pending_to_retry_sessions_.cbegin(); + retry_it != pending_to_retry_sessions_.cend();) + { + const auto session = *retry_it; + const auto operation = session ? session->GetOperation().get() : nullptr; + + if (!operation) + { + retry_it = pending_to_retry_sessions_.erase(retry_it); + } + else if (operation->NextRetryTime() < now) + { + auto easy_handle = operation->GetCurlEasyHandle(); + curl_multi_remove_handle(multi_handle_, easy_handle); + curl_multi_add_handle(multi_handle_, easy_handle); + retry_it = pending_to_retry_sessions_.erase(retry_it); + has_data = true; + } + else + { + break; + } + } + + report_all = report_all && !pending_to_retry_sessions_.empty(); + return has_data || report_all; +} +#else +bool HttpClient::doRetrySessions(bool /* report_all */) +{ + return false; +} +#endif // ENABLE_OTLP_RETRY_PREVIEW + void HttpClient::resetMultiHandle() { std::list> sessions; diff --git a/ext/src/http/client/curl/http_operation_curl.cc b/ext/src/http/client/curl/http_operation_curl.cc index 5795b0a875..1af00a0ff5 100644 --- a/ext/src/http/client/curl/http_operation_curl.cc +++ b/ext/src/http/client/curl/http_operation_curl.cc @@ -4,14 +4,22 @@ #include #include #include + +#ifdef ENABLE_OTLP_RETRY_PREVIEW +# include +#endif // ENABLE_OTLP_RETRY_PREVIEW + #include #include #include +#include #include #include #include #include #include +#include +#include #include #include #include @@ -262,7 +270,8 @@ HttpOperation::HttpOperation(opentelemetry::ext::http::client::Method method, bool is_raw_response, std::chrono::milliseconds http_conn_timeout, bool reuse_connection, - bool is_log_enabled) + bool is_log_enabled, + const RetryPolicy &retry_policy) : is_aborted_(false), is_finished_(false), is_cleaned_(false), @@ -283,6 +292,13 @@ HttpOperation::HttpOperation(opentelemetry::ext::http::client::Method method, session_state_(opentelemetry::ext::http::client::SessionState::Created), compression_(compression), is_log_enabled_(is_log_enabled), + retry_policy_(retry_policy), + retry_attempts_((retry_policy.max_attempts > 0U && + retry_policy.initial_backoff > SecondsDecimal::zero() && + retry_policy.max_backoff > SecondsDecimal::zero() && + retry_policy.backoff_multiplier > 0.0f) + ? 0 + : retry_policy.max_attempts), response_code_(0) { /* get a curl handle */ @@ -425,6 +441,53 @@ void HttpOperation::Cleanup() } } +bool HttpOperation::IsRetryable() +{ +#ifdef ENABLE_OTLP_RETRY_PREVIEW + static constexpr auto kRetryableStatusCodes = std::array{ + 429, // Too Many Requests + 502, // Bad Gateway + 503, // Service Unavailable + 504 // Gateway Timeout + }; + + const auto is_retryable = std::find(kRetryableStatusCodes.cbegin(), kRetryableStatusCodes.cend(), + response_code_) != kRetryableStatusCodes.cend(); + + return is_retryable && (last_curl_result_ == CURLE_OK) && + (retry_attempts_ < retry_policy_.max_attempts); +#else + return false; +#endif // ENABLE_OTLP_RETRY_PREVIEW +} + +std::chrono::system_clock::time_point HttpOperation::NextRetryTime() +{ + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_real_distribution dis(0.8, 1.2); + + // The initial retry attempt will occur after initialBackoff * random(0.8, 1.2) + auto backoff = retry_policy_.initial_backoff; + + // After that, the n-th attempt will occur after + // min(initialBackoff*backoffMultiplier**(n-1), maxBackoff) * random(0.8, 1.2)) + if (retry_attempts_ > 1) + { + backoff = (std::min)(retry_policy_.initial_backoff * + std::pow(retry_policy_.backoff_multiplier, + static_cast(retry_attempts_ - 1)), + retry_policy_.max_backoff); + } + + // Jitter of plus or minus 0.2 is applied to the backoff delay to avoid hammering servers at the + // same time from a large number of clients. Note that this means that the backoff delay may + // actually be slightly lower than initialBackoff or slightly higher than maxBackoff + backoff *= dis(gen); + + return last_attempt_time_ + std::chrono::duration_cast(backoff); +} + /* Support for TLS min version, TLS max version. @@ -1197,11 +1260,6 @@ CURLcode HttpOperation::Send() CURLcode code = curl_easy_perform(curl_resource_.easy_handle); PerformCurlMessage(code); - if (CURLE_OK != code) - { - return code; - } - return code; } @@ -1247,7 +1305,7 @@ CURLcode HttpOperation::SendAsync(Session *session, std::functioncallback = std::move(callback); session->GetHttpClient().ScheduleAddSession(session->GetSessionId()); - return code; + return CURLE_OK; } Headers HttpOperation::GetResponseHeaders() @@ -1268,7 +1326,7 @@ Headers HttpOperation::GetResponseHeaders() // switching to string comparison. Need to debug and revert back. /*std::smatch match; - std::regex http_headers_regex(http_header_regexp); + std::regex http_headers_regex(kHttpHeaderRegexp); if (std::regex_search(header, match, http_headers_regex)) result.insert(std::pair( static_cast(match[1]), static_cast(match[2]))); @@ -1305,7 +1363,10 @@ void HttpOperation::Abort() void HttpOperation::PerformCurlMessage(CURLcode code) { - last_curl_result_ = code; + ++retry_attempts_; + last_attempt_time_ = std::chrono::system_clock::now(); + last_curl_result_ = code; + if (code != CURLE_OK) { switch (GetSessionState()) @@ -1351,8 +1412,20 @@ void HttpOperation::PerformCurlMessage(CURLcode code) DispatchEvent(opentelemetry::ext::http::client::SessionState::Response); } - // Cleanup and unbind easy handle from multi handle, and finish callback - Cleanup(); + if (IsRetryable()) + { + // Clear any response data received in previous attempt + ReleaseResponse(); + // Rewind request data so that read callback can re-transfer the payload + request_nwrite_ = 0; + // Reset session state + DispatchEvent(opentelemetry::ext::http::client::SessionState::Connecting); + } + else + { + // Cleanup and unbind easy handle from multi handle, and finish callback + Cleanup(); + } } } // namespace curl diff --git a/ext/test/http/curl_http_test.cc b/ext/test/http/curl_http_test.cc index 4494a20702..e8299202b0 100644 --- a/ext/test/http/curl_http_test.cc +++ b/ext/test/http/curl_http_test.cc @@ -2,8 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 #include -#include +#include "gtest/gtest.h" + +#ifdef ENABLE_OTLP_RETRY_PREVIEW +# include +# include "gmock/gmock.h" +#endif // ENABLE_OTLP_RETRY_PREVIEW + #include +#include #include #include #include @@ -109,6 +116,17 @@ class FinishInCallbackHandler : public CustomEventHandler std::shared_ptr session_; }; +class RetryEventHandler : public CustomEventHandler +{ + void OnResponse(http_client::Response &response) noexcept override + { + ASSERT_EQ(429, response.GetStatusCode()); + ASSERT_EQ(response.GetBody().size(), 0); + is_called_.store(true, std::memory_order_release); + got_response_.store(true, std::memory_order_release); + } +}; + class BasicCurlHttpTests : public ::testing::Test, public HTTP_SERVER_NS::HttpRequestCallback { protected: @@ -140,6 +158,7 @@ class BasicCurlHttpTests : public ::testing::Test, public HTTP_SERVER_NS::HttpRe server_.addHandler("/simple/", *this); server_.addHandler("/get/", *this); server_.addHandler("/post/", *this); + server_.addHandler("/retry/", *this); server_.start(); is_running_ = true; } @@ -171,6 +190,13 @@ class BasicCurlHttpTests : public ::testing::Test, public HTTP_SERVER_NS::HttpRe response.body = "{'k1':'v1', 'k2':'v2', 'k3':'v3'}"; response_status = 200; } + else if (request.uri == "/retry/") + { + std::unique_lock lk1(mtx_requests); + received_requests_.push_back(request); + response.headers["Content-Type"] = "text/plain"; + response_status = 429; + } cv_got_events.notify_one(); @@ -331,6 +357,91 @@ TEST_F(BasicCurlHttpTests, CurlHttpOperations) delete handler; } +#ifdef ENABLE_OTLP_RETRY_PREVIEW +TEST_F(BasicCurlHttpTests, RetryPolicyEnabled) +{ + RetryEventHandler handler; + http_client::HttpSslOptions no_ssl; + http_client::Body body; + http_client::Headers headers; + http_client::Compression compression = http_client::Compression::kNone; + http_client::RetryPolicy retry_policy = {5, std::chrono::duration{1.0f}, + std::chrono::duration{5.0f}, 1.5f}; + + curl::HttpOperation operation(http_client::Method::Post, "http://127.0.0.1:19000/retry/", no_ssl, + &handler, headers, body, compression, false, + curl::kDefaultHttpConnTimeout, false, false, retry_policy); + + ASSERT_EQ(CURLE_OK, operation.Send()); + ASSERT_TRUE(operation.IsRetryable()); +} + +TEST_F(BasicCurlHttpTests, RetryPolicyDisabled) +{ + RetryEventHandler handler; + http_client::HttpSslOptions no_ssl; + http_client::Body body; + http_client::Headers headers; + http_client::Compression compression = http_client::Compression::kNone; + http_client::RetryPolicy no_retry_policy = {0, std::chrono::duration::zero(), + std::chrono::duration::zero(), 0.0f}; + + curl::HttpOperation operation(http_client::Method::Post, "http://127.0.0.1:19000/retry/", no_ssl, + &handler, headers, body, compression, false, + curl::kDefaultHttpConnTimeout, false, false, no_retry_policy); + + ASSERT_EQ(CURLE_OK, operation.Send()); + ASSERT_FALSE(operation.IsRetryable()); +} + +TEST_F(BasicCurlHttpTests, ExponentialBackoffRetry) +{ + using ::testing::AllOf; + using ::testing::Gt; + using ::testing::Lt; + + RetryEventHandler handler; + http_client::HttpSslOptions no_ssl; + http_client::Body body; + http_client::Headers headers; + http_client::Compression compression = http_client::Compression::kNone; + http_client::RetryPolicy retry_policy = {4, std::chrono::duration{1.0f}, + std::chrono::duration{5.0f}, 2.0f}; + + curl::HttpOperation operation(http_client::Method::Post, "http://127.0.0.1:19000/retry/", no_ssl, + &handler, headers, body, compression, false, + curl::kDefaultHttpConnTimeout, false, false, retry_policy); + + auto first_attempt_time = std::chrono::system_clock::now(); + ASSERT_EQ(CURLE_OK, operation.Send()); + ASSERT_TRUE(operation.IsRetryable()); + ASSERT_THAT( + operation.NextRetryTime().time_since_epoch().count(), + AllOf(Gt((first_attempt_time + std::chrono::milliseconds{750}).time_since_epoch().count()), + Lt((first_attempt_time + std::chrono::milliseconds{1250}).time_since_epoch().count()))); + + auto second_attempt_time = std::chrono::system_clock::now(); + ASSERT_EQ(CURLE_OK, operation.Send()); + ASSERT_TRUE(operation.IsRetryable()); + ASSERT_THAT( + operation.NextRetryTime().time_since_epoch().count(), + AllOf( + Gt((second_attempt_time + std::chrono::milliseconds{1550}).time_since_epoch().count()), + Lt((second_attempt_time + std::chrono::milliseconds{2450}).time_since_epoch().count()))); + + auto third_attempt_time = std::chrono::system_clock::now(); + ASSERT_EQ(CURLE_OK, operation.Send()); + ASSERT_TRUE(operation.IsRetryable()); + ASSERT_THAT( + operation.NextRetryTime().time_since_epoch().count(), + AllOf(Gt((third_attempt_time + std::chrono::milliseconds{3150}).time_since_epoch().count()), + Lt((third_attempt_time + std::chrono::milliseconds{4850}).time_since_epoch().count()))); + + ASSERT_EQ(CURLE_OK, operation.Send()); + ASSERT_FALSE(operation.IsRetryable()); +} +#endif // ENABLE_OTLP_RETRY_PREVIEW + TEST_F(BasicCurlHttpTests, SendGetRequestSync) { received_requests_.clear(); @@ -560,6 +671,7 @@ TEST_F(BasicCurlHttpTests, ElegantQuitQuick) ASSERT_TRUE(handler->is_called_); ASSERT_TRUE(handler->got_response_); } + TEST_F(BasicCurlHttpTests, BackgroundThreadWaitMore) { { diff --git a/sdk/include/opentelemetry/sdk/common/env_variables.h b/sdk/include/opentelemetry/sdk/common/env_variables.h index a02a66c29e..7bf28e2c1c 100644 --- a/sdk/include/opentelemetry/sdk/common/env_variables.h +++ b/sdk/include/opentelemetry/sdk/common/env_variables.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include "opentelemetry/version.h" @@ -39,6 +40,22 @@ bool GetDurationEnvironmentVariable(const char *env_var_name, */ bool GetStringEnvironmentVariable(const char *env_var_name, std::string &value); +/** + Read a uint32_t environment variable. + @param env_var_name Environment variable name + @param [out] value Variable value, if it exists + @return true if the variable exists +*/ +bool GetUintEnvironmentVariable(const char *env_var_name, std::uint32_t &value); + +/** + Read a float environment variable. + @param env_var_name Environment variable name + @param [out] value Variable value, if it exists + @return true if the variable exists +*/ +bool GetFloatEnvironmentVariable(const char *env_var_name, float &value); + #if defined(_MSC_VER) inline int setenv(const char *name, const char *value, int) { diff --git a/sdk/src/common/env_variables.cc b/sdk/src/common/env_variables.cc index 586a8ed420..016f738d4e 100644 --- a/sdk/src/common/env_variables.cc +++ b/sdk/src/common/env_variables.cc @@ -10,7 +10,10 @@ # include #endif +#include +#include #include +#include #include #include "opentelemetry/nostd/string_view.h" @@ -89,10 +92,10 @@ static bool GetTimeoutFromString(const char *input, std::chrono::system_clock::d std::chrono::system_clock::duration::rep result = 0; // Skip spaces - for (; *input && (' ' == *input || '\t' == *input || '\r' == *input || '\n' == *input); ++input) + for (; *input && std::isspace(*input); ++input) ; - for (; *input && (*input >= '0' && *input <= '9'); ++input) + for (; *input && std::isdigit(*input); ++input) { result = result * 10 + (*input - '0'); } @@ -193,6 +196,83 @@ bool GetStringEnvironmentVariable(const char *env_var_name, std::string &value) return true; } +bool GetUintEnvironmentVariable(const char *env_var_name, std::uint32_t &value) +{ + static constexpr auto kDefaultValue = 0U; + std::string raw_value; + bool exists = GetRawEnvironmentVariable(env_var_name, raw_value); + + if (!exists || raw_value.empty()) + { + value = kDefaultValue; + return false; + } + + const char *end = raw_value.c_str() + raw_value.length(); + char *actual_end = nullptr; + const auto temp = std::strtoull(raw_value.c_str(), &actual_end, 10); + + if (errno == ERANGE) + { + errno = 0; + OTEL_INTERNAL_LOG_WARN("Environment variable <" << env_var_name << "> is out of range <" + << raw_value << ">, defaulting to " + << kDefaultValue); + } + else if (actual_end != end || std::numeric_limits::max() < temp) + { + OTEL_INTERNAL_LOG_WARN("Environment variable <" << env_var_name << "> has an invalid value <" + << raw_value << ">, defaulting to " + << kDefaultValue); + } + else + { + value = static_cast(temp); + return true; + } + + value = kDefaultValue; + return false; +} + +bool GetFloatEnvironmentVariable(const char *env_var_name, float &value) +{ + static constexpr auto kDefaultValue = 0.0f; + std::string raw_value; + bool exists = GetRawEnvironmentVariable(env_var_name, raw_value); + + if (!exists || raw_value.empty()) + { + value = kDefaultValue; + return false; + } + + const char *end = raw_value.c_str() + raw_value.length(); + char *actual_end = nullptr; + value = std::strtof(raw_value.c_str(), &actual_end); + + if (errno == ERANGE) + { + errno = 0; + OTEL_INTERNAL_LOG_WARN("Environment variable <" << env_var_name << "> is out of range <" + << raw_value << ">, defaulting to " + << kDefaultValue); + } + else if (actual_end != end) + { + OTEL_INTERNAL_LOG_WARN("Environment variable <" << env_var_name << "> has an invalid value <" + << raw_value << ">, defaulting to " + << kDefaultValue); + } + else + { + return true; + } + + value = kDefaultValue; + return false; +} + } // namespace common } // namespace sdk OPENTELEMETRY_END_NAMESPACE diff --git a/sdk/test/common/env_var_test.cc b/sdk/test/common/env_var_test.cc index 8f64dc1f10..b4cc0d2e12 100644 --- a/sdk/test/common/env_var_test.cc +++ b/sdk/test/common/env_var_test.cc @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 #include -#include + #include +#include +#include #include #include "opentelemetry/sdk/common/env_variables.h" @@ -15,7 +17,9 @@ using opentelemetry::sdk::common::unsetenv; using opentelemetry::sdk::common::GetBoolEnvironmentVariable; using opentelemetry::sdk::common::GetDurationEnvironmentVariable; +using opentelemetry::sdk::common::GetFloatEnvironmentVariable; using opentelemetry::sdk::common::GetStringEnvironmentVariable; +using opentelemetry::sdk::common::GetUintEnvironmentVariable; #ifndef NO_GETENV TEST(EnvVarTest, BoolEnvVar) @@ -210,4 +214,139 @@ TEST(EnvVarTest, DurationEnvVar) unsetenv("STRING_ENV_VAR_BROKEN_2"); } -#endif +TEST(EnvVarTest, UintEnvVar) +{ + unsetenv("UINT_ENV_VAR_NONE"); + setenv("UINT_ENV_VAR_EMPTY", "", 1); + setenv("UINT_ENV_VAR_POSITIVE_INT", "42", 1); + setenv("UINT_ENV_VAR_NEGATIVE_INT", "-42", 1); + setenv("UINT_ENV_VAR_POSITIVE_DEC", "12.34", 1); + setenv("UINT_ENV_VAR_NEGATIVE_DEC", "-12.34", 1); + setenv("UINT_ENV_VAR_POSITIVE_INT_MAX", "4294967295", 1); + setenv("UINT_ENV_VAR_POSITIVE_OVERFLOW", "4294967296", 1); + setenv("UINT_ENV_VAR_NEGATIVE_INT_MIN", "-2147483648", 1); + setenv("UINT_ENV_VAR_NEGATIVE_OVERFLOW", "-4294967296", 1); + setenv("UINT_ENV_VAR_TOO_LARGE_INT", "99999999999999999999", 1); + setenv("UINT_ENV_VAR_TOO_LARGE_DEC", "3.9999e+99", 1); + setenv("UINT_ENV_VAR_WITH_NOISE", " \t \n 9.12345678.9", 1); + setenv("UINT_ENV_VAR_ONLY_SPACES", " ", 1); + + std::uint32_t value; + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_NONE", value)); + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_EMPTY", value)); + + ASSERT_TRUE(GetUintEnvironmentVariable("UINT_ENV_VAR_POSITIVE_INT", value)); + ASSERT_EQ(42, value); + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_NEGATIVE_INT", value)); + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_POSITIVE_DEC", value)); + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_NEGATIVE_DEC", value)); + + ASSERT_TRUE(GetUintEnvironmentVariable("UINT_ENV_VAR_POSITIVE_INT_MAX", value)); + ASSERT_EQ(4294967295, value); + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_POSITIVE_OVERFLOW", value)); + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_NEGATIVE_INT_MIN", value)); + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_NEGATIVE_OVERFLOW", value)); + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_TOO_LARGE_INT", value)); + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_TOO_LARGE_DEC", value)); + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_WITH_NOISE", value)); + + ASSERT_FALSE(GetUintEnvironmentVariable("UINT_ENV_VAR_ONLY_SPACES", value)); + + unsetenv("UINT_ENV_VAR_EMPTY"); + unsetenv("UINT_ENV_VAR_POSITIVE_INT"); + unsetenv("UINT_ENV_VAR_NEGATIVE_INT"); + unsetenv("UINT_ENV_VAR_POSITIVE_DEC"); + unsetenv("UINT_ENV_VAR_NEGATIVE_DEC"); + unsetenv("UINT_ENV_VAR_POSITIVE_INT_MAX"); + unsetenv("UINT_ENV_VAR_POSITIVE_OVERFLOW"); + unsetenv("UINT_ENV_VAR_NEGATIVE_INT_MIN"); + unsetenv("UINT_ENV_VAR_NEGATIVE_OVERFLOW"); + unsetenv("UINT_ENV_VAR_TOO_LARGE_INT"); + unsetenv("UINT_ENV_VAR_TOO_LARGE_DEC"); + unsetenv("UINT_ENV_VAR_WITH_NOISE"); + unsetenv("UINT_ENV_VAR_ONLY_SPACES"); +} + +TEST(EnvVarTest, FloatEnvVar) +{ + unsetenv("FLOAT_ENV_VAR_NONE"); + setenv("FLOAT_ENV_VAR_EMPTY", "", 1); + setenv("FLOAT_ENV_VAR_POSITIVE_INT", "42", 1); + setenv("FLOAT_ENV_VAR_NEGATIVE_INT", "-42", 1); + setenv("FLOAT_ENV_VAR_POSITIVE_DEC", "12.34", 1); + setenv("FLOAT_ENV_VAR_NEGATIVE_DEC", "-12.34", 1); + setenv("FLOAT_ENV_VAR_POSITIVE_INT_MAX", "4294967295", 1); + setenv("FLOAT_ENV_VAR_POSITIVE_OVERFLOW", "4294967296", 1); + setenv("FLOAT_ENV_VAR_NEGATIVE_INT_MIN", "-2147483648", 1); + setenv("FLOAT_ENV_VAR_NEGATIVE_OVERFLOW", "-4294967296", 1); + setenv("FLOAT_ENV_VAR_TOO_LARGE_INT", "99999999999999999999", 1); + setenv("FLOAT_ENV_VAR_TOO_LARGE_DEC", "3.9999e+99", 1); + setenv("FLOAT_ENV_VAR_WITH_NOISE", " \t \n 9.12345678.9", 1); + setenv("FLOAT_ENV_VAR_ONLY_SPACES", " ", 1); + + float value; + + ASSERT_FALSE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_NONE", value)); + + ASSERT_FALSE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_EMPTY", value)); + + ASSERT_TRUE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_POSITIVE_INT", value)); + ASSERT_FLOAT_EQ(42.f, value); + + ASSERT_TRUE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_NEGATIVE_INT", value)); + ASSERT_FLOAT_EQ(-42.f, value); + + ASSERT_TRUE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_POSITIVE_DEC", value)); + ASSERT_FLOAT_EQ(12.34f, value); + + ASSERT_TRUE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_NEGATIVE_DEC", value)); + ASSERT_FLOAT_EQ(-12.34f, value); + + ASSERT_TRUE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_POSITIVE_INT_MAX", value)); + ASSERT_FLOAT_EQ(4294967295.f, value); + + ASSERT_TRUE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_POSITIVE_OVERFLOW", value)); + ASSERT_FLOAT_EQ(4294967296.f, value); + + ASSERT_TRUE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_NEGATIVE_INT_MIN", value)); + ASSERT_FLOAT_EQ(-2147483648.f, value); + + ASSERT_TRUE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_NEGATIVE_OVERFLOW", value)); + ASSERT_FLOAT_EQ(-4294967296.f, value); + + ASSERT_TRUE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_TOO_LARGE_INT", value)); + ASSERT_FLOAT_EQ(99999999999999999999.f, value); + + ASSERT_FALSE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_TOO_LARGE_DEC", value)); + + ASSERT_FALSE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_WITH_NOISE", value)); + + ASSERT_FALSE(GetFloatEnvironmentVariable("FLOAT_ENV_VAR_ONLY_SPACES", value)); + + unsetenv("FLOAT_ENV_VAR_EMPTY"); + unsetenv("FLOAT_ENV_VAR_POSITIVE_INT"); + unsetenv("FLOAT_ENV_VAR_NEGATIVE_INT"); + unsetenv("FLOAT_ENV_VAR_POSITIVE_DEC"); + unsetenv("FLOAT_ENV_VAR_NEGATIVE_DEC"); + unsetenv("FLOAT_ENV_VAR_POSITIVE_INT_MAX"); + unsetenv("FLOAT_ENV_VAR_POSITIVE_OVERFLOW"); + unsetenv("FLOAT_ENV_VAR_NEGATIVE_INT_MIN"); + unsetenv("FLOAT_ENV_VAR_NEGATIVE_OVERFLOW"); + unsetenv("FLOAT_ENV_VAR_TOO_LARGE_INT"); + unsetenv("FLOAT_ENV_VAR_TOO_LARGE_DEC"); + unsetenv("FLOAT_ENV_VAR_WITH_NOISE"); + unsetenv("FLOAT_ENV_VAR_ONLY_SPACES"); +} + +#endif // NO_GETENV diff --git a/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h b/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h index 211b5b3720..6a30e4fd26 100644 --- a/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h +++ b/test_common/include/opentelemetry/test_common/ext/http/client/nosend/http_client_nosend.h @@ -71,6 +71,12 @@ class Request : public opentelemetry::ext::http::client::Request void EnableLogging(bool is_log_enabled) noexcept override { is_log_enabled_ = is_log_enabled; } + void SetRetryPolicy( + const opentelemetry::ext::http::client::RetryPolicy &retry_policy) noexcept override + { + retry_policy_ = retry_policy; + } + public: opentelemetry::ext::http::client::Method method_; opentelemetry::ext::http::client::HttpSslOptions ssl_options_; @@ -81,6 +87,7 @@ class Request : public opentelemetry::ext::http::client::Request opentelemetry::ext::http::client::Compression compression_{ opentelemetry::ext::http::client::Compression::kNone}; bool is_log_enabled_{false}; + opentelemetry::ext::http::client::RetryPolicy retry_policy_; }; class Response : public opentelemetry::ext::http::client::Response