feat: support backoff/retry in OTLP

Cargo.toml

@@ -44,6 +44,7 @@ thiserror = { version = "2", default-features = false }
 tonic = { version = "0.14.1", default-features = false }
 tonic-prost-build = "0.14.1"
 tonic-prost = "0.14.1"
+tonic-types = "0.14.1"
 tokio = { version = "1", default-features = false }
 tokio-stream = "0.1"
 # Using `tracing 0.1.40` because 0.1.39 (which is yanked) introduces the ability to set event names in macros,

opentelemetry-otlp/CHANGELOG.md

@@ -8,6 +8,7 @@ Released 2025-Sep-25
 
 - Update `opentelemetry-proto` and `opentelemetry-http` dependency version to 0.31.0
 - Add HTTP compression support with `gzip-http` and `zstd-http` feature flags
+- Add retry with exponential backoff and throttling support for HTTP and gRPC exporters
 
 ## 0.30.0
 

opentelemetry-otlp/Cargo.toml

@@ -35,10 +35,12 @@ tracing = {workspace = true, optional = true}
 
 prost = { workspace = true, optional = true }
 tonic = { workspace = true, optional = true }
+tonic-types = { workspace = true, optional = true }
 tokio = { workspace = true, features = ["sync", "rt"], optional = true }
 
 reqwest = { workspace = true, optional = true }
 http = { workspace = true, optional = true }
+httpdate = { version = "1.0.3", optional = true }
 serde = { workspace = true, features = ["derive"], optional = true }
 thiserror = { workspace = true }
 serde_json = { workspace = true, optional = true }
@@ -69,10 +71,13 @@ serialize = ["serde", "serde_json"]
 default = ["http-proto", "reqwest-blocking-client", "trace", "metrics", "logs", "internal-logs"]
 
 # grpc using tonic
-grpc-tonic = ["tonic", "prost", "http", "tokio", "opentelemetry-proto/gen-tonic"]
+grpc-tonic = ["tonic", "tonic-types", "prost", "http", "tokio", "opentelemetry-proto/gen-tonic"]
 gzip-tonic = ["tonic/gzip"]
 zstd-tonic = ["tonic/zstd"]
 
+# grpc with retry support
+experimental-grpc-retry = ["grpc-tonic", "opentelemetry_sdk/experimental_async_runtime", "opentelemetry_sdk/rt-tokio"]
+
 # http compression
 gzip-http = ["flate2"]
 zstd-http = ["zstd"]
@@ -82,6 +87,10 @@ tls-webpki-roots = ["tls", "tonic/tls-webpki-roots"]
 
 # http binary
 http-proto = ["prost", "opentelemetry-http", "opentelemetry-proto/gen-tonic-messages", "http", "trace", "metrics"]
+
+# http with retry support.
+experimental-http-retry = ["opentelemetry_sdk/experimental_async_runtime", "opentelemetry_sdk/rt-tokio", "tokio", "httpdate"]
+
 http-json = ["serde_json", "prost", "opentelemetry-http", "opentelemetry-proto/gen-tonic-messages", "opentelemetry-proto/with-serde", "http", "trace", "metrics"]
 reqwest-blocking-client = ["reqwest/blocking", "opentelemetry-http/reqwest-blocking"]
 reqwest-client = ["reqwest", "opentelemetry-http/reqwest"]

opentelemetry-otlp/allowed-external-types.toml

@@ -18,4 +18,7 @@ allowed_external_types = [
     "tonic::transport::tls::Identity",
     "tonic::transport::channel::Channel",
     "tonic::service::interceptor::Interceptor",
+
+    # For retries
+    "tonic::status::Status"
 ]

opentelemetry-otlp/src/exporter/http/logs.rs

@@ -1,61 +1,16 @@
 use super::OtlpHttpClient;
-use http::header::CONTENT_ENCODING;
-use http::{header::CONTENT_TYPE, Method};
-use opentelemetry::otel_debug;
 use opentelemetry_sdk::error::{OTelSdkError, OTelSdkResult};
 use opentelemetry_sdk::logs::{LogBatch, LogExporter};
 use std::time;
 
 impl LogExporter for OtlpHttpClient {
     async fn export(&self, batch: LogBatch<'_>) -> OTelSdkResult {
-        let client = self
-            .client
-            .lock()
-            .map_err(|e| OTelSdkError::InternalFailure(format!("Mutex lock failed: {e}")))?
-            .clone()
-            .ok_or(OTelSdkError::AlreadyShutdown)?;
-
-        let (body, content_type, content_encoding) = self
-            .build_logs_export_body(batch)
-            .map_err(OTelSdkError::InternalFailure)?;
-
-        let mut request_builder = http::Request::builder()
-            .method(Method::POST)
-            .uri(&self.collector_endpoint)
-            .header(CONTENT_TYPE, content_type);
-
-        if let Some(encoding) = content_encoding {
-            request_builder = request_builder.header(CONTENT_ENCODING, encoding);
-        }
-
-        let mut request = request_builder
-            .body(body.into())
-            .map_err(|e| OTelSdkError::InternalFailure(e.to_string()))?;
-
-        for (k, v) in &self.headers {
-            request.headers_mut().insert(k.clone(), v.clone());
-        }
-
-        let request_uri = request.uri().to_string();
-        otel_debug!(name: "HttpLogsClient.ExportStarted");
-        let response = client
-            .send_bytes(request)
-            .await
-            .map_err(|e| OTelSdkError::InternalFailure(format!("{e:?}")))?;
-
-        if !response.status().is_success() {
-            let error = format!(
-                "OpenTelemetry logs export failed. Url: {}, Status Code: {}, Response: {:?}",
-                request_uri,
-                response.status().as_u16(),
-                response.body()
-            );
-            otel_debug!(name: "HttpLogsClient.ExportFailed", error = &error);
-            return Err(OTelSdkError::InternalFailure(error));
-        }
-
-        otel_debug!(name: "HttpLogsClient.ExportSucceeded");
-        Ok(())
+        self.export_http_with_retry(
+            batch,
+            OtlpHttpClient::build_logs_export_body,
+            "HttpLogsClient.Export",
+        )
+        .await
     }
 
     fn shutdown_with_timeout(&self, _timeout: time::Duration) -> OTelSdkResult {

opentelemetry-otlp/src/exporter/http/metrics.rs

@@ -1,70 +1,19 @@
-use std::sync::Arc;
-
 use crate::metric::MetricsClient;
-use http::{header::CONTENT_TYPE, Method};
-use opentelemetry::otel_debug;
 use opentelemetry_sdk::error::{OTelSdkError, OTelSdkResult};
 use opentelemetry_sdk::metrics::data::ResourceMetrics;
 
 use super::OtlpHttpClient;
 
 impl MetricsClient for OtlpHttpClient {
     async fn export(&self, metrics: &ResourceMetrics) -> OTelSdkResult {
-        let client = self
-            .client
-            .lock()
-            .map_err(|e| OTelSdkError::InternalFailure(format!("Failed to acquire lock: {e:?}")))
-            .and_then(|g| match &*g {
-                Some(client) => Ok(Arc::clone(client)),
-                _ => Err(OTelSdkError::AlreadyShutdown),
-            })?;
-
-        let (body, content_type, content_encoding) =
-            self.build_metrics_export_body(metrics).ok_or_else(|| {
-                OTelSdkError::InternalFailure("Failed to serialize metrics".to_string())
-            })?;
-
-        let mut request_builder = http::Request::builder()
-            .method(Method::POST)
-            .uri(&self.collector_endpoint)
-            .header(CONTENT_TYPE, content_type);
-
-        if let Some(encoding) = content_encoding {
-            request_builder = request_builder.header("Content-Encoding", encoding);
-        }
-
-        let mut request = request_builder
-            .body(body.into())
-            .map_err(|e| OTelSdkError::InternalFailure(format!("{e:?}")))?;
-
-        for (k, v) in &self.headers {
-            request.headers_mut().insert(k.clone(), v.clone());
-        }
-
-        otel_debug!(name: "HttpMetricsClient.ExportStarted");
-        let result = client.send_bytes(request).await;
-
-        match result {
-            Ok(response) => {
-                if response.status().is_success() {
-                    otel_debug!(name: "HttpMetricsClient.ExportSucceeded");
-                    Ok(())
-                } else {
-                    let error = format!(
-                        "OpenTelemetry metrics export failed. Status Code: {}, Response: {:?}",
-                        response.status().as_u16(),
-                        response.body()
-                    );
-                    otel_debug!(name: "HttpMetricsClient.ExportFailed", error = &error);
-                    Err(OTelSdkError::InternalFailure(error))
-                }
-            }
-            Err(e) => {
-                let error = format!("{e:?}");
-                otel_debug!(name: "HttpMetricsClient.ExportFailed", error = &error);
-                Err(OTelSdkError::InternalFailure(error))
-            }
-        }
+        let build_body_wrapper = |client: &OtlpHttpClient, metrics: &ResourceMetrics| {
+            client
+                .build_metrics_export_body(metrics)
+                .ok_or_else(|| "Failed to serialize metrics".to_string())
+        };
+
+        self.export_http_with_retry(metrics, build_body_wrapper, "HttpMetricsClient.Export")
+            .await
     }
 
     fn shutdown(&self) -> OTelSdkResult {