add retry to metrics

Author: scottgerring

opentelemetry-otlp/src/exporter/tonic/metrics.rs

@@ -12,6 +12,9 @@ use tonic::{codegen::CompressionEncoding, service::Interceptor, transport::Chann
 use super::BoxInterceptor;
 use crate::metric::MetricsClient;
 
+use opentelemetry_sdk::retry::{retry_with_exponential_backoff, RetryPolicy};
+use opentelemetry_sdk::runtime::Tokio;
+
 pub(crate) struct TonicMetricsClient {
     inner: Mutex<Option<ClientInner>>,
 }
@@ -53,49 +56,63 @@ impl TonicMetricsClient {
 
 impl MetricsClient for TonicMetricsClient {
     async fn export(&self, metrics: &ResourceMetrics) -> OTelSdkResult {
-        let (mut client, metadata, extensions) = self
-            .inner
-            .lock()
-            .map_err(|e| OTelSdkError::InternalFailure(format!("Failed to acquire lock: {e:?}")))
-            .and_then(|mut inner| match &mut *inner {
-                Some(inner) => {
-                    let (m, e, _) = inner
-                        .interceptor
-                        .call(Request::new(()))
-                        .map_err(|e| {
-                            OTelSdkError::InternalFailure(format!(
-                                "unexpected status while exporting {e:?}"
-                            ))
-                        })?
-                        .into_parts();
-                    Ok((inner.client.clone(), m, e))
-                }
-                None => Err(OTelSdkError::InternalFailure(
-                    "exporter is already shut down".into(),
-                )),
-            })?;
-
-        otel_debug!(name: "TonicMetricsClient.ExportStarted");
-
-        let result = client
-            .export(Request::from_parts(
-                metadata,
-                extensions,
-                ExportMetricsServiceRequest::from(metrics),
-            ))
-            .await;
-
-        match result {
-            Ok(_) => {
-                otel_debug!(name: "TonicMetricsClient.ExportSucceeded");
-                Ok(())
-            }
-            Err(e) => {
-                let error = format!("{e:?}");
-                otel_debug!(name: "TonicMetricsClient.ExportFailed", error = &error);
-                Err(OTelSdkError::InternalFailure(error))
+        let policy = RetryPolicy {
+            max_retries: 3,
+            initial_delay_ms: 100,
+            max_delay_ms: 1600,
+            jitter_ms: 100,
+        };
+
+        retry_with_exponential_backoff(Tokio, policy, "TonicMetricsClient.Export", {
+            let inner = &self.inner;
+            move || {
+                Box::pin(async move {
+                    let (mut client, metadata, extensions) = inner
+                        .lock()
+                        .map_err(|e| OTelSdkError::InternalFailure(format!("Failed to acquire lock: {e:?}")))
+                        .and_then(|mut inner| match &mut *inner {
+                            Some(inner) => {
+                                let (m, e, _) = inner
+                                    .interceptor
+                                    .call(Request::new(()))
+                                    .map_err(|e| {
+                                        OTelSdkError::InternalFailure(format!(
+                                            "unexpected status while exporting {e:?}"
+                                        ))
+                                    })?
+                                    .into_parts();
+                                Ok((inner.client.clone(), m, e))
+                            }
+                            None => Err(OTelSdkError::InternalFailure(
+                                "exporter is already shut down".into(),
+                            )),
+                        })?;
+
+                    otel_debug!(name: "TonicMetricsClient.ExportStarted");
+
+                    let result = client
+                        .export(Request::from_parts(
+                            metadata,
+                            extensions,
+                            ExportMetricsServiceRequest::from(metrics),
+                        ))
+                        .await;
+
+                    match result {
+                        Ok(_) => {
+                            otel_debug!(name: "TonicMetricsClient.ExportSucceeded");
+                            Ok(())
+                        }
+                        Err(e) => {
+                            let error = format!("export error: {e:?}");
+                            otel_debug!(name: "TonicMetricsClient.ExportFailed", error = &error);
+                            Err(OTelSdkError::InternalFailure(error))
+                        }
+                    }
+                })
             }
-        }
+        })
+        .await
     }
 
     fn shutdown(&self) -> OTelSdkResult {