Skip to content

Commit 4215968

Browse files
vishweshbankwarCodeBlanch
vishweshbankwar
and
CodeBlanch
authored
[Otlp] Add disk retry enablement (#5527)
Co-authored-by: Mikel Blanchard <[email protected]>
1 parent c47dd7c commit 4215968

File tree

5 files changed

+178
-30
lines changed

5 files changed

+178
-30
lines changed

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,17 @@
1313
`parent_is_remote` information.
1414
([#5563](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5563))
1515

16+
* Introduced experimental support for automatically retrying export to the otlp
17+
endpoint by storing the telemetry offline during transient network errors.
18+
Users can enable this feature by setting the
19+
`OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY` environment variable to `disk`. The
20+
default path where the telemetry is stored is obtained by calling
21+
[Path.GetTempPath()](https://learn.microsoft.com/dotnet/api/system.io.path.gettemppath)
22+
or can be customized by setting
23+
`OTEL_DOTNET_EXPERIMENTAL_OTLP_DISK_RETRY_DIRECTORY_PATH` environment
24+
variable.
25+
([#5527](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5527))
26+
1627
## 1.8.1
1728

1829
Released 2024-Apr-17

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExperimentalOptions.cs‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ internal sealed class ExperimentalOptions
1717

1818
public const string OtlpRetryEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY";
1919

20+
public const string OtlpDiskRetryDirectoryPathEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_DISK_RETRY_DIRECTORY_PATH";
21+
2022
public ExperimentalOptions()
2123
: this(new ConfigurationBuilder().AddEnvironmentVariables().Build())
2224
{
@@ -29,9 +31,29 @@ public ExperimentalOptions(IConfiguration configuration)
2931
this.EmitLogEventAttributes = emitLogEventAttributes;
3032
}
3133

32-
if (configuration.TryGetStringValue(OtlpRetryEnvVar, out var retryPolicy) && retryPolicy != null && retryPolicy.Equals("in_memory", StringComparison.OrdinalIgnoreCase))
34+
if (configuration.TryGetStringValue(OtlpRetryEnvVar, out var retryPolicy) && retryPolicy != null)
3335
{
34-
this.EnableInMemoryRetry = true;
36+
if (retryPolicy.Equals("in_memory", StringComparison.OrdinalIgnoreCase))
37+
{
38+
this.EnableInMemoryRetry = true;
39+
}
40+
else if (retryPolicy.Equals("disk", StringComparison.OrdinalIgnoreCase))
41+
{
42+
this.EnableDiskRetry = true;
43+
if (configuration.TryGetStringValue(OtlpDiskRetryDirectoryPathEnvVar, out var path) && path != null)
44+
{
45+
this.DiskRetryDirectoryPath = path;
46+
}
47+
else
48+
{
49+
// Fallback to temp location.
50+
this.DiskRetryDirectoryPath = Path.GetTempPath();
51+
}
52+
}
53+
else
54+
{
55+
throw new NotSupportedException($"Retry Policy '{retryPolicy}' is not supported.");
56+
}
3557
}
3658
}
3759

@@ -48,4 +70,14 @@ public ExperimentalOptions(IConfiguration configuration)
4870
/// href="https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#retry"/>.
4971
/// </remarks>
5072
public bool EnableInMemoryRetry { get; }
73+
74+
/// <summary>
75+
/// Gets a value indicating whether or not retry via disk should be enabled for transient errors.
76+
/// </summary>
77+
public bool EnableDiskRetry { get; }
78+
79+
/// <summary>
80+
/// Gets the path on disk where the telemetry will be stored for retries at a later point.
81+
/// </summary>
82+
public string? DiskRetryDirectoryPath { get; }
5183
}

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsExtensions.cs‎

Lines changed: 71 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
#if NETSTANDARD2_1 || NET6_0_OR_GREATER
1212
using Grpc.Net.Client;
1313
#endif
14+
using System.Diagnostics;
15+
using Google.Protobuf;
1416
using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Transmission;
1517
using LogOtlpCollector = OpenTelemetry.Proto.Collector.Logs.V1;
1618
using MetricsOtlpCollector = OpenTelemetry.Proto.Collector.Metrics.V1;
@@ -100,9 +102,29 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
100102
? httpTraceExportClient.HttpClient.Timeout.TotalMilliseconds
101103
: options.TimeoutMilliseconds;
102104

103-
return experimentalOptions.EnableInMemoryRetry
104-
? new OtlpExporterRetryTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds)
105-
: new OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
105+
if (experimentalOptions.EnableInMemoryRetry)
106+
{
107+
return new OtlpExporterRetryTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
108+
}
109+
else if (experimentalOptions.EnableDiskRetry)
110+
{
111+
Debug.Assert(!string.IsNullOrEmpty(experimentalOptions.DiskRetryDirectoryPath), $"{nameof(experimentalOptions.DiskRetryDirectoryPath)} is null or empty");
112+
113+
return new OtlpExporterPersistentStorageTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(
114+
exportClient,
115+
timeoutMilliseconds,
116+
(byte[] data) =>
117+
{
118+
var request = new TraceOtlpCollector.ExportTraceServiceRequest();
119+
request.MergeFrom(data);
120+
return request;
121+
},
122+
Path.Combine(experimentalOptions.DiskRetryDirectoryPath, "traces"));
123+
}
124+
else
125+
{
126+
return new OtlpExporterTransmissionHandler<TraceOtlpCollector.ExportTraceServiceRequest>(exportClient, timeoutMilliseconds);
127+
}
106128
}
107129

108130
public static OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest> GetMetricsExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions)
@@ -116,9 +138,29 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
116138
? httpMetricsExportClient.HttpClient.Timeout.TotalMilliseconds
117139
: options.TimeoutMilliseconds;
118140

119-
return experimentalOptions.EnableInMemoryRetry
120-
? new OtlpExporterRetryTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds)
121-
: new OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
141+
if (experimentalOptions.EnableInMemoryRetry)
142+
{
143+
return new OtlpExporterRetryTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
144+
}
145+
else if (experimentalOptions.EnableDiskRetry)
146+
{
147+
Debug.Assert(!string.IsNullOrEmpty(experimentalOptions.DiskRetryDirectoryPath), $"{nameof(experimentalOptions.DiskRetryDirectoryPath)} is null or empty");
148+
149+
return new OtlpExporterPersistentStorageTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(
150+
exportClient,
151+
timeoutMilliseconds,
152+
(byte[] data) =>
153+
{
154+
var request = new MetricsOtlpCollector.ExportMetricsServiceRequest();
155+
request.MergeFrom(data);
156+
return request;
157+
},
158+
Path.Combine(experimentalOptions.DiskRetryDirectoryPath, "metrics"));
159+
}
160+
else
161+
{
162+
return new OtlpExporterTransmissionHandler<MetricsOtlpCollector.ExportMetricsServiceRequest>(exportClient, timeoutMilliseconds);
163+
}
122164
}
123165

124166
public static OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest> GetLogsExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions)
@@ -128,9 +170,29 @@ public static THeaders GetHeaders<THeaders>(this OtlpExporterOptions options, Ac
128170
? httpLogExportClient.HttpClient.Timeout.TotalMilliseconds
129171
: options.TimeoutMilliseconds;
130172

131-
return experimentalOptions.EnableInMemoryRetry
132-
? new OtlpExporterRetryTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds)
133-
: new OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
173+
if (experimentalOptions.EnableInMemoryRetry)
174+
{
175+
return new OtlpExporterRetryTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
176+
}
177+
else if (experimentalOptions.EnableDiskRetry)
178+
{
179+
Debug.Assert(!string.IsNullOrEmpty(experimentalOptions.DiskRetryDirectoryPath), $"{nameof(experimentalOptions.DiskRetryDirectoryPath)} is null or empty");
180+
181+
return new OtlpExporterPersistentStorageTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(
182+
exportClient,
183+
timeoutMilliseconds,
184+
(byte[] data) =>
185+
{
186+
var request = new LogOtlpCollector.ExportLogsServiceRequest();
187+
request.MergeFrom(data);
188+
return request;
189+
},
190+
Path.Combine(experimentalOptions.DiskRetryDirectoryPath, "logs"));
191+
}
192+
else
193+
{
194+
return new OtlpExporterTransmissionHandler<LogOtlpCollector.ExportLogsServiceRequest>(exportClient, timeoutMilliseconds);
195+
}
134196
}
135197

136198
public static IExportClient<TraceOtlpCollector.ExportTraceServiceRequest> GetTraceExportClient(this OtlpExporterOptions options) =>

‎src/OpenTelemetry.Exporter.OpenTelemetryProtocol/README.md‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -627,10 +627,17 @@ want to solicit feedback from the community.
627627

628628
* `OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY`
629629

630-
When set to `in_memory`, it enables in-memory retry for transient errors
630+
* When set to `in_memory`, it enables in-memory retry for transient errors
631631
encountered while sending telemetry.
632632

633-
Added in `1.8.0`.
633+
Added in `1.8.0`.
634+
635+
* When set to `disk` along with setting
636+
`OTEL_DOTNET_EXPERIMENTAL_OTLP_DISK_RETRY_DIRECTORY_PATH` to the path on
637+
disk, it enables retries by storing telemetry on disk during transient
638+
errors.
639+
640+
Added in **TBD** (Unreleased).
634641

635642
* Logs
636643

‎test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpExporterOptionsExtensionsTests.cs‎

Lines changed: 53 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#if NETFRAMEWORK
55
using System.Net.Http;
66
#endif
7+
using Microsoft.Extensions.Configuration;
78
using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation;
89
using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient;
910
using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Transmission;
@@ -130,16 +131,34 @@ public void AppendPathIfNotPresent_TracesPath_AppendsCorrectly(string inputUri,
130131
}
131132

132133
[Theory]
133-
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcTraceExportClient), false, 10000)]
134-
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), false, 10000)]
135-
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), true, 8000)]
136-
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcMetricsExportClient), false, 10000)]
137-
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), false, 10000)]
138-
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), true, 8000)]
139-
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcLogExportClient), false, 10000)]
140-
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), false, 10000)]
141-
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), true, 8000)]
142-
public void GetTransmissionHandler_InitializesCorrectExportClientAndTimeoutValue(OtlpExportProtocol protocol, Type exportClientType, bool customHttpClient, int expectedTimeoutMilliseconds)
134+
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcTraceExportClient), false, 10000, null)]
135+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), false, 10000, null)]
136+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), true, 8000, null)]
137+
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcMetricsExportClient), false, 10000, null)]
138+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), false, 10000, null)]
139+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), true, 8000, null)]
140+
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcLogExportClient), false, 10000, null)]
141+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), false, 10000, null)]
142+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), true, 8000, null)]
143+
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcTraceExportClient), false, 10000, "in_memory")]
144+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), false, 10000, "in_memory")]
145+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), true, 8000, "in_memory")]
146+
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcMetricsExportClient), false, 10000, "in_memory")]
147+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), false, 10000, "in_memory")]
148+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), true, 8000, "in_memory")]
149+
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcLogExportClient), false, 10000, "in_memory")]
150+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), false, 10000, "in_memory")]
151+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), true, 8000, "in_memory")]
152+
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcTraceExportClient), false, 10000, "disk")]
153+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), false, 10000, "disk")]
154+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), true, 8000, "disk")]
155+
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcMetricsExportClient), false, 10000, "disk")]
156+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), false, 10000, "disk")]
157+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), true, 8000, "disk")]
158+
[InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcLogExportClient), false, 10000, "disk")]
159+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), false, 10000, "disk")]
160+
[InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), true, 8000, "disk")]
161+
public void GetTransmissionHandler_InitializesCorrectHandlerExportClientAndTimeoutValue(OtlpExportProtocol protocol, Type exportClientType, bool customHttpClient, int expectedTimeoutMilliseconds, string retryStrategy)
143162
{
144163
var exporterOptions = new OtlpExporterOptions() { Protocol = protocol };
145164
if (customHttpClient)
@@ -150,28 +169,45 @@ public void GetTransmissionHandler_InitializesCorrectExportClientAndTimeoutValue
150169
};
151170
}
152171

172+
var configuration = new ConfigurationBuilder()
173+
.AddInMemoryCollection(new Dictionary<string, string> { [ExperimentalOptions.OtlpRetryEnvVar] = retryStrategy })
174+
.Build();
175+
153176
if (exportClientType == typeof(OtlpGrpcTraceExportClient) || exportClientType == typeof(OtlpHttpTraceExportClient))
154177
{
155-
var transmissionHandler = exporterOptions.GetTraceExportTransmissionHandler(new ExperimentalOptions());
178+
var transmissionHandler = exporterOptions.GetTraceExportTransmissionHandler(new ExperimentalOptions(configuration));
156179

157-
AssertTransmissionHandlerProperties(transmissionHandler, exportClientType, expectedTimeoutMilliseconds);
180+
AssertTransmissionHandler(transmissionHandler, exportClientType, expectedTimeoutMilliseconds, retryStrategy);
158181
}
159182
else if (exportClientType == typeof(OtlpGrpcMetricsExportClient) || exportClientType == typeof(OtlpHttpMetricsExportClient))
160183
{
161-
var transmissionHandler = exporterOptions.GetMetricsExportTransmissionHandler(new ExperimentalOptions());
184+
var transmissionHandler = exporterOptions.GetMetricsExportTransmissionHandler(new ExperimentalOptions(configuration));
162185

163-
AssertTransmissionHandlerProperties(transmissionHandler, exportClientType, expectedTimeoutMilliseconds);
186+
AssertTransmissionHandler(transmissionHandler, exportClientType, expectedTimeoutMilliseconds, retryStrategy);
164187
}
165188
else
166189
{
167-
var transmissionHandler = exporterOptions.GetLogsExportTransmissionHandler(new ExperimentalOptions());
190+
var transmissionHandler = exporterOptions.GetLogsExportTransmissionHandler(new ExperimentalOptions(configuration));
168191

169-
AssertTransmissionHandlerProperties(transmissionHandler, exportClientType, expectedTimeoutMilliseconds);
192+
AssertTransmissionHandler(transmissionHandler, exportClientType, expectedTimeoutMilliseconds, retryStrategy);
170193
}
171194
}
172195

173-
private static void AssertTransmissionHandlerProperties<T>(OtlpExporterTransmissionHandler<T> transmissionHandler, Type exportClientType, int expectedTimeoutMilliseconds)
196+
private static void AssertTransmissionHandler<T>(OtlpExporterTransmissionHandler<T> transmissionHandler, Type exportClientType, int expectedTimeoutMilliseconds, string retryStrategy)
174197
{
198+
if (retryStrategy == "in_memory")
199+
{
200+
Assert.True(transmissionHandler is OtlpExporterRetryTransmissionHandler<T>);
201+
}
202+
else if (retryStrategy == "disk")
203+
{
204+
Assert.True(transmissionHandler is OtlpExporterPersistentStorageTransmissionHandler<T>);
205+
}
206+
else
207+
{
208+
Assert.True(transmissionHandler is OtlpExporterTransmissionHandler<T>);
209+
}
210+
175211
Assert.Equal(exportClientType, transmissionHandler.ExportClient.GetType());
176212

177213
Assert.Equal(expectedTimeoutMilliseconds, transmissionHandler.TimeoutMilliseconds);

0 commit comments

Comments
 (0)