diff --git a/build/Build.cs b/build/Build.cs index 6fbbc8dc..e5005c83 100644 --- a/build/Build.cs +++ b/build/Build.cs @@ -254,4 +254,4 @@ Copyright The OpenTelemetry Authors under Apache License Version 2.0 // .DependsOn(RunUnitTests) // .DependsOn(RunIntegrationTests) .DependsOn(this.PackAWSDistribution); -} +} \ No newline at end of file diff --git a/exporters/AWS.Distro.OpenTelemetry.Exporter.Xray.Udp/OtlpExporterUtils.cs b/exporters/AWS.Distro.OpenTelemetry.Exporter.Xray.Udp/OtlpExporterUtils.cs index 363f9803..236ae1a4 100644 --- a/exporters/AWS.Distro.OpenTelemetry.Exporter.Xray.Udp/OtlpExporterUtils.cs +++ b/exporters/AWS.Distro.OpenTelemetry.Exporter.Xray.Udp/OtlpExporterUtils.cs @@ -7,6 +7,8 @@ using Microsoft.Extensions.Logging; using OpenTelemetry; using OpenTelemetry.Resources; +using OpenTelemetry.Logs; +using OpenTelemetry.Exporter; namespace AWS.Distro.OpenTelemetry.Exporter.Xray.Udp; @@ -16,11 +18,15 @@ public class OtlpExporterUtils private static readonly ILogger Logger = Factory.CreateLogger(); private static readonly MethodInfo? WriteTraceDataMethod; + private static readonly MethodInfo? WriteLogsDataMethod; private static readonly object? SdkLimitOptions; + private static readonly object? ExperimentalOptions; static OtlpExporterUtils() { - Type? otlpSerializerType = Type.GetType("OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Serializer.ProtobufOtlpTraceSerializer, OpenTelemetry.Exporter.OpenTelemetryProtocol"); + Type? otlpTraceSerializerType = Type.GetType("OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Serializer.ProtobufOtlpTraceSerializer, OpenTelemetry.Exporter.OpenTelemetryProtocol"); + Type? otlpLogSerializerType = Type.GetType("OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Serializer.ProtobufOtlpLogSerializer, OpenTelemetry.Exporter.OpenTelemetryProtocol"); Type? sdkLimitOptionsType = Type.GetType("OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.SdkLimitOptions, OpenTelemetry.Exporter.OpenTelemetryProtocol"); + Type? experimentalOptionsType = Type.GetType("OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExperimentalOptions, OpenTelemetry.Exporter.OpenTelemetryProtocol"); if (sdkLimitOptionsType == null) { @@ -28,13 +34,25 @@ static OtlpExporterUtils() { return; } - if (otlpSerializerType == null) + if (experimentalOptionsType == null) { - Logger.LogTrace("OtlpSerializer Type was not found"); + Logger.LogTrace("ExperimentalOptions Type was not found"); return; } - WriteTraceDataMethod = otlpSerializerType.GetMethod( + if (otlpTraceSerializerType == null) + { + Logger.LogTrace("OtlpTraceSerializer Type was not found"); + return; + } + + if (otlpLogSerializerType == null) + { + Logger.LogTrace("OtlpLogSerializer Type was not found"); + return; + } + + WriteTraceDataMethod = otlpTraceSerializerType.GetMethod( "WriteTraceData", BindingFlags.NonPublic | BindingFlags.Static, null, @@ -42,14 +60,34 @@ static OtlpExporterUtils() { { typeof(byte[]).MakeByRefType(), // ref byte[] buffer typeof(int), // int writePosition - sdkLimitOptionsType, // SdkLimitOptions + sdkLimitOptionsType, // SdkLimitOptions typeof(Resource), // Resource? typeof(Batch).MakeByRefType() // in Batch }, null) - ?? throw new MissingMethodException("WriteTraceData not found"); // :contentReference[oaicite:1]{index=1} + ?? throw new MissingMethodException("WriteTraceData not found"); + + // Get the WriteLogsData method from the ProtobufOtlpLogSerializer using reflection. "WriteLogsData" is based on the + // OpenTelemetry.Exporter.OpenTelemetryProtocol dependency found at + // https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/Serializer/ProtobufOtlpLogSerializer.cs + WriteLogsDataMethod = otlpLogSerializerType.GetMethod( + "WriteLogsData", + BindingFlags.NonPublic | BindingFlags.Static, + null, + new[] + { + typeof(byte[]).MakeByRefType(), // ref byte[] buffer + typeof(int), // int writePosition + sdkLimitOptionsType, // SdkLimitOptions + experimentalOptionsType, // ExperimentalOptions + typeof(Resource), // Resource? + typeof(Batch).MakeByRefType() // in Batch + }, + null) + ?? throw new MissingMethodException("WriteLogsData not found"); SdkLimitOptions = GetSdkLimitOptions(); + ExperimentalOptions = GetExperimentalOptions(); } // The WriteTraceData function builds writes data to the buffer byte[] object by calling private "WriteTraceData" function @@ -71,7 +109,7 @@ public static int WriteTraceData( // Pack arguments (ref/in remain by-ref in the args array) object[] args = { buffer, writePosition, SdkLimitOptions, resource!, batch! }; - // Invoke static method (null target) :contentReference[oaicite:2]{index=2} + // Invoke static method (null target) var result = (int)WriteTraceDataMethod?.Invoke(obj: null, parameters: args)!; // Unpack ref-buffer @@ -80,8 +118,35 @@ public static int WriteTraceData( return result; } - // Uses reflection to the get the SdkLimitOptions required to invoke the ToOtlpSpan function used in the - // SerializeSpans function below. More information about SdkLimitOptions can be found in this link: + // The WriteLogsData function writes log data to the buffer byte[] object by calling private "WriteLogsData" function + // using reflection. "WriteLogsData" is based on the OpenTelemetry.Exporter.OpenTelemetryProtocol dependency found at + // https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/Serializer/ProtobufOtlpLogSerializer.cs + public static int WriteLogsData( + ref byte[] buffer, + int writePosition, + Resource? resource, + in Batch batch) + { + if (SdkLimitOptions == null || ExperimentalOptions == null) + { + Logger.LogTrace("SdkLimitOptions or ExperimentalOptions Object was not found/created properly"); + return -1; + } + + // Pack arguments (ref/in remain by-ref in the args array) + object[] args = { buffer, writePosition, SdkLimitOptions, ExperimentalOptions, resource!, batch! }; + + // Invoke static method (null target) + var result = (int)WriteLogsDataMethod?.Invoke(obj: null, parameters: args)!; + + // Unpack ref-buffer + buffer = (byte[])args[0]; + + return result; + } + + // Uses reflection to get the SdkLimitOptions required to invoke the serialization functions. + // More information about SdkLimitOptions can be found in this link: // https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/SdkLimitOptions.cs#L24 private static object? GetSdkLimitOptions() { @@ -97,4 +162,21 @@ public static int WriteTraceData( object? sdkLimitOptionsInstance = Activator.CreateInstance(sdkLimitOptionsType); return sdkLimitOptionsInstance; } + + // Uses reflection to get the ExperimentalOptions required for log serialization. + // More information about ExperimentalOptions can be found in the OpenTelemetry implementation: + // https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExperimentalOptions.cs + private static object? GetExperimentalOptions() + { + Type? experimentalOptionsType = Type.GetType("OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExperimentalOptions, OpenTelemetry.Exporter.OpenTelemetryProtocol"); + if (experimentalOptionsType == null) + { + Logger.LogTrace("ExperimentalOptions Type was not found"); + return null; + } + + // Create an instance of ExperimentalOptions using the default parameterless constructor + object? experimentalOptionsInstance = Activator.CreateInstance(experimentalOptionsType); + return experimentalOptionsInstance; + } } \ No newline at end of file diff --git a/sample-applications/integration-test-app/Dockerfile b/sample-applications/integration-test-app/Dockerfile index 2d55066d..1dce7cb0 100644 --- a/sample-applications/integration-test-app/Dockerfile +++ b/sample-applications/integration-test-app/Dockerfile @@ -26,4 +26,4 @@ ENV OTEL_DOTNET_AUTO_PLUGINS="AWS.Distro.OpenTelemetry.AutoInstrumentation.Plugi ENV OTEL_AWS_APPLICATION_SIGNALS_ENABLED="true" ENV OTEL_TRACES_SAMPLER="always_on" ENV OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf" -ENV OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT="http://otel:4318/v1/metrics" +ENV OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT="http://otel:4318/v1/metrics" \ No newline at end of file diff --git a/sample-applications/integration-test-app/docker-compose.yml b/sample-applications/integration-test-app/docker-compose.yml index 8c56f577..d6bb64ae 100644 --- a/sample-applications/integration-test-app/docker-compose.yml +++ b/sample-applications/integration-test-app/docker-compose.yml @@ -31,4 +31,4 @@ services: ports: - '8080:8080' volumes: - - ~/.aws:/root/.aws:ro + - ~/.aws:/root/.aws:ro \ No newline at end of file diff --git a/sample-applications/integration-test-app/integration-test-app/Controllers/AppController.cs b/sample-applications/integration-test-app/integration-test-app/Controllers/AppController.cs index b9b3cd7f..9699c195 100644 --- a/sample-applications/integration-test-app/integration-test-app/Controllers/AppController.cs +++ b/sample-applications/integration-test-app/integration-test-app/Controllers/AppController.cs @@ -5,7 +5,7 @@ using System.Net.Http; using Amazon.S3; using Microsoft.AspNetCore.Mvc; - + namespace integration_test_app.Controllers; [ApiController] @@ -48,4 +48,4 @@ private string GetTraceId() var random = traceId.Substring(8); return "{" + "\"traceId\"" + ": " + "\"" + version + "-" + epoch + "-" + random + "\"" + "}"; } -} +} \ No newline at end of file diff --git a/sample-applications/integration-test-app/integration-test-app/integration-test-app.csproj b/sample-applications/integration-test-app/integration-test-app/integration-test-app.csproj index 57556212..1cf4fb51 100644 --- a/sample-applications/integration-test-app/integration-test-app/integration-test-app.csproj +++ b/sample-applications/integration-test-app/integration-test-app/integration-test-app.csproj @@ -14,4 +14,4 @@ - + \ No newline at end of file diff --git a/src/AWS.Distro.OpenTelemetry.AutoInstrumentation/OtlpAwsLogExporter/OtlpAwsLogExporter.cs b/src/AWS.Distro.OpenTelemetry.AutoInstrumentation/OtlpAwsLogExporter/OtlpAwsLogExporter.cs new file mode 100644 index 00000000..e78c0f45 --- /dev/null +++ b/src/AWS.Distro.OpenTelemetry.AutoInstrumentation/OtlpAwsLogExporter/OtlpAwsLogExporter.cs @@ -0,0 +1,400 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// Modifications Copyright The OpenTelemetry Authors. Licensed under the Apache License 2.0 License. + +using System.Diagnostics; +using System.Net; +using System.Net.Http; +using System.Reflection; +using System.Runtime.CompilerServices; +using Amazon; +using Amazon.Runtime; +using Amazon.Runtime.Internal; +using Amazon.Runtime.Internal.Auth; +using Amazon.XRay; +using AWS.Distro.OpenTelemetry.AutoInstrumentation.Logging; +using AWS.Distro.OpenTelemetry.Exporter.Xray.Udp; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using OpenTelemetry; +using OpenTelemetry.Exporter; +using OpenTelemetry.Logs; +using OpenTelemetry.Resources; + +#pragma warning disable CS1700 // Assembly reference is invalid and cannot be resolved +[assembly: InternalsVisibleTo("AWS.Distro.OpenTelemetry.AutoInstrumentation.Tests, PublicKey=6ba7de5ce46d6af3")] + +namespace AWS.Distro.OpenTelemetry.AutoInstrumentation; + +/// +/// This exporter OVERRIDES the Export functionality of the http/protobuf OtlpLogExporter to allow logs to be exported +/// to the CloudWatch OTLP endpoint https://logs.[AWSRegion].amazonaws.com/v1/logs. Utilizes the AWSSDK +/// library to sign and directly inject SigV4 Authentication to the exported request's headers. +/// +/// NOTE: In order to properly configure the usage of this exporter. Please make sure you have the +/// following environment variables: +/// +/// export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=https://logs.[AWSRegion].amazonaws.com/v1/logs +/// export OTEL_AWS_SIG_V4_ENABLED=true +/// export OTEL_EXPORTER_OTLP_LOGS_HEADERS=x-aws-log-group=your-log-group,x-aws-log-stream=your-log-stream +/// +/// +/// +/// For more information, see AWS documentation on CloudWatch OTLP Endpoint. +/// +public class OtlpAwsLogExporter : BaseExporter +#pragma warning restore CS1700 // Assembly reference is invalid and cannot be resolved +{ + private static readonly string ServiceName = "logs"; + private static readonly string ContentType = "application/x-protobuf"; +#pragma warning disable CS0436 // Type conflicts with imported type + private static readonly ILoggerFactory Factory = LoggerFactory.Create(builder => builder.AddProvider(new ConsoleLoggerProvider())); +#pragma warning restore CS0436 // Type conflicts with imported type + private static readonly ILogger Logger = Factory.CreateLogger(); + private static readonly string OtelExporterOtlpLogsHeadersConfig = "OTEL_EXPORTER_OTLP_LOGS_HEADERS"; + private readonly HttpClient client = new HttpClient(); + private readonly Uri endpoint; + private readonly string region; + private readonly int timeout; + private readonly Resource processResource; + private readonly Dictionary headers; + private IAwsAuthenticator authenticator; + + /// + /// Initializes a new instance of the class. + /// + /// OpenTelemetry Protocol (OTLP) exporter options. + /// Otel Resource Object + public OtlpAwsLogExporter(OtlpExporterOptions options, Resource processResource) + : this(options, processResource, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// OpenTelemetry Protocol (OTLP) exporter options. + /// Otel Resource Object + /// The authentication used to sign the request with SigV4 + internal OtlpAwsLogExporter(OtlpExporterOptions options, Resource processResource, IAwsAuthenticator? authenticator = null) + { + this.endpoint = options.Endpoint; + this.timeout = options.TimeoutMilliseconds; + + // Verified in Plugin.cs that the endpoint matches the CloudWatch endpoint format. + this.region = this.endpoint.AbsoluteUri.Split('.')[1]; + this.processResource = processResource; + this.authenticator = authenticator == null ? new DefaultAwsAuthenticator() : authenticator; + this.headers = ParseHeaders(System.Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_LOGS_HEADERS")); + } + + /// + public override ExportResult Export(in Batch batch) + { + using IDisposable scope = SuppressInstrumentationScope.Begin(); + + // Inheriting the size from upstream: https://github.com/open-telemetry/opentelemetry-dotnet/blob/24a13ab91c9c152d03fd0871bbb94e8f6ef08698/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpLogExporter.cs#L28-L31 + byte[] serializedData = new byte[750000]; + int serializedDataLength = OtlpExporterUtils.WriteLogsData(ref serializedData, 0, this.processResource, batch); + + if (serializedDataLength == -1) + { + Logger.LogError("Logs cannot be serialized"); + return ExportResult.Failure; + } + + try + { + HttpResponseMessage? message = Task.Run(() => + { + // The retry delay cannot exceed the configured timeout period for otlp exporter. + // If the backend responds with `RetryAfter` duration that would result in exceeding the configured timeout period + // we would fail and drop the data. + return RetryHelper.ExecuteWithRetryAsync(() => this.InjectSigV4AndSendAsync(serializedData, 0, serializedDataLength), TimeSpan.FromMilliseconds(this.timeout)); + }).GetAwaiter().GetResult(); + + if (message == null || message.StatusCode != HttpStatusCode.OK) + { + return ExportResult.Failure; + } + } + catch (Exception) + { + return ExportResult.Failure; + } + + return ExportResult.Success; + } + + /// + protected override bool OnShutdown(int timeoutMilliseconds) + { + return base.OnShutdown(timeoutMilliseconds); + } + + // Creates the UserAgent for the headers. See: + // https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptions.cs#L223 + private static string GetUserAgentString() + { + var assembly = typeof(OtlpExporterOptions).Assembly; + return $"OTel-OTLP-Exporter-Dotnet/{GetPackageVersion(assembly)}"; + } + + // Creates the DotNet instrumentation version for UserAgent header. See: + // https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/src/Shared/AssemblyVersionExtensions.cs#L49 + private static string GetPackageVersion(Assembly assembly) + { + var informationalVersion = assembly.GetCustomAttribute()?.InformationalVersion; + Debug.Assert(!string.IsNullOrEmpty(informationalVersion), "AssemblyInformationalVersionAttribute was not found in assembly"); + + var indexOfPlusSign = informationalVersion!.IndexOf('+'); + return indexOfPlusSign > 0 + ? informationalVersion.Substring(0, indexOfPlusSign) + : informationalVersion; + } + + private static Dictionary ParseHeaders(string? headersString) + { + var headers = new Dictionary(); + if (!string.IsNullOrEmpty(headersString)) + { + var headerPairs = headersString.Split(','); + foreach (var pair in headerPairs) + { + var keyValue = pair.Split('='); + if (keyValue.Length == 2) + { + headers[keyValue[0].Trim()] = keyValue[1].Trim(); + } + } + } + return headers; + } + + private async Task InjectSigV4AndSendAsync(byte[] serializedLogs, int offset, int serializedDataLength) + { + Logger.LogInformation("Attempting to send logs"); + if (!this.headers.TryGetValue("x-aws-log-group", out var logGroup) || + !this.headers.TryGetValue("x-aws-log-stream", out var logStream)) + { + Logger.LogError("Log group and stream must be specified in OTEL_EXPORTER_OTLP_LOGS_HEADERS"); + throw new InvalidOperationException("Missing required log group or stream headers"); + } + + Logger.LogInformation($"Using log group: {logGroup}, stream: {logStream}"); + + HttpRequestMessage httpRequest = new HttpRequestMessage(HttpMethod.Post, this.endpoint.AbsoluteUri); + IRequest sigV4Request = await this.GetSignedSigV4Request(serializedLogs, offset, serializedDataLength); + + sigV4Request.Headers.Remove("content-type"); + sigV4Request.Headers.Add("User-Agent", GetUserAgentString()); + + // Add headers from environment variable + foreach (var header in this.headers) + { + sigV4Request.Headers.Add(header.Key, header.Value); + } + + foreach (var header in sigV4Request.Headers) + { + httpRequest.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + var content = new ByteArrayContent(serializedLogs, offset, serializedDataLength); + content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(ContentType); + + httpRequest.Method = HttpMethod.Post; + httpRequest.Content = content; + + return await this.client.SendAsync(httpRequest); + } + + private async Task GetSignedSigV4Request(byte[] content, int offset, int serializedDataLength) + { + IRequest request = new DefaultRequest(new EmptyAmazonWebServiceRequest(), ServiceName) + { + HttpMethod = "POST", + ContentStream = new MemoryStream(content, offset, serializedDataLength), + Endpoint = this.endpoint, + SignatureVersion = SignatureVersion.SigV4, + }; + + AmazonXRayConfig config = new AmazonXRayConfig() + { + AuthenticationRegion = this.region, + UseHttp = false, + ServiceURL = this.endpoint.AbsoluteUri, + RegionEndpoint = RegionEndpoint.GetBySystemName(this.region), + }; + + ImmutableCredentials credentials = await this.authenticator.GetCredentialsAsync(); + + // Need to explicitly add this for using temporary security credentials from AWS STS. + // SigV4 signing library does not automatically add this header. + if (credentials.UseToken && credentials.Token != null) + { + request.Headers.Add("x-amz-security-token", credentials.Token); + } + + request.Headers.Add("Host", this.endpoint.Host); + request.Headers.Add("content-type", ContentType); + + this.authenticator.Sign(request, config, credentials); + + return request; + } + + private class EmptyAmazonWebServiceRequest : AmazonWebServiceRequest + { + } +} + +// Implementation based on: +// https://github.com/open-telemetry/opentelemetry-dotnet/blob/main/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpRetry.cs#L41 +internal class RetryHelper +{ + private const int InitialBackoffMilliseconds = 1000; + private const int MaxBackoffMilliseconds = 5000; + private const double BackoffMultiplier = 1.5; + + // This is to ensure there is no flakiness with the number of times logs are exported in the retry window. Not part of the upstream's implementation + private const int BufferWindow = 20; +#pragma warning disable CS0436 // Type conflicts with imported type + private static readonly ILoggerFactory Factory = LoggerFactory.Create(builder => builder.AddProvider(new ConsoleLoggerProvider())); +#pragma warning restore CS0436 // Type conflicts with imported type + private static readonly ILogger Logger = Factory.CreateLogger(); + +#if !NET6_0_OR_GREATER + private static readonly Random Randomizer = new Random(); +#endif + + public static async Task ExecuteWithRetryAsync( + Func> sendRequestFunc, + TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + int currentDelay = InitialBackoffMilliseconds; + HttpResponseMessage? response = null; + while (true) + { + try + { + if (HasDeadlinePassed(deadline, 0)) + { + Logger.LogDebug("Timeout of {Deadline}ms reached, stopping retries", deadline.Millisecond); + return response; + } + + // Attempt to send the http request + response = await sendRequestFunc(); + + // Stop and return the response if the status code is success or there is an unretryable status code. + if (response.IsSuccessStatusCode || !IsRetryableStatusCode(response.StatusCode)) + { + string loggingMessage = response.IsSuccessStatusCode ? $"Logs successfully exported with status code {response.StatusCode}" : $"Logs were not exported with unretryable status code: {response.StatusCode}"; + Logger.LogInformation(loggingMessage); + return response; + } + + // First check if the backend responds with a retry delay + TimeSpan? retryAfterDelay = response.Headers.RetryAfter != null ? response.Headers.RetryAfter.Delta : null; + + TimeSpan delayDuration; + + if (retryAfterDelay.HasValue) + { + delayDuration = retryAfterDelay.Value; + + try + { + currentDelay = Convert.ToInt32(retryAfterDelay.Value.TotalMilliseconds); + } + catch (OverflowException) + { + currentDelay = MaxBackoffMilliseconds; + } + } + else + { + // If no response for delay from backend we add our own jitter delay + delayDuration = TimeSpan.FromMilliseconds(GetRandomNumber(0, currentDelay)); + } + + Logger.LogDebug("Logs were not exported with status code: {StatusCode}. Checking to see if retryable again after: {DelayMilliseconds} ms", response.StatusCode, delayDuration.Milliseconds); + + // If delay exceeds deadline. We drop the http request completely. + if (HasDeadlinePassed(deadline, delayDuration.Milliseconds)) + { + Logger.LogDebug("Timeout will be reached after {Delay}ms delay. Dropping logs with status code {StatusCode}.", delayDuration.Milliseconds, response.StatusCode); + return response; + } + + currentDelay = CalculateNextRetryDelay(currentDelay); + await Task.Delay(delayDuration); + } + catch (Exception e) + { + string exceptionName = e.GetType().Name; + var delayDuration = TimeSpan.FromMilliseconds(GetRandomNumber(0, currentDelay)); + + // Handling exceptions. Same logic, we retry with custom jitter delay until it succeeds. If it fails by the time deadline is reached we drop the request completely. + if (!HasDeadlinePassed(deadline, 0)) + { + currentDelay = CalculateNextRetryDelay(currentDelay); + if (!HasDeadlinePassed(deadline, delayDuration.Milliseconds)) + { + Logger.LogDebug("{@ExceptionMessage}. Retrying again after {@Delay}ms", exceptionName, delayDuration.Milliseconds); + + await Task.Delay(delayDuration); + continue; + } + } + + Logger.LogDebug("Timeout will be reached after {Delay}ms delay. Dropping logs with exception: {@ExceptionMessage}", delayDuration.Milliseconds, e); + throw; + } + } + } + + private static bool HasDeadlinePassed(DateTime deadline, double delayDuration) + { + return DateTime.UtcNow.AddMilliseconds(delayDuration) >= + deadline.Subtract(TimeSpan.FromMilliseconds(BufferWindow)); + } + + private static int GetRandomNumber(int min, int max) + { +#if NET6_0_OR_GREATER + return Random.Shared.Next(min, max); +#else + lock (Randomizer) + { + return Randomizer.Next(min, max); + } +#endif + } + + private static bool IsRetryableStatusCode(HttpStatusCode statusCode) + { + switch (statusCode) + { +#if NETSTANDARD2_1_OR_GREATER || NET + case HttpStatusCode.TooManyRequests: +#else + case (HttpStatusCode)429: +#endif + case HttpStatusCode.BadGateway: + case HttpStatusCode.ServiceUnavailable: + case HttpStatusCode.GatewayTimeout: + return true; + default: + return false; + } + } + + private static int CalculateNextRetryDelay(int currentDelayMs) + { + var nextDelay = currentDelayMs * BackoffMultiplier; + return Convert.ToInt32(Math.Min(nextDelay, MaxBackoffMilliseconds)); + } +} \ No newline at end of file diff --git a/src/AWS.Distro.OpenTelemetry.AutoInstrumentation/Plugin.cs b/src/AWS.Distro.OpenTelemetry.AutoInstrumentation/Plugin.cs index b4f24d0c..f8cf3a2f 100644 --- a/src/AWS.Distro.OpenTelemetry.AutoInstrumentation/Plugin.cs +++ b/src/AWS.Distro.OpenTelemetry.AutoInstrumentation/Plugin.cs @@ -628,4 +628,4 @@ private int GetTracesOtlpTimeout() return DefaultOtlpTracesTimeoutMilli; } -} +} \ No newline at end of file