Add TelemetrySource to ExecutionRejectedException (#2346)

peter-csala · web-flow · commit 6f7edf8913b2 · 2024-10-23T12:38:59.000Z
Add `TelemetrySource` for use with `ExecutionRejectedException`.
diff --git a/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs b/src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs
@@ -95,7 +95,9 @@ public ValueTask IsolateCircuitAsync(ResilienceContext context)
 
         lock (_lock)
         {
-            SetLastHandledOutcome_NeedsLock(Outcome.FromException<T>(new IsolatedCircuitException()));
+            var exception = new IsolatedCircuitException();
+            _telemetry.SetTelemetrySource(exception);
+            SetLastHandledOutcome_NeedsLock(Outcome.FromException<T>(exception));
             OpenCircuitFor_NeedsLock(Outcome.FromResult<T>(default), TimeSpan.MaxValue, manual: true, context, out task);
             _circuitState = CircuitState.Isolated;
         }
@@ -123,7 +125,7 @@ public ValueTask CloseCircuitAsync(ResilienceContext context)
     {
         EnsureNotDisposed();
 
-        Exception? exception = null;
+        BrokenCircuitException? exception = null;
         bool isHalfOpen = false;
 
         Task? task = null;
@@ -157,6 +159,7 @@ public ValueTask CloseCircuitAsync(ResilienceContext context)
 
         if (exception is not null)
         {
+            _telemetry.SetTelemetrySource(exception);
             return Outcome.FromException<T>(exception);
         }
 
@@ -308,11 +311,13 @@ private void SetLastHandledOutcome_NeedsLock(Outcome<T> outcome)
     private BrokenCircuitException CreateBrokenCircuitException()
     {
         TimeSpan retryAfter = _blockedUntil - _timeProvider.GetUtcNow();
-        return _breakingException switch
+        var exception = _breakingException switch
         {
-            Exception exception => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter, exception),
+            Exception ex => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter, ex),
             _ => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter)
         };
+        _telemetry.SetTelemetrySource(exception);
+        return exception;
     }
 
     private void OpenCircuit_NeedsLock(Outcome<T> outcome, bool manual, ResilienceContext context, out Task? scheduledTask)
diff --git a/src/Polly.Core/ExecutionRejectedException.cs b/src/Polly.Core/ExecutionRejectedException.cs
@@ -2,6 +2,8 @@
 using System.Runtime.Serialization;
 #endif
 
+using Polly.Telemetry;
+
 namespace Polly;
 
 /// <summary>
@@ -49,4 +51,9 @@ protected ExecutionRejectedException(SerializationInfo info, StreamingContext co
     }
 #endif
 #pragma warning restore RS0016 // Add public types and members to the declared API
+
+    /// <summary>
+    /// Gets the source of the strategy which has thrown the exception, if known.
+    /// </summary>
+    public virtual ResilienceTelemetrySource? TelemetrySource { get; internal set; }
 }
diff --git a/src/Polly.Core/PublicAPI.Unshipped.txt b/src/Polly.Core/PublicAPI.Unshipped.txt
@@ -3,3 +3,5 @@ Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(string! messa
 Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(string! message, System.TimeSpan retryAfter, System.Exception! inner) -> void
 Polly.CircuitBreaker.BrokenCircuitException.BrokenCircuitException(System.TimeSpan retryAfter) -> void
 Polly.CircuitBreaker.BrokenCircuitException.RetryAfter.get -> System.TimeSpan?
+virtual Polly.ExecutionRejectedException.TelemetrySource.get -> Polly.Telemetry.ResilienceTelemetrySource?
+Polly.Telemetry.ResilienceStrategyTelemetry.SetTelemetrySource(Polly.ExecutionRejectedException! exception) -> void
diff --git a/src/Polly.Core/Telemetry/ResilienceStrategyTelemetry.cs b/src/Polly.Core/Telemetry/ResilienceStrategyTelemetry.cs
@@ -1,3 +1,5 @@
+using System.ComponentModel;
+
 namespace Polly.Telemetry;
 
 /// <summary>
@@ -21,6 +23,18 @@ internal ResilienceStrategyTelemetry(ResilienceTelemetrySource source, Telemetry
 
     internal ResilienceTelemetrySource TelemetrySource { get; }
 
+    /// <summary>
+    ///  Sets the source of the telemetry on the provided exception.
+    /// </summary>
+    /// <param name="exception">The to-be-set exception.</param>
+    [EditorBrowsable(EditorBrowsableState.Never)]
+    public void SetTelemetrySource(ExecutionRejectedException exception)
+    {
+        Guard.NotNull(exception);
+
+        exception.TelemetrySource = TelemetrySource;
+    }
+
     /// <summary>
     /// Reports an event that occurred in a resilience strategy.
     /// </summary>
diff --git a/src/Polly.Core/Timeout/TimeoutResilienceStrategy.cs b/src/Polly.Core/Timeout/TimeoutResilienceStrategy.cs
@@ -74,6 +74,7 @@ protected internal override async ValueTask<Outcome<TResult>> ExecuteCore<TResul
                 timeout,
                 e);
 
+            _telemetry.SetTelemetrySource(timeoutException);
             return Outcome.FromException<TResult>(timeoutException.TrySetStackTrace());
         }
 
diff --git a/src/Polly.RateLimiting/RateLimiterRejectedException.cs b/src/Polly.RateLimiting/RateLimiterRejectedException.cs
@@ -44,7 +44,8 @@ public RateLimiterRejectedException(string message)
     /// <param name="message">The message that describes the error.</param>
     /// <param name="retryAfter">The retry after value.</param>
     public RateLimiterRejectedException(string message, TimeSpan retryAfter)
-        : base(message) => RetryAfter = retryAfter;
+        : base(message)
+        => RetryAfter = retryAfter;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="RateLimiterRejectedException"/> class.
@@ -63,7 +64,8 @@ public RateLimiterRejectedException(string message, Exception inner)
     /// <param name="retryAfter">The retry after value.</param>
     /// <param name="inner">The inner exception.</param>
     public RateLimiterRejectedException(string message, TimeSpan retryAfter, Exception inner)
-        : base(message, inner) => RetryAfter = retryAfter;
+        : base(message, inner)
+        => RetryAfter = retryAfter;
 
     /// <summary>
     /// Gets the amount of time to wait before retrying again.
@@ -84,10 +86,10 @@ public RateLimiterRejectedException(string message, TimeSpan retryAfter, Excepti
     private RateLimiterRejectedException(SerializationInfo info, StreamingContext context)
         : base(info, context)
     {
-        var value = info.GetDouble("RetryAfter");
-        if (value >= 0.0)
+        var retryAfter = info.GetDouble(nameof(RetryAfter));
+        if (retryAfter >= 0.0)
         {
-            RetryAfter = TimeSpan.FromSeconds(value);
+            RetryAfter = TimeSpan.FromSeconds(retryAfter);
         }
     }
 
@@ -96,14 +98,7 @@ public override void GetObjectData(SerializationInfo info, StreamingContext cont
     {
         Guard.NotNull(info);
 
-        if (RetryAfter.HasValue)
-        {
-            info.AddValue("RetryAfter", RetryAfter.Value.TotalSeconds);
-        }
-        else
-        {
-            info.AddValue("RetryAfter", -1.0);
-        }
+        info.AddValue(nameof(RetryAfter), RetryAfter.HasValue ? RetryAfter.Value.TotalSeconds : -1.0);
 
         base.GetObjectData(info, context);
     }
diff --git a/src/Polly.RateLimiting/RateLimiterResilienceStrategy.cs b/src/Polly.RateLimiting/RateLimiterResilienceStrategy.cs
@@ -65,7 +65,11 @@ protected override async ValueTask<Outcome<TResult>> ExecuteCore<TResult, TState
             await OnLeaseRejected(new OnRateLimiterRejectedArguments(context, lease)).ConfigureAwait(context.ContinueOnCapturedContext);
         }
 
-        var exception = retryAfter.HasValue ? new RateLimiterRejectedException(retryAfter.Value) : new RateLimiterRejectedException();
+        var exception = retryAfter is not null
+            ? new RateLimiterRejectedException(retryAfterValue)
+            : new RateLimiterRejectedException();
+
+        _telemetry.SetTelemetrySource(exception);
 
         return Outcome.FromException<TResult>(exception.TrySetStackTrace());
     }
diff --git a/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs b/test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs
@@ -52,8 +52,9 @@ public async Task IsolateAsync_Ok()
         called.Should().BeTrue();
 
         var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
-        outcome.Value.Exception.Should().BeOfType<IsolatedCircuitException>()
-            .And.Subject.As<IsolatedCircuitException>().RetryAfter.Should().BeNull();
+        var exception = outcome.Value.Exception.Should().BeOfType<IsolatedCircuitException>().Subject;
+        exception.RetryAfter.Should().BeNull();
+        exception.TelemetrySource.Should().NotBeNull();
 
         // now close it
         await controller.CloseCircuitAsync(ResilienceContextPool.Shared.Get());
@@ -119,8 +120,9 @@ public async Task OnActionPreExecute_CircuitOpenedByValue()
         using var controller = CreateController();
 
         await OpenCircuit(controller, Outcome.FromResult(99));
-        var error = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
-        error.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();
+        var exception = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
+        exception.RetryAfter.Should().NotBeNull();
+        exception.TelemetrySource.Should().NotBeNull();
 
         GetBlockedTill(controller).Should().Be(_timeProvider.GetUtcNow() + _options.BreakDuration);
     }
@@ -149,6 +151,7 @@ await OpenCircuit(
                 stacks.Add(e.StackTrace!);
                 e.Message.Should().Be("The circuit is now open and is not allowing calls.");
                 e.RetryAfter.Should().NotBeNull();
+                e.TelemetrySource.Should().NotBeNull();
 
                 if (innerException)
                 {
@@ -206,9 +209,10 @@ public async Task OnActionPreExecute_CircuitOpenedByException()
         using var controller = CreateController();
 
         await OpenCircuit(controller, Outcome.FromException<int>(new InvalidOperationException()));
-        var error = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
-        error.InnerException.Should().BeOfType<InvalidOperationException>();
-        error.RetryAfter.Should().NotBeNull();
+        var exception = (BrokenCircuitException)(await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception!;
+        exception.InnerException.Should().BeOfType<InvalidOperationException>();
+        exception.RetryAfter.Should().NotBeNull();
+        exception.TelemetrySource.Should().NotBeNull();
     }
 
     [Fact]
@@ -261,9 +265,11 @@ public async Task OnActionPreExecute_HalfOpen()
         // act
         await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
         var error = (await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get())).Value.Exception;
-        error.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();
 
         // assert
+        var exception = error.Should().BeOfType<BrokenCircuitException>().Subject;
+        exception.RetryAfter.Should().NotBeNull();
+        exception.TelemetrySource.Should().NotBeNull();
         controller.CircuitState.Should().Be(CircuitState.HalfOpen);
         called.Should().BeTrue();
     }
@@ -465,7 +471,9 @@ public async Task OnActionFailureAsync_VoidResult_EnsureBreakingExceptionNotSet(
         // assert
         controller.LastException.Should().BeNull();
         var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
-        outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>().And.Subject.As<BrokenCircuitException>().RetryAfter.Should().NotBeNull();
+        var exception = outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>().Subject;
+        exception.RetryAfter.Should().NotBeNull();
+        exception.TelemetrySource.Should().NotBeNull();
     }
 
     [Fact]
@@ -501,8 +509,9 @@ public async Task Flow_Closed_HalfOpen_Open_HalfOpen_Closed()
         TimeSpan advanceTimeRejected = TimeSpan.FromMilliseconds(1);
         AdvanceTime(advanceTimeRejected);
         var outcome = await controller.OnActionPreExecuteAsync(ResilienceContextPool.Shared.Get());
-        outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>()
-            .And.Subject.As<BrokenCircuitException>().RetryAfter.Should().Be(_options.BreakDuration - advanceTimeRejected);
+        var exception = outcome.Value.Exception.Should().BeOfType<BrokenCircuitException>().Subject;
+        exception.RetryAfter.Should().Be(_options.BreakDuration - advanceTimeRejected);
+        exception.TelemetrySource.Should().NotBeNull();
 
         // wait and try, transition to half open
         AdvanceTime(_options.BreakDuration + _options.BreakDuration);
diff --git a/test/Polly.Core.Tests/Telemetry/ResilienceStrategyTelemetryTests.cs b/test/Polly.Core.Tests/Telemetry/ResilienceStrategyTelemetryTests.cs
@@ -1,4 +1,5 @@
 using Polly.Telemetry;
+using Polly.Timeout;
 
 namespace Polly.Core.Tests.Telemetry;
 
@@ -94,4 +95,25 @@ public void Report_NoListener_ShouldNotThrow()
            .Should()
            .NotThrow();
     }
+
+    [Fact]
+    public void SetTelemetrySource_Ok()
+    {
+        var sut = new ResilienceStrategyTelemetry(_source, null);
+        var exception = new TimeoutRejectedException();
+
+        sut.SetTelemetrySource(exception);
+
+        exception.TelemetrySource.Should().Be(_source);
+    }
+
+    [Fact]
+    public void SetTelemetrySource_ShouldThrow()
+    {
+        ExecutionRejectedException? exception = null;
+
+        _sut.Invoking(s => s.SetTelemetrySource(exception!))
+            .Should()
+            .Throw<ArgumentNullException>();
+    }
 }
diff --git a/test/Polly.Core.Tests/Timeout/TimeoutResilienceStrategyTests.cs b/test/Polly.Core.Tests/Timeout/TimeoutResilienceStrategyTests.cs
@@ -173,6 +173,27 @@ public async Task Execute_Timeout_EnsureStackTrace()
         }
     }
 
+    [Fact]
+    public async Task Execute_Timeout_EnsureTelemetrySource()
+    {
+        SetTimeout(TimeSpan.FromSeconds(2));
+        var sut = CreateSut();
+
+        var outcome = await sut.ExecuteOutcomeAsync(
+            async (c, _) =>
+            {
+                var delay = _timeProvider.Delay(TimeSpan.FromSeconds(4), c.CancellationToken);
+                _timeProvider.Advance(TimeSpan.FromSeconds(2));
+                await delay;
+
+                return Outcome.FromResult("dummy");
+            },
+            ResilienceContextPool.Shared.Get(),
+            "state");
+
+        outcome.Exception.Should().BeOfType<TimeoutRejectedException>().Subject.TelemetrySource.Should().NotBeNull();
+    }
+
     [Fact]
     public async Task Execute_Cancelled_EnsureNoTimeout()
     {
diff --git a/test/Polly.RateLimiting.Tests/RateLimiterRejectedExceptionTests.cs b/test/Polly.RateLimiting.Tests/RateLimiterRejectedExceptionTests.cs
@@ -4,19 +4,67 @@ namespace Polly.Core.Tests.Timeout;
 
 public class RateLimiterRejectedExceptionTests
 {
+    private readonly string _message = "dummy";
+    private readonly TimeSpan _retryAfter = TimeSpan.FromSeconds(4);
+
     [Fact]
     public void Ctor_Ok()
     {
-        var retryAfter = TimeSpan.FromSeconds(4);
+        var exception = new RateLimiterRejectedException();
+        exception.InnerException.Should().BeNull();
+        exception.Message.Should().Be("The operation could not be executed because it was rejected by the rate limiter.");
+        exception.RetryAfter.Should().BeNull();
+        exception.TelemetrySource.Should().BeNull();
+    }
+
+    [Fact]
+    public void Ctor_RetryAfter_Ok()
+    {
+        var exception = new RateLimiterRejectedException(_retryAfter);
+        exception.InnerException.Should().BeNull();
+        exception.Message.Should().Be($"The operation could not be executed because it was rejected by the rate limiter. It can be retried after '00:00:04'.");
+        exception.RetryAfter.Should().Be(_retryAfter);
+        exception.TelemetrySource.Should().BeNull();
+    }
+
+    [Fact]
+    public void Ctor_Message_Ok()
+    {
+        var exception = new RateLimiterRejectedException(_message);
+        exception.InnerException.Should().BeNull();
+        exception.Message.Should().Be(_message);
+        exception.RetryAfter.Should().BeNull();
+        exception.TelemetrySource.Should().BeNull();
+    }
+
+    [Fact]
+    public void Ctor_Message_RetryAfter_Ok()
+    {
+        var exception = new RateLimiterRejectedException(_message, _retryAfter);
+        exception.InnerException.Should().BeNull();
+        exception.Message.Should().Be(_message);
+        exception.RetryAfter.Should().Be(_retryAfter);
+        exception.TelemetrySource.Should().BeNull();
+    }
+
+    [Fact]
+    public void Ctor_Message_InnerException_Ok()
+    {
+        var exception = new RateLimiterRejectedException(_message, new InvalidOperationException());
+        exception.InnerException.Should().BeOfType<InvalidOperationException>();
+        exception.Message.Should().Be(_message);
+        exception.RetryAfter.Should().BeNull();
+        exception.TelemetrySource.Should().BeNull();
+    }
 
-        new RateLimiterRejectedException().Message.Should().Be("The operation could not be executed because it was rejected by the rate limiter.");
-        new RateLimiterRejectedException().RetryAfter.Should().BeNull();
-        new RateLimiterRejectedException("dummy").Message.Should().Be("dummy");
-        new RateLimiterRejectedException("dummy", new InvalidOperationException()).Message.Should().Be("dummy");
-        new RateLimiterRejectedException(retryAfter).RetryAfter.Should().Be(retryAfter);
-        new RateLimiterRejectedException(retryAfter).Message.Should().Be($"The operation could not be executed because it was rejected by the rate limiter. It can be retried after '{retryAfter}'.");
-        new RateLimiterRejectedException("dummy", retryAfter).RetryAfter.Should().Be(retryAfter);
-        new RateLimiterRejectedException("dummy", retryAfter, new InvalidOperationException()).RetryAfter.Should().Be(retryAfter);
+    [Fact]
+    public void Ctor_Message_RetryAfter_InnerException_Ok()
+    {
+        var exception = new RateLimiterRejectedException(_message, _retryAfter, new InvalidOperationException());
+        exception.InnerException.Should().BeOfType<InvalidOperationException>();
+        exception.Message.Should().Be(_message);
+        exception.RetryAfter.Should().Be(_retryAfter);
+        exception.TelemetrySource.Should().BeNull();
     }
 
 #if !NETCOREAPP
diff --git a/test/Polly.RateLimiting.Tests/RateLimiterResilienceStrategyTests.cs b/test/Polly.RateLimiting.Tests/RateLimiterResilienceStrategyTests.cs

Original file line number	Diff line number	Diff line change
`@@ -95,7 +95,9 @@ public ValueTask IsolateCircuitAsync(ResilienceContext context)`
`95`	`95`
`96`	`96`	`lock (_lock)`
`97`	`97`	`{`
`98`		`- SetLastHandledOutcome_NeedsLock(Outcome.FromException<T>(new IsolatedCircuitException()));`
	`98`	`+ var exception = new IsolatedCircuitException();`
	`99`	`+ _telemetry.SetTelemetrySource(exception);`
	`100`	`+ SetLastHandledOutcome_NeedsLock(Outcome.FromException<T>(exception));`
`99`	`101`	`OpenCircuitFor_NeedsLock(Outcome.FromResult<T>(default), TimeSpan.MaxValue, manual: true, context, out task);`
`100`	`102`	`_circuitState = CircuitState.Isolated;`
`101`	`103`	`}`
`@@ -123,7 +125,7 @@ public ValueTask CloseCircuitAsync(ResilienceContext context)`
`123`	`125`	`{`
`124`	`126`	`EnsureNotDisposed();`
`125`	`127`
`126`		`- Exception? exception = null;`
	`128`	`+ BrokenCircuitException? exception = null;`
`127`	`129`	`bool isHalfOpen = false;`
`128`	`130`
`129`	`131`	`Task? task = null;`
`@@ -157,6 +159,7 @@ public ValueTask CloseCircuitAsync(ResilienceContext context)`
`157`	`159`
`158`	`160`	`if (exception is not null)`
`159`	`161`	`{`
	`162`	`+ _telemetry.SetTelemetrySource(exception);`
`160`	`163`	`return Outcome.FromException<T>(exception);`
`161`	`164`	`}`
`162`	`165`
`@@ -308,11 +311,13 @@ private void SetLastHandledOutcome_NeedsLock(Outcome<T> outcome)`
`308`	`311`	`private BrokenCircuitException CreateBrokenCircuitException()`
`309`	`312`	`{`
`310`	`313`	`TimeSpan retryAfter = _blockedUntil - _timeProvider.GetUtcNow();`
`311`		`- return _breakingException switch`
	`314`	`+ var exception = _breakingException switch`
`312`	`315`	`{`
`313`		`- Exception exception => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter, exception),`
	`316`	`+ Exception ex => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter, ex),`
`314`	`317`	`_ => new BrokenCircuitException(BrokenCircuitException.DefaultMessage, retryAfter)`
`315`	`318`	`};`
	`319`	`+ _telemetry.SetTelemetrySource(exception);`
	`320`	`+ return exception;`
`316`	`321`	`}`
`317`	`322`
`318`	`323`	`private void OpenCircuit_NeedsLock(Outcome<T> outcome, bool manual, ResilienceContext context, out Task? scheduledTask)`
Original file line number	Diff line number	Diff line change
`@@ -74,6 +74,7 @@ protected internal override async ValueTask<Outcome<TResult>> ExecuteCore<TResul`
`74`	`74`	`timeout,`
`75`	`75`	`e);`
`76`	`76`
	`77`	`+ _telemetry.SetTelemetrySource(timeoutException);`
`77`	`78`	`return Outcome.FromException<TResult>(timeoutException.TrySetStackTrace());`
`78`	`79`	`}`
`79`	`80`
Original file line number	Diff line number	Diff line change
`@@ -65,7 +65,11 @@ protected override async ValueTask<Outcome<TResult>> ExecuteCore<TResult, TState`
`65`	`65`	`await OnLeaseRejected(new OnRateLimiterRejectedArguments(context, lease)).ConfigureAwait(context.ContinueOnCapturedContext);`
`66`	`66`	`}`
`67`	`67`
`68`		`- var exception = retryAfter.HasValue ? new RateLimiterRejectedException(retryAfter.Value) : new RateLimiterRejectedException();`
	`68`	`+ var exception = retryAfter is not null`
	`69`	`+ ? new RateLimiterRejectedException(retryAfterValue)`
	`70`	`+ : new RateLimiterRejectedException();`
	`71`	`+`
	`72`	`+ _telemetry.SetTelemetrySource(exception);`
`69`	`73`
`70`	`74`	`return Outcome.FromException<TResult>(exception.TrySetStackTrace());`
`71`	`75`	`}`