Skip to content

Commit 6d5e535

Browse files
andreasohlundramonsmits
andreasohlund
and
ramonsmits
authored
Add backoff timeout when the async timer fails (#4891)
* Cleanup * Add token to timer stop * Add exponential backoff delay * Use min * Use 10, 20, 30..60 as back off instead * use task.wait * Apply suggestions from code review Co-authored-by: Ramon Smits <[email protected]> --------- Co-authored-by: Ramon Smits <[email protected]>
1 parent 6e331ee commit 6d5e535

File tree

6 files changed

+91
-98
lines changed

6 files changed

+91
-98
lines changed

‎src/ServiceControl.Infrastructure/AsyncTimer.cs‎

Lines changed: 84 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,96 +1,108 @@
1-
namespace ServiceControl.Infrastructure.BackgroundTasks
1+
namespace ServiceControl.Infrastructure.BackgroundTasks;
2+
3+
using System;
4+
using System.Threading;
5+
using System.Threading.Tasks;
6+
7+
public enum TimerJobExecutionResult
28
{
3-
using System;
4-
using System.Threading;
5-
using System.Threading.Tasks;
9+
ScheduleNextExecution,
10+
ExecuteImmediately,
11+
DoNotContinueExecuting
12+
}
613

7-
public enum TimerJobExecutionResult
14+
public class TimerJob
15+
{
16+
public TimerJob(Func<CancellationToken, Task<TimerJobExecutionResult>> callback, TimeSpan due, TimeSpan interval, Action<Exception> errorCallback)
817
{
9-
ScheduleNextExecution,
10-
ExecuteImmediately,
11-
DoNotContinueExecuting
12-
}
18+
tokenSource = new CancellationTokenSource();
19+
var token = tokenSource.Token;
1320

14-
public class TimerJob
15-
{
16-
public TimerJob(Func<CancellationToken, Task<TimerJobExecutionResult>> callback, TimeSpan due, TimeSpan interval, Action<Exception> errorCallback)
21+
task = Task.Run(async () =>
1722
{
18-
tokenSource = new CancellationTokenSource();
19-
var token = tokenSource.Token;
20-
21-
task = Task.Run(async () =>
23+
try
2224
{
23-
try
24-
{
25-
await Task.Delay(due, token).ConfigureAwait(false);
25+
await Task.Delay(due, token).ConfigureAwait(false);
26+
27+
var consecutiveFailures = 0;
2628

27-
while (!token.IsCancellationRequested)
29+
while (!token.IsCancellationRequested)
30+
{
31+
try
2832
{
29-
try
30-
{
31-
var result = await callback(token).ConfigureAwait(false);
32-
if (result == TimerJobExecutionResult.DoNotContinueExecuting)
33-
{
34-
tokenSource.Cancel();
35-
}
36-
else if (result == TimerJobExecutionResult.ScheduleNextExecution)
37-
{
38-
await Task.Delay(interval, token).ConfigureAwait(false);
39-
}
40-
41-
//Otherwise execute immediately
42-
}
43-
catch (OperationCanceledException) when (token.IsCancellationRequested)
33+
var result = await callback(token).ConfigureAwait(false);
34+
35+
consecutiveFailures = 0;
36+
if (result == TimerJobExecutionResult.DoNotContinueExecuting)
4437
{
45-
break;
38+
tokenSource.Cancel();
4639
}
47-
catch (Exception ex)
40+
else if (result == TimerJobExecutionResult.ScheduleNextExecution)
4841
{
49-
errorCallback(ex);
42+
await Task.Delay(interval, token).ConfigureAwait(false);
5043
}
44+
45+
//Otherwise execute immediately
5146
}
52-
}
53-
catch (OperationCanceledException) when (token.IsCancellationRequested)
54-
{
55-
// no-op
56-
}
57-
}, CancellationToken.None);
58-
}
47+
catch (OperationCanceledException) when (token.IsCancellationRequested)
48+
{
49+
break;
50+
}
51+
catch (Exception ex)
52+
{
53+
consecutiveFailures++;
54+
const int MaxDelayDurationInSeconds = 60;
55+
var delayInSeconds = consecutiveFailures * 10;
56+
var backoffDelay = TimeSpan.FromSeconds(int.Min(MaxDelayDurationInSeconds, delayInSeconds));
5957

60-
public async Task Stop()
61-
{
62-
if (tokenSource == null)
58+
await Task.Delay(backoffDelay, token).ConfigureAwait(false);
59+
60+
errorCallback(ex);
61+
}
62+
}
63+
}
64+
catch (OperationCanceledException) when (token.IsCancellationRequested)
6365
{
64-
return;
66+
// no-op
6567
}
68+
}, CancellationToken.None);
69+
}
6670

67-
await tokenSource.CancelAsync().ConfigureAwait(false);
68-
tokenSource.Dispose();
71+
public async Task Stop(CancellationToken cancellationToken)
72+
{
73+
if (tokenSource == null)
74+
{
75+
return;
76+
}
6977

70-
if (task != null)
71-
{
72-
try
73-
{
74-
await task.ConfigureAwait(false);
75-
}
76-
catch (OperationCanceledException) when (tokenSource.IsCancellationRequested)
77-
{
78-
//NOOP
79-
}
80-
}
78+
await tokenSource.CancelAsync().ConfigureAwait(false);
79+
tokenSource.Dispose();
80+
81+
if (task == null)
82+
{
83+
return;
8184
}
8285

83-
Task task;
84-
CancellationTokenSource tokenSource;
86+
try
87+
{
88+
await task.WaitAsync(cancellationToken).ConfigureAwait(false);
89+
}
90+
catch (OperationCanceledException) when (tokenSource.IsCancellationRequested)
91+
{
92+
//NOOP
93+
}
8594
}
8695

87-
public interface IAsyncTimer
88-
{
89-
TimerJob Schedule(Func<CancellationToken, Task<TimerJobExecutionResult>> callback, TimeSpan due, TimeSpan interval, Action<Exception> errorCallback);
90-
}
96+
readonly Task task;
97+
readonly CancellationTokenSource tokenSource;
98+
}
9199

92-
public class AsyncTimer : IAsyncTimer
93-
{
94-
public TimerJob Schedule(Func<CancellationToken, Task<TimerJobExecutionResult>> callback, TimeSpan due, TimeSpan interval, Action<Exception> errorCallback) => new TimerJob(callback, due, interval, errorCallback);
95-
}
100+
public interface IAsyncTimer
101+
{
102+
TimerJob Schedule(Func<CancellationToken, Task<TimerJobExecutionResult>> callback, TimeSpan due, TimeSpan interval, Action<Exception> errorCallback);
103+
}
104+
105+
public class AsyncTimer : IAsyncTimer
106+
{
107+
public TimerJob Schedule(Func<CancellationToken, Task<TimerJobExecutionResult>> callback, TimeSpan due, TimeSpan interval, Action<Exception> errorCallback) => new(callback, due, interval, errorCallback);
96108
}

‎src/ServiceControl.Monitoring/Licensing/LicenseCheckHostedService.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ public Task StartAsync(CancellationToken cancellationToken)
2020
return Task.CompletedTask;
2121
}
2222

23-
public Task StopAsync(CancellationToken cancellationToken) => timer.Stop();
23+
public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(cancellationToken);
2424

2525
TimerJob timer;
2626

‎src/ServiceControl/CustomChecks/InternalCustomChecks/InternalCustomCheckManager.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ async Task<TimerJobExecutionResult> Run(CancellationToken cancellationToken)
6767
: TimerJobExecutionResult.DoNotContinueExecuting;
6868
}
6969

70-
public Task Stop() => timer?.Stop() ?? Task.CompletedTask;
70+
public Task Stop() => timer?.Stop(CancellationToken.None) ?? Task.CompletedTask;
7171

7272
TimerJob timer;
7373
readonly ICustomCheck check;

‎src/ServiceControl/Licensing/LicenseCheckHostedService.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ public Task StartAsync(CancellationToken cancellationToken)
2121
return Task.CompletedTask;
2222
}
2323

24-
public Task StopAsync(CancellationToken cancellationToken) => timer.Stop();
24+
public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(cancellationToken);
2525

2626
TimerJob timer;
2727

‎src/ServiceControl/Monitoring/HeartbeatMonitoringHostedService.cs‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,7 @@ public async Task StartAsync(CancellationToken cancellationToken)
2424
timer = scheduler.Schedule(_ => CheckEndpoints(), TimeSpan.Zero, TimeSpan.FromSeconds(5), e => { log.Error("Exception occurred when monitoring endpoint instances", e); });
2525
}
2626

27-
public async Task StopAsync(CancellationToken cancellationToken)
28-
{
29-
try
30-
{
31-
await timer.Stop();
32-
}
33-
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
34-
{
35-
//NOOP, invoked Stop does not
36-
}
37-
}
27+
public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(cancellationToken);
3828

3929
async Task<TimerJobExecutionResult> CheckEndpoints()
4030
{

‎src/ServiceControl/Recoverability/RecoverabilityComponent.cs‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -174,10 +174,7 @@ public Task StartAsync(CancellationToken cancellationToken)
174174
return Task.CompletedTask;
175175
}
176176

177-
public Task StopAsync(CancellationToken cancellationToken)
178-
{
179-
return timer?.Stop() ?? Task.CompletedTask;
180-
}
177+
public Task StopAsync(CancellationToken cancellationToken) => timer?.Stop(cancellationToken) ?? Task.CompletedTask;
181178

182179
async Task<TimerJobExecutionResult> ProcessRequestedBulkRetryOperations()
183180
{
@@ -237,10 +234,7 @@ public Task StartAsync(CancellationToken cancellationToken)
237234
return Task.CompletedTask;
238235
}
239236

240-
public Task StopAsync(CancellationToken cancellationToken)
241-
{
242-
return timer.Stop();
243-
}
237+
public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(cancellationToken);
244238

245239
TimerJob timer;
246240
readonly IAsyncTimer scheduler;
@@ -266,10 +260,7 @@ public Task StartAsync(CancellationToken cancellationToken)
266260
return Task.CompletedTask;
267261
}
268262

269-
public async Task StopAsync(CancellationToken cancellationToken)
270-
{
271-
await timer.Stop();
272-
}
263+
public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(cancellationToken);
273264

274265
async Task<TimerJobExecutionResult> Process(CancellationToken cancellationToken)
275266
{

0 commit comments

Comments
 (0)