Skip to content

Commit 789f7d9

Browse files
phongnguyendphongnguyend
committed
(#122) Retry and Circuit Breaker Patterns
1 parent 62fe032 commit 789f7d9

File tree

13 files changed

+300
-16
lines changed

13 files changed

+300
-16
lines changed

‎src/Monolith/ClassifiedAds.Application/EmailMessages/Services/EmailMessageService.cs‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using ClassifiedAds.CrossCuttingConcerns.OS;
1+
using ClassifiedAds.CrossCuttingConcerns.CircuitBreakers;
2+
using ClassifiedAds.CrossCuttingConcerns.OS;
23
using ClassifiedAds.Domain.Notification;
34
using ClassifiedAds.Domain.Repositories;
45
using Microsoft.Extensions.Logging;
@@ -15,21 +16,26 @@ public class EmailMessageService
1516
private readonly IEmailMessageRepository _repository;
1617
private readonly IEmailNotification _emailNotification;
1718
private readonly IDateTimeProvider _dateTimeProvider;
19+
private readonly ICircuitBreakerManager _circuitBreakerManager;
1820

1921
public EmailMessageService(ILogger<EmailMessageService> logger,
2022
IEmailMessageRepository repository,
2123
IEmailNotification emailNotification,
22-
IDateTimeProvider dateTimeProvider
23-
)
24+
IDateTimeProvider dateTimeProvider,
25+
ICircuitBreakerManager circuitBreakerManager)
2426
{
2527
_logger = logger;
2628
_repository = repository;
2729
_emailNotification = emailNotification;
2830
_dateTimeProvider = dateTimeProvider;
31+
_circuitBreakerManager = circuitBreakerManager;
2932
}
3033

3134
public async Task<int> SendEmailMessagesAsync()
3235
{
36+
var circuit = _circuitBreakerManager.GetCircuitBreaker("EmailService", TimeSpan.FromMinutes(1));
37+
circuit.EnsureOkStatus();
38+
3339
var deplayedTimes = new[]
3440
{
3541
TimeSpan.FromMinutes(1),
@@ -65,11 +71,15 @@ public async Task<int> SendEmailMessagesAsync()
6571
await _emailNotification.SendAsync(email);
6672
email.SentDateTime = _dateTimeProvider.OffsetNow;
6773
email.Log += log + "Succeed.";
74+
75+
_circuitBreakerManager.LogSuccess(circuit);
6876
}
6977
catch (Exception ex)
7078
{
7179
email.Log += log + ex.ToString();
7280
email.NextAttemptDateTime = _dateTimeProvider.OffsetNow + deplayedTimes[email.AttemptCount];
81+
82+
_circuitBreakerManager.LogFailure(circuit, 5, TimeSpan.FromMinutes(5));
7383
}
7484

7585
email.AttemptCount += 1;
@@ -82,6 +92,8 @@ public async Task<int> SendEmailMessagesAsync()
8292
}
8393

8494
await _repository.UnitOfWork.SaveChangesAsync();
95+
96+
circuit.EnsureOkStatus();
8597
}
8698
}
8799
else

‎src/Monolith/ClassifiedAds.Application/SmsMessages/Services/SmsMessageService.cs‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using ClassifiedAds.CrossCuttingConcerns.OS;
1+
using ClassifiedAds.CrossCuttingConcerns.CircuitBreakers;
2+
using ClassifiedAds.CrossCuttingConcerns.OS;
23
using ClassifiedAds.Domain.Notification;
34
using ClassifiedAds.Domain.Repositories;
45
using Microsoft.Extensions.Logging;
@@ -15,20 +16,26 @@ public class SmsMessageService
1516
private readonly ISmsMessageRepository _repository;
1617
private readonly ISmsNotification _smsNotification;
1718
private readonly IDateTimeProvider _dateTimeProvider;
19+
private readonly ICircuitBreakerManager _circuitBreakerManager;
1820

1921
public SmsMessageService(ILogger<SmsMessageService> logger,
2022
ISmsMessageRepository repository,
2123
ISmsNotification smsNotification,
22-
IDateTimeProvider dateTimeProvider)
24+
IDateTimeProvider dateTimeProvider,
25+
ICircuitBreakerManager circuitBreakerManager)
2326
{
2427
_logger = logger;
2528
_repository = repository;
2629
_smsNotification = smsNotification;
2730
_dateTimeProvider = dateTimeProvider;
31+
_circuitBreakerManager = circuitBreakerManager;
2832
}
2933

3034
public async Task<int> SendSmsMessagesAsync()
3135
{
36+
var circuit = _circuitBreakerManager.GetCircuitBreaker("SmsService", TimeSpan.FromMinutes(1));
37+
circuit.EnsureOkStatus();
38+
3239
var deplayedTimes = new[]
3340
{
3441
TimeSpan.FromMinutes(1),
@@ -64,11 +71,15 @@ public async Task<int> SendSmsMessagesAsync()
6471
await _smsNotification.SendAsync(sms);
6572
sms.SentDateTime = _dateTimeProvider.OffsetNow;
6673
sms.Log += log + "Succeed.";
74+
75+
_circuitBreakerManager.LogSuccess(circuit);
6776
}
6877
catch (Exception ex)
6978
{
7079
sms.Log += log + ex.ToString();
7180
sms.NextAttemptDateTime = _dateTimeProvider.OffsetNow + deplayedTimes[sms.AttemptCount];
81+
82+
_circuitBreakerManager.LogFailure(circuit, 5, TimeSpan.FromMinutes(5));
7283
}
7384

7485
sms.AttemptCount += 1;
@@ -81,6 +92,8 @@ public async Task<int> SendSmsMessagesAsync()
8192
}
8293

8394
await _repository.UnitOfWork.SaveChangesAsync();
95+
96+
circuit.EnsureOkStatus();
8497
}
8598
}
8699
else

‎src/Monolith/ClassifiedAds.BackgroundServer/HostedServices/SendEmailHostedService.cs‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using ClassifiedAds.Application.EmailMessages.Services;
2+
using ClassifiedAds.CrossCuttingConcerns.CircuitBreakers;
23
using Microsoft.Extensions.DependencyInjection;
34
using Microsoft.Extensions.Hosting;
45
using Microsoft.Extensions.Logging;
@@ -34,14 +35,21 @@ private async Task DoWork(CancellationToken stoppingToken)
3435

3536
int rs = 0;
3637

37-
using (var scope = _services.CreateScope())
38+
try
3839
{
39-
var emailService = scope.ServiceProvider.GetRequiredService<EmailMessageService>();
40+
using (var scope = _services.CreateScope())
41+
{
42+
var emailService = scope.ServiceProvider.GetRequiredService<EmailMessageService>();
4043

41-
rs = await emailService.SendEmailMessagesAsync();
42-
}
44+
rs = await emailService.SendEmailMessagesAsync();
45+
}
4346

44-
if (rs == 0)
47+
if (rs == 0)
48+
{
49+
await Task.Delay(10000, stoppingToken);
50+
}
51+
}
52+
catch (CircuitBreakerOpenException)
4553
{
4654
await Task.Delay(10000, stoppingToken);
4755
}

‎src/Monolith/ClassifiedAds.BackgroundServer/HostedServices/SendSmsHostedService.cs‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using ClassifiedAds.Application.SmsMessages.Services;
2+
using ClassifiedAds.CrossCuttingConcerns.CircuitBreakers;
23
using Microsoft.Extensions.DependencyInjection;
34
using Microsoft.Extensions.Hosting;
45
using Microsoft.Extensions.Logging;
@@ -34,14 +35,21 @@ private async Task DoWork(CancellationToken stoppingToken)
3435

3536
int rs = 0;
3637

37-
using (var scope = _services.CreateScope())
38+
try
3839
{
39-
var smsService = scope.ServiceProvider.GetRequiredService<SmsMessageService>();
40+
using (var scope = _services.CreateScope())
41+
{
42+
var smsService = scope.ServiceProvider.GetRequiredService<SmsMessageService>();
4043

41-
rs = await smsService.SendSmsMessagesAsync();
42-
}
44+
rs = await smsService.SendSmsMessagesAsync();
45+
}
4346

44-
if (rs == 0)
47+
if (rs == 0)
48+
{
49+
await Task.Delay(10000, stoppingToken);
50+
}
51+
}
52+
catch (CircuitBreakerOpenException)
4553
{
4654
await Task.Delay(10000, stoppingToken);
4755
}

‎src/Monolith/ClassifiedAds.CrossCuttingConcerns/CircuitBreakers/CircuitBreakerOpenException.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
using System;
2+
3+
namespace ClassifiedAds.CrossCuttingConcerns.CircuitBreakers
4+
{
5+
public class CircuitBreakerOpenException : Exception
6+
{
7+
}
8+
}

‎src/Monolith/ClassifiedAds.CrossCuttingConcerns/CircuitBreakers/ICircuitBreaker.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using System;
2+
3+
namespace ClassifiedAds.CrossCuttingConcerns.CircuitBreakers
4+
{
5+
public interface ICircuitBreaker
6+
{
7+
public string Name { get; set; }
8+
9+
public CircuitStatus Status { get; set; }
10+
11+
public DateTimeOffset LastStatusUpdated { get; set; }
12+
13+
void EnsureOkStatus();
14+
}
15+
16+
public enum CircuitStatus
17+
{
18+
Closed = 1,
19+
Open = 2,
20+
HalfOpen = 3,
21+
}
22+
}

‎src/Monolith/ClassifiedAds.CrossCuttingConcerns/CircuitBreakers/ICircuitBreakerManager.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
using System;
2+
3+
namespace ClassifiedAds.CrossCuttingConcerns.CircuitBreakers
4+
{
5+
public interface ICircuitBreakerManager
6+
{
7+
ICircuitBreaker GetCircuitBreaker(string name, TimeSpan openTime);
8+
9+
void LogSuccess(ICircuitBreaker circuitBreaker);
10+
11+
void LogFailure(ICircuitBreaker circuitBreaker, int maximumNumberOfFailures, TimeSpan period);
12+
}
13+
}

‎src/Monolith/ClassifiedAds.Persistence/CircuitBreakers/CircuitBreaker.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using ClassifiedAds.CrossCuttingConcerns.CircuitBreakers;
2+
using System;
3+
using System.Collections.Generic;
4+
5+
namespace ClassifiedAds.Persistence.CircuitBreakers
6+
{
7+
public class CircuitBreaker : ICircuitBreaker
8+
{
9+
public Guid Id { get; set; }
10+
11+
public string Name { get; set; }
12+
13+
public CircuitStatus Status { get; set; }
14+
15+
public DateTimeOffset CreatedDateTime { get; set; }
16+
17+
public DateTimeOffset LastStatusUpdated { get; set; }
18+
19+
public ICollection<CircuitBreakerLog> CircuitBreakerLogs { get; set; }
20+
21+
public void EnsureOkStatus()
22+
{
23+
if (Status == CircuitStatus.Open)
24+
{
25+
throw new CircuitBreakerOpenException();
26+
}
27+
}
28+
}
29+
}

‎src/Monolith/ClassifiedAds.Persistence/CircuitBreakers/CircuitBreakerLog.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
using ClassifiedAds.CrossCuttingConcerns.CircuitBreakers;
2+
using System;
3+
4+
namespace ClassifiedAds.Persistence.CircuitBreakers
5+
{
6+
public class CircuitBreakerLog
7+
{
8+
public Guid Id { get; set; }
9+
10+
public Guid CircuitBreakerId { get; set; }
11+
12+
public CircuitStatus Status { get; set; }
13+
14+
public bool Succeeded { get; set; }
15+
16+
public DateTimeOffset CreatedDateTime { get; set; }
17+
}
18+
}

0 commit comments

Comments
 (0)