Skip to content

Commit 525f476

Browse files
stephhoustephhou
authored
fix: RetrieveCohortDistribution API updated to check if a superseded by nhs number has existing record (#1762)
* fix: Added logic to check a superseded by nhs number has an existing record. * feat: updates to branch to include Feature flag for releasing change * Tidied up comments * fix: changes made to address sonarcloud warnings * chore: made variable names clearer
1 parent e92415f commit 525f476

File tree

16 files changed

+417
-86
lines changed

16 files changed

+417
-86
lines changed

‎application/CohortManager/compose.cohort-distribution.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ services:
4444
- CohortDistributionDataServiceURL=http://cohort-distribution-data-service:7992/api/CohortDistributionDataService/
4545
- BsSelectRequestAuditDataService=http://bs-request-audit-data-service:7989/api/BsSelectRequestAuditDataService/
4646
- AcceptableLatencyThresholdMs=500
47+
- RetrieveSupersededRecordsLast=false
4748

4849
retrieve-cohort-request-audit:
4950
container_name: retrieve-cohort-request-audit

‎application/CohortManager/src/Functions/CohortDistributionServices/RetrieveCohortDistribution/Program.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
.ConfigureFunctionsWorkerDefaults()
2121
.ConfigureServices(services =>
2222
{
23+
services.AddTransient<IExtractCohortDistributionRecordsStrategy, ExtractCohortDistributionRecords>();
2324
services.AddTransient<ICreateCohortDistributionData, CreateCohortDistributionData>();
2425
services.AddSingleton<ICreateResponse, CreateResponse>();
2526
services.AddSingleton<IDatabaseHelper, DatabaseHelper>();

‎application/CohortManager/src/Functions/CohortDistributionServices/RetrieveCohortDistribution/RetrieveCohortDistribution.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ public async Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymou
5656
List<CohortDistributionParticipantDto> cohortDistributionParticipants;
5757
try
5858
{
59-
6059
int rowCount = _config.MaxRowCount;
60+
bool retrieveSupersededRecordsLast = _config.RetrieveSupersededRecordsLast;
6161
if (!string.IsNullOrEmpty(req.Query["rowCount"]) && int.TryParse(req.Query["rowCount"], out int rowCountParam))
6262
{
6363
rowCount = Math.Min(rowCount, rowCountParam);
@@ -67,7 +67,7 @@ public async Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymou
6767
if (string.IsNullOrEmpty(req.Query["requestId"]))
6868
{
6969
cohortDistributionParticipants = await _createCohortDistributionData
70-
.GetUnextractedCohortDistributionParticipants(rowCount);
70+
.GetUnextractedCohortDistributionParticipants(rowCount, retrieveSupersededRecordsLast);
7171

7272
return CreateResponse(cohortDistributionParticipants, req);
7373
}
@@ -87,7 +87,7 @@ public async Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymou
8787
return CreateResponse(cohortDistributionParticipants, req);
8888
}
8989

90-
cohortDistributionParticipants = await _createCohortDistributionData.GetUnextractedCohortDistributionParticipants(rowCount);
90+
cohortDistributionParticipants = await _createCohortDistributionData.GetUnextractedCohortDistributionParticipants(rowCount, retrieveSupersededRecordsLast);
9191

9292
return CreateResponse(cohortDistributionParticipants, req);
9393

‎application/CohortManager/src/Functions/CohortDistributionServices/RetrieveCohortDistribution/RetrieveCohortDistributionConfig.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,6 @@ public class RetrieveCohortDistributionConfig
1212
[Required]
1313
public string BsSelectRequestAuditDataService { get; set; }
1414
public int MaxRowCount { get; set; } = 1_000;
15+
public bool RetrieveSupersededRecordsLast { get; set; } = false;
1516

1617
}

‎application/CohortManager/src/Functions/Shared/Common/Interfaces/ICreateCohortDistributionData.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ namespace Common.Interfaces;
55

66
public interface ICreateCohortDistributionData
77
{
8-
Task<List<CohortDistributionParticipantDto>> GetUnextractedCohortDistributionParticipants(int rowCount);
8+
Task<List<CohortDistributionParticipantDto>> GetUnextractedCohortDistributionParticipants(int rowCount, bool retrieveSupersededRecordsLast);
99
Task<List<CohortDistributionParticipantDto>> GetCohortDistributionParticipantsByRequestId(Guid requestId);
1010
Task<List<CohortRequestAudit>> GetCohortRequestAudit(string? requestId, string? statusCode, DateTime? dateFrom);
1111
Task<CohortRequestAudit> GetNextCohortRequestAudit(Guid requestId);

‎application/CohortManager/src/Functions/Shared/Common/Interfaces/IExtractCohortDistributionRecordsStrategy.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
namespace Common.Interfaces;
2+
3+
using Model;
4+
5+
/// <summary>
6+
/// Strategy interface for extracting cohort distribution participants.
7+
/// </summary>
8+
public interface IExtractCohortDistributionRecordsStrategy
9+
{
10+
/// <summary>
11+
/// Gets unextracted cohort distribution participants using the strategy's specific logic.
12+
/// </summary>
13+
/// <param name="rowCount">Maximum number of participants to extract</param>
14+
/// <param name="retrieveSupersededRecordsLast">Flag to determine if superseded records should be retrieved last</param>
15+
/// <returns>List of cohort distribution entities to be extracted</returns>
16+
Task<List<CohortDistribution>> GetUnextractedParticipants(int rowCount, bool retrieveSupersededRecordsLast);
17+
}

‎application/CohortManager/src/Functions/Shared/Data/Database/CreateCohortDistributionData.cs‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,35 @@ namespace Data.Database;
1414
public class CreateCohortDistributionData : ICreateCohortDistributionData
1515
{
1616
private readonly IDataServiceClient<CohortDistribution> _cohortDistributionDataServiceClient;
17-
1817
private readonly IDataServiceClient<BsSelectRequestAudit> _bsSelectRequestAuditDataServiceClient;
18+
private readonly IExtractCohortDistributionRecordsStrategy? _extractionStrategy;
1919

20-
public CreateCohortDistributionData(IDataServiceClient<CohortDistribution> cohortDistributionDataServiceClient, IDataServiceClient<BsSelectRequestAudit> bsSelectRequestAuditDataServiceClient)
20+
public CreateCohortDistributionData(
21+
IDataServiceClient<CohortDistribution> cohortDistributionDataServiceClient,
22+
IDataServiceClient<BsSelectRequestAudit> bsSelectRequestAuditDataServiceClient,
23+
IExtractCohortDistributionRecordsStrategy? extractionStrategy = null)
2124
{
2225
_cohortDistributionDataServiceClient = cohortDistributionDataServiceClient;
2326
_bsSelectRequestAuditDataServiceClient = bsSelectRequestAuditDataServiceClient;
27+
_extractionStrategy = extractionStrategy;
28+
2429
}
2530

2631

27-
public async Task<List<CohortDistributionParticipantDto>> GetUnextractedCohortDistributionParticipants(int rowCount)
32+
public async Task<List<CohortDistributionParticipantDto>> GetUnextractedCohortDistributionParticipants(int rowCount, bool retrieveSupersededRecordsLast)
2833
{
29-
var participantsList = await _cohortDistributionDataServiceClient.GetByFilter(x => x.IsExtracted.Equals(0) && x.RequestId == Guid.Empty);
34+
List<CohortDistribution> participantsToBeExtracted;
35+
// Use new extraction logic if environment variable is set and strategy is injected
36+
if (retrieveSupersededRecordsLast && _extractionStrategy != null)
37+
{
38+
participantsToBeExtracted = await _extractionStrategy.GetUnextractedParticipants(rowCount, retrieveSupersededRecordsLast);
39+
}
40+
else
41+
{
42+
var participantsList = await _cohortDistributionDataServiceClient.GetByFilter(x => x.IsExtracted.Equals(0) && x.RequestId == Guid.Empty);
43+
participantsToBeExtracted = participantsList.OrderBy(x => x.RecordUpdateDateTime ?? x.RecordInsertDateTime).Take(rowCount).ToList();
44+
}
3045

31-
var participantsToBeExtracted = participantsList.OrderBy(x => x.RecordUpdateDateTime ?? x.RecordInsertDateTime).Take(rowCount).ToList();
3246
//TODO do this filtering on the data services
3347
var CohortDistributionParticipantList = participantsToBeExtracted.Select(x => new CohortDistributionParticipant(x)).ToList();
3448

@@ -49,7 +63,6 @@ public async Task<List<CohortDistributionParticipantDto>> GetUnextractedCohortDi
4963

5064
public async Task<List<CohortDistributionParticipantDto>> GetCohortDistributionParticipantsByRequestId(Guid requestId)
5165
{
52-
var requestIdString = requestId.ToString();
5366
if (requestId == Guid.Empty)
5467
{
5568
CohortDistributionParticipantDto(new List<CohortDistributionParticipant>());
@@ -141,16 +154,14 @@ private async Task<bool> MarkCohortDistributionParticipantsAsExtracted(List<Coho
141154

142155
var extractedParticipants = cohortParticipants.Select(x => x.CohortDistributionId);
143156

144-
145-
146157
foreach (var participantId in extractedParticipants)
147158
{
148-
149159
var participant = await _cohortDistributionDataServiceClient.GetSingle(participantId.ToString());
150160
participant.IsExtracted = 1;
151161
participant.RequestId = requestId;
152162

153163
var updatedRecord = await _cohortDistributionDataServiceClient.Update(participant);
164+
154165
if (!updatedRecord)
155166
{
156167
return false;
@@ -205,8 +216,6 @@ private List<CohortRequestAudit> BuildCohortRequestAudits(IEnumerable<BsSelectRe
205216
}).OrderBy(x => x.CreatedDateTime).ToList();
206217
}
207218

208-
209-
210219
private static Expression<Func<BsSelectRequestAudit, bool>> BuildCohortRequestAuditQuery(string? requestId, string? statusCode, DateTime? dateFrom)
211220
{
212221
var conditions = new List<Expression<Func<BsSelectRequestAudit, bool>>>();
@@ -231,7 +240,6 @@ private static Expression<Func<BsSelectRequestAudit, bool>> BuildCohortRequestAu
231240
conditions.Add(predicate);
232241
}
233242

234-
235243
Expression<Func<BsSelectRequestAudit, bool>> finalPredicate;
236244
if (conditions.Count > 0)
237245
{
@@ -253,7 +261,6 @@ private static Expression<Func<T, bool>> CombineWithAnd<T>(List<Expression<Func<
253261
return firstExpression;
254262
}
255263

256-
257264
var body = expressions
258265
.Skip(1)
259266
.Aggregate(

‎application/CohortManager/src/Functions/Shared/Data/Database/ExtractCohortDistributionRecords.cs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
namespace Data.Database;
2+
3+
using Common.Interfaces;
4+
using DataServices.Client;
5+
using Model;
6+
7+
/// <summary>
8+
/// Extract Cohort Distribution Records without superseded nhs by nhs number first, if none found, get records with superseded by nhs number
9+
/// </summary>
10+
public class ExtractCohortDistributionRecords : IExtractCohortDistributionRecordsStrategy
11+
{
12+
private readonly IDataServiceClient<CohortDistribution> _cohortDistributionDataServiceClient;
13+
14+
public async Task<List<CohortDistribution>> GetUnextractedParticipants(int rowCount, bool retrieveSupersededRecordsLast)
15+
{
16+
// Try unextracted participants without superseded by nhs number first, if none found, get records with superseded by nhs number
17+
return await GetRegularUnextractedParticipants(rowCount)
18+
?? await GetSupersededParticipants(rowCount);
19+
}
20+
21+
public ExtractCohortDistributionRecords(IDataServiceClient<CohortDistribution> cohortDistributionDataServiceClient)
22+
{
23+
_cohortDistributionDataServiceClient = cohortDistributionDataServiceClient;
24+
}
25+
26+
private async Task<List<CohortDistribution>?> GetRegularUnextractedParticipants(int rowCount)
27+
{
28+
var unextractedParticipants = await _cohortDistributionDataServiceClient.GetByFilter(
29+
participant => participant.IsExtracted.Equals(0) && participant.RequestId == Guid.Empty && participant.SupersededNHSNumber == null);
30+
31+
return unextractedParticipants.Any()
32+
? OrderAndTakeParticipants(unextractedParticipants, rowCount)
33+
: null;
34+
}
35+
36+
private async Task<List<CohortDistribution>> GetSupersededParticipants(int rowCount)
37+
{
38+
var supersededParticipants = await _cohortDistributionDataServiceClient.GetByFilter(
39+
participant => participant.IsExtracted.Equals(0) && participant.RequestId == Guid.Empty && participant.SupersededNHSNumber != null);
40+
41+
// Get distinct non-null superseded NHS numbers
42+
var supersededNhsNumbers = supersededParticipants
43+
.Select(sp => sp.SupersededNHSNumber.Value)
44+
.Distinct()
45+
.ToList();
46+
47+
// Find matching extracted participants
48+
var matchingParticipants = new List<CohortDistribution>();
49+
foreach (var nhsNumber in supersededNhsNumbers)
50+
{
51+
var matches = await _cohortDistributionDataServiceClient.GetByFilter(
52+
participant => participant.NHSNumber == nhsNumber && participant.IsExtracted.Equals(1));
53+
matchingParticipants.AddRange(matches);
54+
}
55+
56+
// Filter superseded participants that have matching records
57+
var filteredParticipants = supersededParticipants
58+
.Where(sp => matchingParticipants.Any(mp => mp.NHSNumber == sp.SupersededNHSNumber))
59+
.ToList();
60+
61+
return OrderAndTakeParticipants(filteredParticipants, rowCount);
62+
}
63+
64+
private static List<CohortDistribution> OrderAndTakeParticipants(IEnumerable<CohortDistribution> participants, int rowCount)
65+
{
66+
return participants
67+
.OrderBy(participant => (participant.RecordUpdateDateTime ?? participant.RecordInsertDateTime) ?? DateTime.MinValue)
68+
.Take(rowCount)
69+
.ToList();
70+
}
71+
}

‎infrastructure/tf-core/environments/development.tfvars‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -671,6 +671,7 @@ function_apps = {
671671
]
672672
env_vars_static = {
673673
AcceptableLatencyThresholdMs = "500"
674+
RetrieveSupersededRecordsLast = "false"
674675
}
675676
}
676677

‎infrastructure/tf-core/environments/integration.tfvars‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -672,6 +672,7 @@ function_apps = {
672672
]
673673
env_vars_static = {
674674
AcceptableLatencyThresholdMs = "500"
675+
RetrieveSupersededRecordsLast = "false"
675676
}
676677
}
677678

0 commit comments

Comments
 (0)