Skip to content

Commit baa51ab

Browse files
dattipoeHMCTSdattipoeHMCTS
committed
SSCS-10722 Add tests, improve logging.
1 parent 9cc73ae commit baa51ab

File tree

5 files changed

+141
-71
lines changed

5 files changed

+141
-71
lines changed

‎modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessor.java‎

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -45,18 +45,6 @@ public class CaseMigrationProcessor {
4545
@Getter
4646
private Long totalCases = 0L;
4747

48-
@Value("${migration.parallel}")
49-
private boolean parallel;
50-
51-
@Value("${migration.pageSize}")
52-
private int pageSize;
53-
54-
@Value("${migration.numThreads}")
55-
private int numThreads;
56-
57-
@Value("${migration.maxCasesToProcess}")
58-
private int maxCasesToProcess;
59-
6048
public void processSingleCase(String userToken, String caseId, boolean dryRun) {
6149
CaseDetails caseDetails;
6250
try {
@@ -72,30 +60,32 @@ public void processSingleCase(String userToken, String caseId, boolean dryRun) {
7260
}
7361
}
7462

75-
public void fetchAndProcessCases(String userToken, boolean dryRun) throws InterruptedException {
63+
public void fetchAndProcessCases(String userToken, boolean dryRun, int numThreads, MigrationPageParams pageParams)
64+
throws InterruptedException {
7665

7766
SearchResult initialSearch = coreCaseDataService.searchCases(userToken,
7867
fetchAllUnsetCaseAccessManagementFieldsCasesQuery());
7968

80-
int totalCasesToProcess = resolveTotalCasesToProcess(initialSearch);
69+
if (initialSearch.getTotal() <= 0) {
70+
return;
71+
}
8172

8273
totalTimer.start();
83-
//Need to fix the off by one issue.
84-
Long searchFrom = handleFirstCase(userToken, dryRun, initialSearch);
8574

86-
totalCasesToProcess -= 1;
75+
int totalCasesToProcess = resolveTotalCasesToProcess(initialSearch, pageParams.getMaxCasesToProcess());
8776

88-
int casesFetched = 0;
77+
Long searchFrom = handleFirstCase(userToken, dryRun, initialSearch);
8978

9079
ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
9180

92-
fetchAndSubmitTasks(userToken, dryRun, totalCasesToProcess, searchFrom, casesFetched, executorService);
81+
fetchAndSubmitTasks(userToken, dryRun, totalCasesToProcess, pageParams.getPageSize(), searchFrom,
82+
executorService);
9383

9484
executorService.shutdown();
9585
executorService.awaitTermination(Integer.MAX_VALUE, TimeUnit.DAYS);
9686
}
9787

98-
private int resolveTotalCasesToProcess(SearchResult initialSearch) {
88+
private int resolveTotalCasesToProcess(SearchResult initialSearch, int maxCasesToProcess) {
9989
int totalCasesToProcess = 0;
10090

10191
if (maxCasesToProcess > 0) {
@@ -109,12 +99,16 @@ private int resolveTotalCasesToProcess(SearchResult initialSearch) {
10999
return totalCasesToProcess;
110100
}
111101

112-
private void fetchAndSubmitTasks(String userToken, boolean dryRun, int totalCasesToProcess, Long searchFrom, int casesFetched,
113-
ExecutorService executorService) {
102+
private void fetchAndSubmitTasks(String userToken, boolean dryRun, int totalCasesToProcess, int pageSize,
103+
Long searchFrom, ExecutorService executorService) {
104+
int casesFetched = 1;
105+
int numCasesToFetch = pageSize;
114106

115107
while (casesFetched < totalCasesToProcess) {
108+
numCasesToFetch = resolvePageSize(totalCasesToProcess, casesFetched, numCasesToFetch, pageSize);
109+
116110
List<CaseDetails> caseDetails =
117-
coreCaseDataService.fetchNCases(userToken, pageSize, searchFrom);
111+
coreCaseDataService.fetchNCases(userToken, numCasesToFetch, searchFrom);
118112

119113
if (caseDetails.isEmpty()) {
120114
break;
@@ -134,6 +128,14 @@ private void fetchAndSubmitTasks(String userToken, boolean dryRun, int totalCase
134128
}
135129
}
136130

131+
private int resolvePageSize(int totalCasesToProcess, int casesFetched, int numCasesToFetch, int pageSize) {
132+
int remainingCases = totalCasesToProcess - casesFetched;
133+
if (remainingCases < pageSize) {
134+
numCasesToFetch = remainingCases;
135+
}
136+
return numCasesToFetch;
137+
}
138+
137139
private Long handleFirstCase(String userToken, boolean dryRun, SearchResult initialSearch) {
138140
log.info("Processing first case...");
139141
CaseDetails firstCase = initialSearch.getCases().get(0);
@@ -177,7 +179,7 @@ private void updateCase(String authorisation, Long id, Map<String, Object> data,
177179
}
178180

179181
if (totalCases % 1000 == 0) {
180-
log.info("--------{} cases migrated in {} minutes ({} seconds)-------", totalCases,
182+
log.info("----------{} cases migrated in {} minutes ({} seconds)----------", totalCases,
181183
totalTimer.getTime(TimeUnit.MINUTES), totalTimer.getTime(TimeUnit.SECONDS));
182184
}
183185
}

‎modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationRunner.java‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,15 @@ public class CaseMigrationRunner implements CommandLineRunner {
2121
private String idamPassword;
2222
@Value("${migration.caseId}")
2323
private String ccdCaseId;
24-
@Value("${migration.startDate}")
25-
private String startDate;
26-
@Value("${migration.endDate}")
27-
private String endDate;
2824
@Value("${migration.dryrun}")
2925
private boolean dryrun;
26+
@Value("${migration.pageSize}")
27+
private int pageSize;
28+
@Value("${migration.maxCasesToProcess}")
29+
private int maxCasesToProcess;
30+
@Value("${migration.numThreads}")
31+
private int numThreads;
32+
3033

3134
private final IdamClient idamClient;
3235
private final CaseMigrationProcessor caseMigrationProcessor;
@@ -40,26 +43,29 @@ public void run(String... args) {
4043
try {
4144
String userToken = idamClient.authenticateUser(idamUsername, idamPassword);
4245

43-
StopWatch stopWatch = StopWatch.createStarted();
46+
MigrationPageParams pageParams = new MigrationPageParams(pageSize, maxCasesToProcess);
4447

45-
log.info(userToken);
48+
StopWatch stopWatch = StopWatch.createStarted();
4649

4750
if (ccdCaseId != null && !ccdCaseId.isBlank()) {
4851
log.info("Data migration of single case started");
4952
caseMigrationProcessor.processSingleCase(userToken, ccdCaseId, dryrun);
5053
} else {
5154
log.info("Data migration of cases started");
52-
caseMigrationProcessor.fetchAndProcessCases(userToken, dryrun);
55+
caseMigrationProcessor.fetchAndProcessCases(userToken, dryrun, numThreads, pageParams);
5356
}
5457

5558
stopWatch.stop();
5659

5760
log.info("-----------------------------------------");
5861
log.info("Total number of cases: {}", caseMigrationProcessor.getTotalCases());
59-
log.info("Total number of processed cases: {}", caseMigrationProcessor.getMigratedCases().size() + caseMigrationProcessor.getFailedCases().size());
62+
log.info("Total number of processed cases: {}", caseMigrationProcessor.getMigratedCases().size()
63+
+ caseMigrationProcessor.getFailedCases().size());
6064
log.info("Total number of migrations performed: {}", caseMigrationProcessor.getMigratedCases().size());
6165
log.info("-----------------------------------------");
62-
log.info("Migrated cases: {} ", !caseMigrationProcessor.getMigratedCases().isEmpty() ? caseMigrationProcessor.getMigratedCases() : "NONE");
66+
log.info("Migrated cases: {} ", !caseMigrationProcessor.getMigratedCases().isEmpty()
67+
? caseMigrationProcessor.getMigratedCases()
68+
: "NONE");
6369
log.info("Failed cases: {}", caseMigrationProcessor.getFailedCases().size());
6470
log.info("-----------------------------------------");
6571
log.info("Data migration completed in: {} minutes ({} seconds).",

‎modules/processor/src/main/java/uk/gov/hmcts/reform/migration/MigrationPageParams.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package uk.gov.hmcts.reform.migration;
2+
3+
4+
import lombok.Getter;
5+
import lombok.RequiredArgsConstructor;
6+
7+
@Getter
8+
@RequiredArgsConstructor
9+
public class MigrationPageParams {
10+
11+
private final int pageSize;
12+
13+
private final int maxCasesToProcess;
14+
}

‎modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataService.java‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,17 @@
11
package uk.gov.hmcts.reform.migration.ccd;
22

33
import static java.util.Collections.emptyList;
4-
import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.fetchAllUnsetCaseAccessManagementFieldsCasesQuery;
54
import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.oldestCaseQuery;
65
import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.pageForUnsetCaseAccessManagementFieldsFieldsQuery;
76

87
import feign.FeignException;
98

10-
import java.time.LocalDate;
119
import java.util.HashMap;
1210
import java.util.List;
1311
import java.util.Map;
14-
import java.util.Optional;
1512
import java.util.concurrent.TimeUnit;
1613
import java.util.stream.Collectors;
1714
import java.util.stream.IntStream;
18-
import java.util.stream.Stream;
1915

2016
import lombok.RequiredArgsConstructor;
2117
import lombok.extern.slf4j.Slf4j;
@@ -42,17 +38,12 @@ public class CoreCaseDataService {
4238

4339
private static final String SSCS_CASE_TYPE = "Benefit";
4440

45-
46-
4741
@Value("${migration.jurisdiction}")
4842
private String jurisdiction;
4943

5044
@Value("${migration.caseType}")
5145
private String caseType;
5246

53-
@Value("${migration.indexCases}")
54-
private boolean indexCases;
55-
5647
private final IdamClient idamClient;
5748
private final AuthTokenGenerator authTokenGenerator;
5849
private final CoreCaseDataApi coreCaseDataApi;

‎modules/processor/src/test/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessorTest.java‎

Lines changed: 86 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,29 @@
11
package uk.gov.hmcts.reform.migration;
22

3+
import static org.codehaus.groovy.runtime.InvokerHelper.asList;
34
import static org.hamcrest.Matchers.contains;
4-
import static org.hamcrest.Matchers.containsInAnyOrder;
55
import static org.hamcrest.Matchers.hasSize;
6-
import static org.junit.Assert.assertEquals;
6+
import static org.hamcrest.Matchers.is;
77
import static org.junit.Assert.assertThat;
8+
import static org.mockito.ArgumentMatchers.any;
9+
import static org.mockito.ArgumentMatchers.anyString;
810
import static org.mockito.Mockito.times;
911
import static org.mockito.Mockito.verify;
1012
import static org.mockito.Mockito.when;
1113

12-
import java.time.LocalDate;
14+
import java.util.ArrayList;
1315
import java.util.HashMap;
1416
import java.util.List;
1517
import java.util.Map;
16-
import org.junit.Ignore;
18+
19+
import org.elasticsearch.search.builder.SearchSourceBuilder;
1720
import org.junit.Test;
1821
import org.junit.runner.RunWith;
1922
import org.mockito.InjectMocks;
2023
import org.mockito.Mock;
2124
import org.mockito.junit.MockitoJUnitRunner;
2225
import uk.gov.hmcts.reform.ccd.client.model.CaseDetails;
26+
import uk.gov.hmcts.reform.ccd.client.model.SearchResult;
2327
import uk.gov.hmcts.reform.migration.ccd.CoreCaseDataService;
2428
import uk.gov.hmcts.reform.migration.service.DataMigrationService;
2529

@@ -120,51 +124,104 @@ public void shouldProcessNoCaseWhenNoCasesAvailable() {
120124
}
121125

122126
@Test
123-
public void shouldContainSingleDate() {
124-
String firstDate = "2021-01-01";
125-
String lastDate = "2021-01-02";
127+
public void shouldDoNothingIfNoCasesToProcess() throws InterruptedException {
128+
SearchResult result = SearchResult.builder()
129+
.cases(new ArrayList<>())
130+
.total(0).build();
131+
132+
MigrationPageParams pageParams = new MigrationPageParams(10, 10);
133+
134+
when(coreCaseDataService.searchCases(anyString(),
135+
any(SearchSourceBuilder.class))).thenReturn(result);
136+
137+
caseMigrationProcessor.fetchAndProcessCases(USER_TOKEN, false, 1, pageParams);
138+
139+
assertThat(caseMigrationProcessor.getFailedCases().size(), is(0));
140+
assertThat(caseMigrationProcessor.getMigratedCases().size(), is(0));
141+
}
142+
143+
@Test
144+
public void shouldUseOverrideIfProvided() throws InterruptedException {
145+
SearchResult result = SearchResult.builder()
146+
.cases(List.of(caseDetails1, caseDetails2, caseDetails3))
147+
.total(3).build();
148+
149+
MigrationPageParams pageParams = new MigrationPageParams(10, 2);
150+
151+
when(coreCaseDataService.searchCases(anyString(),
152+
any(SearchSourceBuilder.class))).thenReturn(result);
126153

127-
List<LocalDate> listOfDates = caseMigrationProcessor.getListOfDates(LocalDate.parse(firstDate),
128-
LocalDate.parse(lastDate));
154+
when(coreCaseDataService.fetchNCases(USER_TOKEN, 1,
155+
caseDetails1.getId())).thenReturn(List.of(caseDetails2));
129156

130-
assertEquals(1, listOfDates.size());
157+
caseMigrationProcessor.fetchAndProcessCases(USER_TOKEN, false, 1, pageParams);
158+
159+
assertThat(caseMigrationProcessor.getFailedCases().size(), is(0));
160+
assertThat(caseMigrationProcessor.getMigratedCases().size(), is(2));
131161
}
132162

133163
@Test
134-
public void shouldContainTwoDates() {
135-
String firstDate = "2021-01-01";
136-
String lastDate = "2021-01-03";
164+
public void shouldProcessAllCasesIfNoOverride() throws InterruptedException {
165+
SearchResult result = SearchResult.builder()
166+
.cases(List.of(caseDetails1, caseDetails2, caseDetails3))
167+
.total(3).build();
168+
169+
MigrationPageParams pageParams = new MigrationPageParams(10, 0);
170+
171+
when(coreCaseDataService.searchCases(anyString(),
172+
any(SearchSourceBuilder.class))).thenReturn(result);
137173

138-
List<LocalDate> listOfDates = caseMigrationProcessor.getListOfDates(LocalDate.parse(firstDate),
139-
LocalDate.parse(lastDate));
174+
when(coreCaseDataService.fetchNCases(USER_TOKEN, 2,
175+
caseDetails1.getId())).thenReturn(List.of(caseDetails2, caseDetails3));
140176

141-
assertEquals(2, listOfDates.size());
177+
caseMigrationProcessor.fetchAndProcessCases(USER_TOKEN, false, 1, pageParams);
178+
179+
assertThat(caseMigrationProcessor.getFailedCases().size(), is(0));
180+
assertThat(caseMigrationProcessor.getMigratedCases().size(), is(3));
142181
}
143182

144183
@Test
145-
public void shouldContainNormalYearOfDates() {
146-
String firstDate = "2021-01-01";
147-
String lastDate = "2022-01-01";
184+
public void shouldBreakWhenNoMoreCasesReturned() throws InterruptedException {
185+
SearchResult result = SearchResult.builder()
186+
.cases(List.of(caseDetails1, caseDetails2, caseDetails3))
187+
.total(3).build();
188+
189+
MigrationPageParams pageParams = new MigrationPageParams(10, 2);
148190

149-
List<LocalDate> listOfDates = caseMigrationProcessor.getListOfDates(LocalDate.parse(firstDate),
150-
LocalDate.parse(lastDate));
191+
when(coreCaseDataService.searchCases(any(String.class),
192+
any(SearchSourceBuilder.class))).thenReturn(result);
151193

152-
assertEquals(365, listOfDates.size());
194+
caseMigrationProcessor.fetchAndProcessCases(USER_TOKEN, false, 1, pageParams);
195+
196+
assertThat(caseMigrationProcessor.getFailedCases().size(), is(0));
197+
assertThat(caseMigrationProcessor.getMigratedCases().size(), is(1));
153198
}
154199

155200
@Test
156-
public void shouldContainLeapYearOfDates() {
157-
String firstDate = "2020-01-01";
158-
String lastDate = "2021-01-01";
201+
public void shouldSearchFromLastCaseInPreviousResult() throws InterruptedException {
202+
SearchResult result = SearchResult.builder()
203+
.cases(List.of(caseDetails1, caseDetails2, caseDetails3))
204+
.total(3).build();
205+
206+
MigrationPageParams pageParams = new MigrationPageParams(1, 0);
207+
208+
when(coreCaseDataService.searchCases(any(String.class),
209+
any(SearchSourceBuilder.class))).thenReturn(result);
210+
211+
when(coreCaseDataService.fetchNCases(USER_TOKEN, 1,
212+
caseDetails1.getId())).thenReturn(List.of(caseDetails2));
213+
214+
when(coreCaseDataService.fetchNCases(USER_TOKEN, 1,
215+
caseDetails2.getId())).thenReturn(List.of(caseDetails3));
159216

160-
List<LocalDate> listOfDates = caseMigrationProcessor.getListOfDates(LocalDate.parse(firstDate),
161-
LocalDate.parse(lastDate));
217+
caseMigrationProcessor.fetchAndProcessCases(USER_TOKEN, false, 1, pageParams);
162218

163-
assertEquals(366, listOfDates.size());
219+
assertThat(caseMigrationProcessor.getFailedCases().size(), is(0));
220+
assertThat(caseMigrationProcessor.getMigratedCases().size(), is(3));
164221
}
165222

166223
private void mockDataFetch(CaseDetails... caseDetails) {
167-
// when(coreCaseDataService.fetchAllForDay(eq(USER_TOKEN), anyString(), eq(false))).thenReturn(Optional.of(Stream.of(caseDetails)));
224+
when(coreCaseDataService.fetchAll(USER_TOKEN, USER_ID)).thenReturn(asList(caseDetails));
168225
}
169226

170227
private void mockDataUpdate(CaseDetails caseDetails) {

0 commit comments

Comments
 (0)