diff --git a/build.gradle b/build.gradle index 47d96eca..7db15fc2 100644 --- a/build.gradle +++ b/build.gradle @@ -239,7 +239,7 @@ subprojects { apply plugin: 'com.jfrog.bintray' group = 'uk.gov.hmcts.reform.ccd-case-migration' - version = '3.0.0' + version = 'DEV-SNAPSHOT' sourceCompatibility = '11' targetCompatibility = '11' @@ -257,6 +257,10 @@ subprojects { mavenCentral() jcenter() + // jitpack should be last resort + // see: https://github.com/jitpack/jitpack.io/issues/1939 + maven { url 'https://jitpack.io' } + maven { url "https://dl.bintray.com/hmcts/hmcts-maven" } maven { url "https://repo.maven.apache.org/maven2" } } diff --git a/modules/processor/build.gradle b/modules/processor/build.gradle index d1fe5d70..289adde9 100644 --- a/modules/processor/build.gradle +++ b/modules/processor/build.gradle @@ -5,13 +5,14 @@ plugins { mainClassName = 'uk.gov.hmcts.reform.migration.CaseMigrationRunner' dependencies { - implementation group: 'org.springframework', name: 'spring-context-support' - implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-parameter-names' - implementation group: 'uk.gov.hmcts.reform', name: 'java-logging', version: '5.0.1' - implementation group: 'uk.gov.hmcts.reform', name: 'idam-client', version: '1.0.2' - implementation group: 'uk.gov.hmcts.reform', name: 'service-auth-provider-client', version: '3.0.0' - implementation group: 'uk.gov.hmcts.reform', name: 'core-case-data-store-client', version: '4.7.3' - implementation group: 'io.rest-assured', name: 'rest-assured', version: '3.3.0' + compile group: 'org.springframework', name: 'spring-context-support' + compile group: 'com.fasterxml.jackson.module', name: 'jackson-module-parameter-names' + compile group: 'uk.gov.hmcts.reform', name: 'java-logging', version: '5.0.1' + compile group: 'uk.gov.hmcts.reform', name: 'idam-client', version: '1.0.2' + compile group: 'uk.gov.hmcts.reform', name: 'service-auth-provider-client', version: '3.0.0' + compile group: 'com.github.hmcts', name: 'ccd-client', version: '4.8.6' + compile group: 'org.elasticsearch', name: 'elasticsearch', version: '7.16.3' + compile group: 'io.rest-assured', name: 'rest-assured', version: '3.3.0' testImplementation group: 'org.springframework.boot', name: 'spring-boot-starter-test' testImplementation group: 'com.github.tomakehurst', name: 'wiremock', version: '2.23.2' diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessor.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessor.java index bf11646c..cf5df025 100644 --- a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessor.java +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessor.java @@ -1,65 +1,190 @@ package uk.gov.hmcts.reform.migration; +import java.time.LocalDate; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.time.StopWatch; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import uk.gov.hmcts.reform.ccd.client.model.CaseDetails; -import uk.gov.hmcts.reform.migration.service.DataMigrationService; +import uk.gov.hmcts.reform.ccd.client.model.SearchResult; import uk.gov.hmcts.reform.migration.ccd.CoreCaseDataService; +import uk.gov.hmcts.reform.migration.service.DataMigrationService; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.fetchAllUnsetCaseAccessManagementFieldsCasesQuery; -@Slf4j @Component +@RequiredArgsConstructor public class CaseMigrationProcessor { + + private final Logger errorLogger = LoggerFactory.getLogger("ccd-migration-error"); + + private final Logger infoLogger = LoggerFactory.getLogger("ccd-migration-info"); + private static final String EVENT_ID = "migrateCase"; private static final String EVENT_SUMMARY = "Migrate Case"; private static final String EVENT_DESCRIPTION = "Migrate Case"; - @Autowired - private CoreCaseDataService coreCaseDataService; - @Autowired - private DataMigrationService dataMigrationService; + private final StopWatch totalTimer = new StopWatch(); + + private final CoreCaseDataService coreCaseDataService; + private final DataMigrationService dataMigrationService; + + @Getter + private final Set migratedCases = new HashSet<>(); + @Getter - private List migratedCases = new ArrayList<>(); + private final Set failedCases = new HashSet<>(); + @Getter - private List failedCases = new ArrayList<>(); + private Long totalCases = 0L; - public void processSingleCase(String userToken, String caseId) { + public void processSingleCase(String userToken, String caseId, boolean dryRun) { CaseDetails caseDetails; try { caseDetails = coreCaseDataService.fetchOne(userToken, caseId); } catch (Exception ex) { - log.error("Case {} not found due to: {}", caseId, ex.getMessage()); + errorLogger.error("Case {} not found due to: {}", caseId, ex.getMessage()); return; } if (dataMigrationService.accepts().test(caseDetails)) { - updateCase(userToken, caseDetails.getId(), caseDetails.getData()); + updateCase(userToken, caseDetails.getId(), caseDetails.getData(), dryRun); + } else { + infoLogger.info("Case {} already migrated", caseDetails.getId()); + } + } + + public void fetchAndProcessCases(String userToken, boolean dryRun, int numThreads, MigrationPageParams pageParams) + throws InterruptedException { + + SearchResult initialSearch = coreCaseDataService.searchCases(userToken, + fetchAllUnsetCaseAccessManagementFieldsCasesQuery()); + + if (initialSearch.getTotal() <= 0) { + return; + } + + totalTimer.start(); + + int totalCasesToProcess = resolveTotalCasesToProcess(initialSearch, pageParams.getMaxCasesToProcess()); + + Long searchFrom = handleFirstCase(userToken, dryRun, initialSearch); + + ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + + fetchAndSubmitTasks(userToken, dryRun, totalCasesToProcess, pageParams.getPageSize(), searchFrom, + executorService); + + executorService.shutdown(); + executorService.awaitTermination(Integer.MAX_VALUE, TimeUnit.DAYS); + } + + private int resolveTotalCasesToProcess(SearchResult initialSearch, int maxCasesToProcess) { + int totalCasesToProcess = 0; + + if (maxCasesToProcess > 0) { + infoLogger.info("Manual case override in use, limiting to {} cases", maxCasesToProcess); + totalCasesToProcess = maxCasesToProcess; } else { - log.info("Case {} already migrated", caseDetails.getId()); + infoLogger.info("No manual case override in use, fetching all: {} cases", initialSearch.getTotal()); + totalCasesToProcess = initialSearch.getTotal(); + } + + return totalCasesToProcess; + } + + private void fetchAndSubmitTasks(String userToken, boolean dryRun, int totalCasesToProcess, int pageSize, + Long searchFrom, ExecutorService executorService) { + int casesFetched = 1; + int numCasesToFetch = pageSize; + + while (casesFetched < totalCasesToProcess) { + numCasesToFetch = resolvePageSize(totalCasesToProcess, casesFetched, numCasesToFetch, pageSize); + + List caseDetails = + coreCaseDataService.fetchNCases(userToken, numCasesToFetch, searchFrom); + + if (caseDetails.isEmpty()) { + break; + } + + searchFrom = caseDetails.get(caseDetails.size() - 1).getId(); + + executorService.execute(() -> caseDetails + .forEach(caseDetail -> + updateCase(userToken, caseDetail.getId(), caseDetail.getData(), dryRun))); + + infoLogger.info("New task submitted"); + + casesFetched += caseDetails.size(); + + infoLogger.info("{} cases fetched out of {}", casesFetched, totalCasesToProcess); } } - public void processAllCases(String userToken, String userId) { + private int resolvePageSize(int totalCasesToProcess, int casesFetched, int numCasesToFetch, int pageSize) { + int remainingCases = totalCasesToProcess - casesFetched; + if (remainingCases < pageSize) { + numCasesToFetch = remainingCases; + } + return numCasesToFetch; + } + + private Long handleFirstCase(String userToken, boolean dryRun, SearchResult initialSearch) { + infoLogger.info("Processing first case..."); + CaseDetails firstCase = initialSearch.getCases().get(0); + updateCase(userToken, firstCase.getId(), firstCase.getData(), dryRun); + return firstCase.getId(); + } + + public void processAllCases(String userToken, String userId, boolean dryRun) { coreCaseDataService.fetchAll(userToken, userId).stream() .filter(dataMigrationService.accepts()) - .forEach(caseDetails -> updateCase(userToken, caseDetails.getId(), caseDetails.getData())); + .forEach(caseDetails -> updateCase(userToken, caseDetails.getId(), caseDetails.getData(), dryRun)); } - private void updateCase(String authorisation, Long id, Map data) { - log.info("Updating case {}", id); + protected List getListOfDates(LocalDate startDate, LocalDate endDate) { + return startDate + .datesUntil(endDate) + .collect(Collectors.toList()); + } + + private void updateCase(String authorisation, Long id, Map data, boolean dryRun) { + + totalCases++; + try { - log.debug("Case data: {}", data); - coreCaseDataService.update(authorisation, id.toString(), - EVENT_ID, EVENT_SUMMARY, EVENT_DESCRIPTION, dataMigrationService.migrate(data)); - log.info("Case {} successfully updated", id); + var migratedData = dataMigrationService.migrate(data, id); + if (!dryRun) { + coreCaseDataService.update( + authorisation, + id.toString(), + EVENT_ID, + EVENT_SUMMARY, + EVENT_DESCRIPTION, + migratedData); + infoLogger.info("Case {} successfully updated", id); + } migratedCases.add(id); + } catch (Exception e) { - log.error("Case {} update failed due to: {}", id, e.getMessage()); + errorLogger.error("Case {} update failed due to: {}", id, e.getMessage()); failedCases.add(id); } + + if (totalCases % 1000 == 0) { + infoLogger.info("----------{} cases migrated in {} minutes ({} seconds)----------", totalCases, + totalTimer.getTime(TimeUnit.MINUTES), totalTimer.getTime(TimeUnit.SECONDS)); + } } } diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationRunner.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationRunner.java index a59e0f75..dc8b97bb 100644 --- a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationRunner.java +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationRunner.java @@ -1,29 +1,37 @@ package uk.gov.hmcts.reform.migration; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; +import java.util.concurrent.TimeUnit; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.time.StopWatch; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.PropertySource; import uk.gov.hmcts.reform.idam.client.IdamClient; -@Slf4j @SpringBootApplication -@PropertySource("classpath:application.properties") +@RequiredArgsConstructor public class CaseMigrationRunner implements CommandLineRunner { + private final Logger log = LoggerFactory.getLogger("ccd-migration-info"); @Value("${migration.idam.username}") private String idamUsername; @Value("${migration.idam.password}") private String idamPassword; @Value("${migration.caseId}") private String ccdCaseId; - @Autowired - private IdamClient idamClient; - @Autowired - private CaseMigrationProcessor caseMigrationProcessor; + @Value("${migration.dryrun}") + private boolean dryrun; + @Value("${migration.pageSize}") + private int pageSize; + @Value("${migration.maxCasesToProcess}") + private int maxCasesToProcess; + @Value("${migration.numThreads}") + private int numThreads; + private final IdamClient idamClient; + private final CaseMigrationProcessor caseMigrationProcessor; public static void main(String[] args) { SpringApplication.run(CaseMigrationRunner.class, args); @@ -33,26 +41,39 @@ public static void main(String[] args) { public void run(String... args) { try { String userToken = idamClient.authenticateUser(idamUsername, idamPassword); - log.debug("User token: {}", userToken); - String userId = idamClient.getUserDetails(userToken).getId(); - log.debug("User ID: {}", userId); + + MigrationPageParams pageParams = new MigrationPageParams(pageSize, maxCasesToProcess); + + StopWatch stopWatch = StopWatch.createStarted(); if (ccdCaseId != null && !ccdCaseId.isBlank()) { log.info("Data migration of single case started"); - caseMigrationProcessor.processSingleCase(userToken, ccdCaseId); + caseMigrationProcessor.processSingleCase(userToken, ccdCaseId, dryrun); } else { - log.info("Data migration of all cases started"); - caseMigrationProcessor.processAllCases(userToken, userId); + log.info("Data migration of cases started"); + caseMigrationProcessor.fetchAndProcessCases(userToken, dryrun, numThreads, pageParams); } + stopWatch.stop(); + log.info("-----------------------------------------"); - log.info("Data migration completed"); - log.info("-----------------------------------------"); - log.info("Total number of processed cases: {}", caseMigrationProcessor.getMigratedCases().size() + caseMigrationProcessor.getFailedCases().size()); + log.info("Total number of cases: {}", caseMigrationProcessor.getTotalCases()); + log.info("Total number of processed cases: {}", caseMigrationProcessor.getMigratedCases().size() + + caseMigrationProcessor.getFailedCases().size()); log.info("Total number of migrations performed: {}", caseMigrationProcessor.getMigratedCases().size()); log.info("-----------------------------------------"); - log.info("Migrated cases: {} ", !caseMigrationProcessor.getMigratedCases().isEmpty() ? caseMigrationProcessor.getMigratedCases() : "NONE"); - log.info("Failed cases: {}", !caseMigrationProcessor.getFailedCases().isEmpty() ? caseMigrationProcessor.getFailedCases() : "NONE"); + log.info("Number of migrated cases: {}", caseMigrationProcessor.getMigratedCases().size()); + log.info("Migrated cases: {} ", !caseMigrationProcessor.getMigratedCases().isEmpty() + ? caseMigrationProcessor.getMigratedCases() + : "NONE"); + log.info("Number of failed cases: {}", caseMigrationProcessor.getFailedCases().size()); + log.info("Failed cases: {} ", !caseMigrationProcessor.getFailedCases().isEmpty() + ? caseMigrationProcessor.getFailedCases() + : "NONE"); + log.info("-----------------------------------------"); + log.info("Data migration completed in: {} minutes ({} seconds).", + stopWatch.getTime(TimeUnit.MINUTES), stopWatch.getTime(TimeUnit.SECONDS)); + log.info("-----------------------------------------"); } catch (Throwable e) { log.error("Migration failed with the following reason: {}", e.getMessage(), e); } diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/MigrationPageParams.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/MigrationPageParams.java new file mode 100644 index 00000000..55f8e119 --- /dev/null +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/MigrationPageParams.java @@ -0,0 +1,14 @@ +package uk.gov.hmcts.reform.migration; + + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public class MigrationPageParams { + + private final int pageSize; + + private final int maxCasesToProcess; +} diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataService.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataService.java index d210773f..d4892a04 100644 --- a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataService.java +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataService.java @@ -1,6 +1,22 @@ package uk.gov.hmcts.reform.migration.ccd; -import org.springframework.beans.factory.annotation.Autowired; +import static java.util.Collections.emptyList; +import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.oldestCaseQuery; +import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.pageForUnsetCaseAccessManagementFieldsFieldsQuery; + +import feign.FeignException; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.time.StopWatch; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import uk.gov.hmcts.reform.authorisation.generators.AuthTokenGenerator; @@ -9,53 +25,66 @@ import uk.gov.hmcts.reform.ccd.client.model.CaseDetails; import uk.gov.hmcts.reform.ccd.client.model.Event; import uk.gov.hmcts.reform.ccd.client.model.PaginatedSearchMetadata; +import uk.gov.hmcts.reform.ccd.client.model.SearchResult; import uk.gov.hmcts.reform.ccd.client.model.StartEventResponse; -import uk.gov.hmcts.reform.migration.auth.AuthUtil; import uk.gov.hmcts.reform.idam.client.IdamClient; import uk.gov.hmcts.reform.idam.client.models.UserDetails; +import uk.gov.hmcts.reform.migration.auth.AuthUtil; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - +@Slf4j @Service +@RequiredArgsConstructor public class CoreCaseDataService { + private static final String SSCS_CASE_TYPE = "Benefit"; + + @Value("${migration.jurisdiction}") private String jurisdiction; + @Value("${migration.caseType}") private String caseType; - @Autowired - private IdamClient idamClient; - @Autowired - private AuthTokenGenerator authTokenGenerator; - @Autowired - private CoreCaseDataApi coreCaseDataApi; + private final IdamClient idamClient; + private final AuthTokenGenerator authTokenGenerator; + private final CoreCaseDataApi coreCaseDataApi; public CaseDetails fetchOne(String authorisation, String caseId) { return coreCaseDataApi.getCase(authorisation, authTokenGenerator.generate(), caseId); } - public List fetchAll(String authorisation, String userId) { - int numberOfPages = getNumberOfPages(authorisation, userId, new HashMap<>()); - return IntStream.rangeClosed(1, numberOfPages).boxed() - .flatMap(pageNumber -> fetchPage(authorisation, userId, pageNumber).stream()) - .collect(Collectors.toList()); + public List fetchNCases(String authorisation, int casesToFetch, long searchFrom) { + + StopWatch stopWatch = StopWatch.createStarted(); + + List page = fetchPage(authorisation, + pageForUnsetCaseAccessManagementFieldsFieldsQuery(searchFrom, casesToFetch)); + + stopWatch.stop(); + + log.info("Case search with page size: {} completed in: {} minutes ({} seconds).", casesToFetch, + stopWatch.getTime(TimeUnit.MINUTES), stopWatch.getTime(TimeUnit.SECONDS)); + + return page; + } - private int getNumberOfPages(String authorisation, String userId, Map searchCriteria) { - PaginatedSearchMetadata metadata = coreCaseDataApi.getPaginationInfoForSearchForCaseworkers(authorisation, - authTokenGenerator.generate(), userId, jurisdiction, caseType, searchCriteria); - return metadata.getTotalPagesCount(); + private List fetchPage(String authorisation, SearchSourceBuilder searchSourceBuilder) { + List caseDetails = emptyList(); + + try { + caseDetails = searchCases(authorisation, searchSourceBuilder).getCases(); + } catch (FeignException fe) { + log.error("Feign Exception message: {} with search string: {}", + fe.contentUTF8(), searchSourceBuilder); + } + + return caseDetails; } - private List fetchPage(String authorisation, String userId, int pageNumber) { - Map searchCriteria = new HashMap<>(); - searchCriteria.put("page", String.valueOf(pageNumber)); - return coreCaseDataApi.searchForCaseworker(authorisation, authTokenGenerator.generate(), userId, jurisdiction, - caseType, searchCriteria); + public CaseDetails fetchOldestCase(String authorisation) { + return searchCases(authorisation, oldestCaseQuery()) + .getCases() + .get(0); } public CaseDetails update(String authorisation, String caseId, String eventId, String eventSummary, String eventDescription, Object data) { @@ -91,4 +120,34 @@ public CaseDetails update(String authorisation, String caseId, String eventId, S true, caseDataContent); } + + public SearchResult searchCases(String authorisation, SearchSourceBuilder searchBuilder) { + return coreCaseDataApi.searchCases( + authorisation, + authTokenGenerator.generate(), + SSCS_CASE_TYPE, + searchBuilder.toString()); + } + + + public List fetchAll(String authorisation, String userId) { + int numberOfPages = getNumberOfPages(authorisation, userId, new HashMap<>()); + return IntStream.rangeClosed(1, numberOfPages).boxed() + .flatMap(pageNumber -> fetchPage(authorisation, userId, pageNumber).stream()) + .collect(Collectors.toList()); + } + + private List fetchPage(String authorisation, String userId, int pageNumber) { + Map searchCriteria = new HashMap<>(); + searchCriteria.put("page", String.valueOf(pageNumber)); + return coreCaseDataApi.searchForCaseworker(authorisation, authTokenGenerator.generate(), userId, jurisdiction, + caseType, searchCriteria); + } + + private int getNumberOfPages(String authorisation, String userId, Map searchCriteria) { + PaginatedSearchMetadata metadata = coreCaseDataApi.getPaginationInfoForSearchForCaseworkers(authorisation, + authTokenGenerator.generate(), userId, jurisdiction, caseType, searchCriteria); + return metadata.getTotalPagesCount(); + } + } diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/queries/CcdElasticSearchQueries.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/queries/CcdElasticSearchQueries.java new file mode 100644 index 00000000..d4926d89 --- /dev/null +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/queries/CcdElasticSearchQueries.java @@ -0,0 +1,76 @@ +package uk.gov.hmcts.reform.migration.queries; + +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; + +import static org.elasticsearch.index.query.QueryBuilders.existsQuery; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; + +public class CcdElasticSearchQueries { + + private CcdElasticSearchQueries() { + } + + private static final String CREATED_DATE = "created_date"; + + public static final String REFERENCE_KEYWORD = "reference.keyword"; + + public static SearchSourceBuilder oldestCaseQuery() { + return SearchSourceBuilder.searchSource() + .size(1) + .sort(CREATED_DATE, SortOrder.ASC) + .query(QueryBuilders.boolQuery()); + } + + public static SearchSourceBuilder pageQuery(String day, int pageNumber, int pageSize) { + return SearchSourceBuilder.searchSource() + .size(pageSize) + .from(pageNumber * pageSize) + .query(QueryBuilders.boolQuery() + .must(matchQuery(CREATED_DATE, day))); + } + + public static SearchSourceBuilder fetchAllUnsetCaseAccessManagementFieldsCasesQuery() { + return SearchSourceBuilder.searchSource() + .size(1) + .query(unsetCaseAccessManagementFieldsQuery()) + .sort(REFERENCE_KEYWORD, SortOrder.ASC); + } + + public static BoolQueryBuilder unsetCaseAccessManagementFieldsQuery() { + return QueryBuilders.boolQuery() + .must(QueryBuilders.boolQuery() + .should(existsQuery("data.appeal.appellant.address.postcode")) + .should(existsQuery("data.appeal.appellant.appointee.address.postcode")) + .minimumShouldMatch(1)) + .mustNot( + QueryBuilders.boolQuery() + .should(existsQuery("data.CaseAccessCategory")) + .should(existsQuery("data.caseManagementCategory")) + .should(existsQuery("data.caseManagementLocation")) + .should(existsQuery("data.caseNameHmctsRestricted")) + .should(existsQuery("data.caseNamePublic")) + .should(existsQuery("data.caseNameHmctsInternal")) + .should(existsQuery("data.ogdType")) + .minimumShouldMatch(7)); + } + + public static SearchSourceBuilder pageForUnsetCaseAccessManagementFieldsFieldsQuery(int pageSize) { + return SearchSourceBuilder.searchSource() + .size(pageSize) + .query(unsetCaseAccessManagementFieldsQuery()) + .sort(REFERENCE_KEYWORD, SortOrder.ASC); + } + + public static SearchSourceBuilder pageForUnsetCaseAccessManagementFieldsFieldsQuery(Long lastCaseId, + int pageSize) { + + return SearchSourceBuilder.searchSource() + .size(pageSize) + .query(unsetCaseAccessManagementFieldsQuery()) + .searchAfter(new Object[] { lastCaseId}) + .sort(REFERENCE_KEYWORD, SortOrder.ASC); + } +} diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/service/DataMigrationService.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/service/DataMigrationService.java index b6fea5c0..a9974994 100644 --- a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/service/DataMigrationService.java +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/service/DataMigrationService.java @@ -8,5 +8,5 @@ public interface DataMigrationService { Predicate accepts(); - T migrate(Map data); + T migrate(Map data, Long id); } diff --git a/modules/processor/src/test/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessorTest.java b/modules/processor/src/test/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessorTest.java index 55e9cf4f..5be82e16 100644 --- a/modules/processor/src/test/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessorTest.java +++ b/modules/processor/src/test/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessorTest.java @@ -1,31 +1,37 @@ package uk.gov.hmcts.reform.migration; +import static org.codehaus.groovy.runtime.InvokerHelper.asList; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import uk.gov.hmcts.reform.ccd.client.model.CaseDetails; +import uk.gov.hmcts.reform.ccd.client.model.SearchResult; import uk.gov.hmcts.reform.migration.ccd.CoreCaseDataService; import uk.gov.hmcts.reform.migration.service.DataMigrationService; -import java.util.HashMap; -import java.util.Map; - -import static java.util.Arrays.asList; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.hasSize; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - - @RunWith(MockitoJUnitRunner.class) public class CaseMigrationProcessorTest { - private static final String USER_TOKEN = "Bearer eeeejjjttt"; private static final String USER_ID = "30"; + private static final String USER_TOKEN = "Bearer eeeejjjttt"; private static final String CASE_ID = "11111"; private static final String EVENT_ID = "migrateCase"; private static final String EVENT_SUMMARY = "Migrate Case"; @@ -35,18 +41,20 @@ public class CaseMigrationProcessorTest { private final CaseDetails caseDetails2 = createCaseDetails(1112L, "case-2"); private final CaseDetails caseDetails3 = createCaseDetails(1113L, "case-3"); - @InjectMocks - private CaseMigrationProcessor caseMigrationProcessor; @Mock private CoreCaseDataService coreCaseDataService; + @Mock private DataMigrationService dataMigrationService; + @InjectMocks + private CaseMigrationProcessor caseMigrationProcessor; + @Test - public void shouldNotProcessASingleCaseWithOutRedundantFields() { + public void shouldNotProcessASingleCaseWithoutRedundantFields() { when(coreCaseDataService.fetchOne(USER_TOKEN, CASE_ID)).thenReturn(caseDetails1); when(dataMigrationService.accepts()).thenReturn(candidate -> false); - caseMigrationProcessor.processSingleCase(USER_TOKEN, CASE_ID); + caseMigrationProcessor.processSingleCase(USER_TOKEN, CASE_ID, false); verify(coreCaseDataService, times(1)).fetchOne(USER_TOKEN, CASE_ID); assertThat(caseMigrationProcessor.getFailedCases(), hasSize(0)); assertThat(caseMigrationProcessor.getMigratedCases(), hasSize(0)); @@ -56,7 +64,7 @@ public void shouldNotProcessASingleCaseWithOutRedundantFields() { public void shouldProcessASingleCaseAndMigrationIsSuccessful() { when(coreCaseDataService.fetchOne(USER_TOKEN, CASE_ID)).thenReturn(caseDetails1); when(dataMigrationService.accepts()).thenReturn(candidate -> true); - caseMigrationProcessor.processSingleCase(USER_TOKEN, CASE_ID); + caseMigrationProcessor.processSingleCase(USER_TOKEN, CASE_ID, false); verify(coreCaseDataService, times(1)).fetchOne(USER_TOKEN, CASE_ID); assertThat(caseMigrationProcessor.getFailedCases(), hasSize(0)); assertThat(caseMigrationProcessor.getMigratedCases(), contains(1111L)); @@ -66,9 +74,9 @@ public void shouldProcessASingleCaseAndMigrationIsSuccessful() { public void shouldProcessASingleCaseAndMigrationIsFailed() { when(coreCaseDataService.fetchOne(USER_TOKEN, CASE_ID)).thenReturn(caseDetails1); when(dataMigrationService.accepts()).thenReturn(candidate -> true); - when(dataMigrationService.migrate(caseDetails1.getData())).thenReturn(caseDetails1.getData()); + when(dataMigrationService.migrate(caseDetails1.getData(), caseDetails1.getId())).thenReturn(caseDetails1.getData()); when(coreCaseDataService.update(USER_TOKEN, caseDetails1.getId().toString(), EVENT_ID, EVENT_SUMMARY, EVENT_DESCRIPTION, caseDetails1.getData())).thenThrow(new RuntimeException("Internal server error")); - caseMigrationProcessor.processSingleCase(USER_TOKEN, CASE_ID); + caseMigrationProcessor.processSingleCase(USER_TOKEN, CASE_ID, false); verify(coreCaseDataService, times(1)).fetchOne(USER_TOKEN, CASE_ID); verify(coreCaseDataService, times(1)).update(USER_TOKEN, "1111", EVENT_ID, EVENT_SUMMARY, EVENT_DESCRIPTION, caseDetails1.getData()); assertThat(caseMigrationProcessor.getFailedCases(), contains(1111L)); @@ -81,11 +89,11 @@ public void shouldProcessAllTheCandidateCases_whenOneCaseFailed() { mockDataUpdate(caseDetails1); mockDataUpdate(caseDetails2); when(dataMigrationService.accepts()).thenReturn(candidate -> true); - when(dataMigrationService.migrate(caseDetails1.getData())).thenReturn(caseDetails1.getData()); - when(dataMigrationService.migrate(caseDetails2.getData())).thenReturn(caseDetails2.getData()); - when(dataMigrationService.migrate(caseDetails3.getData())).thenReturn(caseDetails3.getData()); + when(dataMigrationService.migrate(caseDetails1.getData(), caseDetails1.getId())).thenReturn(caseDetails1.getData()); + when(dataMigrationService.migrate(caseDetails2.getData(), caseDetails2.getId())).thenReturn(caseDetails2.getData()); + when(dataMigrationService.migrate(caseDetails3.getData(), caseDetails3.getId())).thenReturn(caseDetails3.getData()); when(coreCaseDataService.update(USER_TOKEN, caseDetails3.getId().toString(), EVENT_ID, EVENT_SUMMARY, EVENT_DESCRIPTION, caseDetails3.getData())).thenThrow(new RuntimeException("Internal server error")); - caseMigrationProcessor.processAllCases(USER_TOKEN, USER_ID); + caseMigrationProcessor.processAllCases(USER_TOKEN, USER_ID, false); assertThat(caseMigrationProcessor.getFailedCases(), contains(1113L)); assertThat(caseMigrationProcessor.getMigratedCases(), contains(1111L, 1112L)); } @@ -95,12 +103,12 @@ public void shouldProcessAllTheCandidateCases_whenTwoCasesFailed() { mockDataFetch(caseDetails1, caseDetails2, caseDetails3); mockDataUpdate(caseDetails1); when(dataMigrationService.accepts()).thenReturn(candidate -> true); - when(dataMigrationService.migrate(caseDetails1.getData())).thenReturn(caseDetails1.getData()); - when(dataMigrationService.migrate(caseDetails2.getData())).thenReturn(caseDetails2.getData()); - when(dataMigrationService.migrate(caseDetails3.getData())).thenReturn(caseDetails3.getData()); + when(dataMigrationService.migrate(caseDetails1.getData(), caseDetails1.getId())).thenReturn(caseDetails1.getData()); + when(dataMigrationService.migrate(caseDetails2.getData(), caseDetails2.getId())).thenReturn(caseDetails2.getData()); + when(dataMigrationService.migrate(caseDetails3.getData(), caseDetails3.getId())).thenReturn(caseDetails3.getData()); when(coreCaseDataService.update(USER_TOKEN, caseDetails2.getId().toString(), EVENT_ID, EVENT_SUMMARY, EVENT_DESCRIPTION, caseDetails2.getData())).thenThrow(new RuntimeException("Internal server error")); when(coreCaseDataService.update(USER_TOKEN, caseDetails3.getId().toString(), EVENT_ID, EVENT_SUMMARY, EVENT_DESCRIPTION, caseDetails3.getData())).thenThrow(new RuntimeException("Internal server error")); - caseMigrationProcessor.processAllCases(USER_TOKEN, USER_ID); + caseMigrationProcessor.processAllCases(USER_TOKEN, USER_ID, false); assertThat(caseMigrationProcessor.getFailedCases(), contains(1112L, 1113L)); assertThat(caseMigrationProcessor.getMigratedCases(), contains(1111L)); } @@ -110,11 +118,108 @@ public void shouldProcessNoCaseWhenNoCasesAvailable() { mockDataFetch(); when(dataMigrationService.accepts()).thenReturn(candidate -> true); - caseMigrationProcessor.processAllCases(USER_TOKEN, USER_ID); + caseMigrationProcessor.processAllCases(USER_TOKEN, USER_ID, false); assertThat(caseMigrationProcessor.getFailedCases(), hasSize(0)); assertThat(caseMigrationProcessor.getFailedCases(), hasSize(0)); } + @Test + public void shouldDoNothingIfNoCasesToProcess() throws InterruptedException { + SearchResult result = SearchResult.builder() + .cases(new ArrayList<>()) + .total(0).build(); + + MigrationPageParams pageParams = new MigrationPageParams(10, 10); + + when(coreCaseDataService.searchCases(anyString(), + any(SearchSourceBuilder.class))).thenReturn(result); + + caseMigrationProcessor.fetchAndProcessCases(USER_TOKEN, false, 1, pageParams); + + assertThat(caseMigrationProcessor.getFailedCases().size(), is(0)); + assertThat(caseMigrationProcessor.getMigratedCases().size(), is(0)); + } + + @Test + public void shouldUseOverrideIfProvided() throws InterruptedException { + SearchResult result = SearchResult.builder() + .cases(List.of(caseDetails1, caseDetails2, caseDetails3)) + .total(3).build(); + + MigrationPageParams pageParams = new MigrationPageParams(10, 2); + + when(coreCaseDataService.searchCases(anyString(), + any(SearchSourceBuilder.class))).thenReturn(result); + + when(coreCaseDataService.fetchNCases(USER_TOKEN, 1, + caseDetails1.getId())).thenReturn(List.of(caseDetails2)); + + caseMigrationProcessor.fetchAndProcessCases(USER_TOKEN, false, 1, pageParams); + + assertThat(caseMigrationProcessor.getFailedCases().size(), is(0)); + assertThat(caseMigrationProcessor.getMigratedCases().size(), is(2)); + } + + @Test + public void shouldProcessAllCasesIfNoOverride() throws InterruptedException { + SearchResult result = SearchResult.builder() + .cases(List.of(caseDetails1, caseDetails2, caseDetails3)) + .total(3).build(); + + MigrationPageParams pageParams = new MigrationPageParams(10, 0); + + when(coreCaseDataService.searchCases(anyString(), + any(SearchSourceBuilder.class))).thenReturn(result); + + when(coreCaseDataService.fetchNCases(USER_TOKEN, 2, + caseDetails1.getId())).thenReturn(List.of(caseDetails2, caseDetails3)); + + caseMigrationProcessor.fetchAndProcessCases(USER_TOKEN, false, 1, pageParams); + + assertThat(caseMigrationProcessor.getFailedCases().size(), is(0)); + assertThat(caseMigrationProcessor.getMigratedCases().size(), is(3)); + } + + @Test + public void shouldBreakWhenNoMoreCasesReturned() throws InterruptedException { + SearchResult result = SearchResult.builder() + .cases(List.of(caseDetails1, caseDetails2, caseDetails3)) + .total(3).build(); + + MigrationPageParams pageParams = new MigrationPageParams(10, 2); + + when(coreCaseDataService.searchCases(any(String.class), + any(SearchSourceBuilder.class))).thenReturn(result); + + caseMigrationProcessor.fetchAndProcessCases(USER_TOKEN, false, 1, pageParams); + + assertThat(caseMigrationProcessor.getFailedCases().size(), is(0)); + assertThat(caseMigrationProcessor.getMigratedCases().size(), is(1)); + } + + @Test + public void shouldSearchFromLastCaseInPreviousResult() throws InterruptedException { + SearchResult result = SearchResult.builder() + .cases(List.of(caseDetails1, caseDetails2, caseDetails3)) + .total(3).build(); + + MigrationPageParams pageParams = new MigrationPageParams(1, 0); + + when(coreCaseDataService.searchCases(any(String.class), + any(SearchSourceBuilder.class))).thenReturn(result); + + when(coreCaseDataService.fetchNCases(USER_TOKEN, 1, + caseDetails1.getId())).thenReturn(List.of(caseDetails2)); + + when(coreCaseDataService.fetchNCases(USER_TOKEN, 1, + caseDetails2.getId())).thenReturn(List.of(caseDetails3)); + + caseMigrationProcessor.fetchAndProcessCases(USER_TOKEN, false, 1, pageParams); + + assertThat(caseMigrationProcessor.getFailedCases().size(), is(0)); + assertThat(caseMigrationProcessor.getMigratedCases().size(), is(3)); + } + private void mockDataFetch(CaseDetails... caseDetails) { when(coreCaseDataService.fetchAll(USER_TOKEN, USER_ID)).thenReturn(asList(caseDetails)); } diff --git a/modules/processor/src/test/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataServiceTest.java b/modules/processor/src/test/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataServiceTest.java index edba5ee2..2d2796c2 100644 --- a/modules/processor/src/test/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataServiceTest.java +++ b/modules/processor/src/test/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataServiceTest.java @@ -36,18 +36,17 @@ public class CoreCaseDataServiceTest { private static final String EVENT_SUMMARY = "Migrate Case"; private static final String EVENT_DESC = "Migrate Case"; - @InjectMocks - private CoreCaseDataService underTest; - - @Mock - CoreCaseDataApi coreCaseDataApi; - @Mock private IdamClient idamClient; @Mock private AuthTokenGenerator authTokenGenerator; + @Mock + private CoreCaseDataApi coreCaseDataApi; + + @InjectMocks + private CoreCaseDataService underTest; @Before public void setUp() {