From 5495b671a629918e7df4ff6158979d3ec7e28ccf Mon Sep 17 00:00:00 2001 From: ThomasKKC Date: Fri, 30 Sep 2022 17:09:40 +0100 Subject: [PATCH 1/2] Init RIA-6220 commit --- .../migration/CaseMigrationProcessor.java | 30 ++++++++++++------ .../reform/migration/CaseMigrationRunner.java | 26 +++++++++++++++- .../migration/ccd/CoreCaseDataService.java | 12 +++---- .../reform/migration/ccd/MigrationEvent.java | 24 ++++++++++++++ .../queries/CcdElasticSearchQueries.java | 31 +++++++++++++++++++ 5 files changed, 107 insertions(+), 16 deletions(-) create mode 100644 modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/MigrationEvent.java diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessor.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessor.java index c5389b2e..80ed32eb 100644 --- a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessor.java +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationProcessor.java @@ -13,6 +13,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.time.StopWatch; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,10 +21,10 @@ import uk.gov.hmcts.reform.ccd.client.model.CaseDetails; import uk.gov.hmcts.reform.ccd.client.model.SearchResult; import uk.gov.hmcts.reform.migration.ccd.CoreCaseDataService; +import uk.gov.hmcts.reform.migration.ccd.MigrationEvent; import uk.gov.hmcts.reform.migration.service.DataMigrationService; -import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.fetchAllCaseNameInternalCasesQuery; -import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.fetchAllUnmigratedGlobalSearchCasesQuery; +import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.*; @Component @RequiredArgsConstructor @@ -33,7 +34,7 @@ public class CaseMigrationProcessor { private final Logger infoLogger = LoggerFactory.getLogger("ccd-migration-info"); - private static final String EVENT_ID = "migrateWorkAllocationR3"; + private MigrationEvent eventId = MigrationEvent.UNKNOWN_MIGRATION; private static final String EVENT_SUMMARY = "Migrate Case"; private static final String EVENT_DESCRIPTION = "Migrate Case"; @@ -66,13 +67,24 @@ public void processSingleCase(String userToken, String caseId, boolean dryRun) { } } - public void fetchAndProcessCases(String userToken, boolean dryRun, int numThreads, MigrationPageParams pageParams, boolean migrateCaseNameInternalFlag) + public void fetchAndProcessCases(String userToken, boolean dryRun, int numThreads, MigrationPageParams pageParams, + MigrationEvent migrationEvent, boolean migrateCaseNameInternalFlag, + boolean migrateCaseFlagsInternalFlag) throws InterruptedException { + eventId = migrationEvent; + SearchSourceBuilder currentQuery = fetchAllUnmigratedGlobalSearchCasesQuery(); + BoolQueryBuilder queryBuilder = (BoolQueryBuilder) fetchAllUnmigratedGlobalSearchCasesQuery().query(); - if (migrateCaseNameInternalFlag){ + if (migrationEvent.equals(MigrationEvent.MIGRATE_WORK_ALLOCATION_R3) && migrateCaseNameInternalFlag){ currentQuery = fetchAllCaseNameInternalCasesQuery(); + queryBuilder = (BoolQueryBuilder) fetchAllCaseNameInternalCasesQuery().query(); + } + + if (migrationEvent.equals(MigrationEvent.MIGRATE_CASE_FLAGS) && migrateCaseFlagsInternalFlag){ + currentQuery = fetchAllUnmigratedCaseFlagsInternalCasesQuery(); + queryBuilder = (BoolQueryBuilder) fetchAllUnmigratedCaseFlagsInternalCasesQuery().query(); } SearchResult initialSearch = coreCaseDataService.searchCases(userToken, @@ -91,7 +103,7 @@ public void fetchAndProcessCases(String userToken, boolean dryRun, int numThread ExecutorService executorService = Executors.newFixedThreadPool(numThreads); fetchAndSubmitTasks(userToken, dryRun, totalCasesToProcess, pageParams.getPageSize(), searchFrom, - executorService, migrateCaseNameInternalFlag); + executorService, queryBuilder); executorService.shutdown(); executorService.awaitTermination(Integer.MAX_VALUE, TimeUnit.DAYS); @@ -112,7 +124,7 @@ private int resolveTotalCasesToProcess(SearchResult initialSearch, int maxCasesT } private void fetchAndSubmitTasks(String userToken, boolean dryRun, int totalCasesToProcess, int pageSize, - Long searchFrom, ExecutorService executorService, boolean migrateCaseNameInternalFlag) { + Long searchFrom, ExecutorService executorService, BoolQueryBuilder queryBuilder) { int casesFetched = 1; int numCasesToFetch = pageSize; @@ -120,7 +132,7 @@ private void fetchAndSubmitTasks(String userToken, boolean dryRun, int totalCase numCasesToFetch = resolvePageSize(totalCasesToProcess, casesFetched, numCasesToFetch, pageSize); List caseDetails = - coreCaseDataService.fetchNCases(userToken, numCasesToFetch, searchFrom); + coreCaseDataService.fetchNCases(userToken, numCasesToFetch, searchFrom, queryBuilder); if (caseDetails.isEmpty()) { break; @@ -177,7 +189,7 @@ private void updateCase(String authorisation, Long id, Map data, coreCaseDataService.update( authorisation, id.toString(), - EVENT_ID, + eventId.toString(), EVENT_SUMMARY, EVENT_DESCRIPTION, migratedData); diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationRunner.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationRunner.java index d97440ab..ecf8e30b 100644 --- a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationRunner.java +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/CaseMigrationRunner.java @@ -10,6 +10,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import uk.gov.hmcts.reform.idam.client.IdamClient; +import uk.gov.hmcts.reform.migration.ccd.MigrationEvent; @SpringBootApplication @RequiredArgsConstructor @@ -32,6 +33,8 @@ public class CaseMigrationRunner implements CommandLineRunner { private int numThreads; @Value("${migration.migrateCaseNameInternal}") private boolean migrateCaseNameInternalFlag; + @Value("${migration.migrateCaseFlagsInternal}") + private boolean migrateCaseFlagsInternalFlag; private final IdamClient idamClient; private final CaseMigrationProcessor caseMigrationProcessor; @@ -48,12 +51,20 @@ public void run(String... args) { StopWatch stopWatch = StopWatch.createStarted(); + log.info("-----------------------------------------"); + log.info("Is this DryRun: {}", dryrun); + log.info("-----------------------------------------"); + log.info("-----------------------------------------"); + log.info("Migration Event: {}", getMigrationEvent()); + log.info("-----------------------------------------"); + if (ccdCaseId != null && !ccdCaseId.isBlank()) { log.info("Data migration of single case started"); caseMigrationProcessor.processSingleCase(userToken, ccdCaseId, dryrun); } else { log.info("Data migration of cases started"); - caseMigrationProcessor.fetchAndProcessCases(userToken, dryrun, numThreads, pageParams, migrateCaseNameInternalFlag); + caseMigrationProcessor.fetchAndProcessCases(userToken, dryrun, numThreads, pageParams, + getMigrationEvent(), migrateCaseNameInternalFlag, migrateCaseFlagsInternalFlag); } stopWatch.stop(); @@ -76,4 +87,17 @@ public void run(String... args) { log.error("Migration failed with the following reason: {}", e.getMessage(), e); } } + + private MigrationEvent getMigrationEvent(){ + + if (migrateCaseNameInternalFlag){ + return MigrationEvent.MIGRATE_WORK_ALLOCATION_R3; + } + else if (migrateCaseFlagsInternalFlag) { + return MigrationEvent.MIGRATE_CASE_FLAGS; + } + else { + return MigrationEvent.MIGRATE_WORK_ALLOCATION_R3; + } + } } diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataService.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataService.java index 45450243..7db2e689 100644 --- a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataService.java +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/CoreCaseDataService.java @@ -1,8 +1,7 @@ package uk.gov.hmcts.reform.migration.ccd; import static java.util.Collections.emptyList; -import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.oldestCaseQuery; -import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.pageForUnsetCaseAccessManagementFieldsFieldsQuery; +import static uk.gov.hmcts.reform.migration.queries.CcdElasticSearchQueries.*; import feign.FeignException; @@ -16,6 +15,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.time.StopWatch; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -36,7 +36,7 @@ @RequiredArgsConstructor public class CoreCaseDataService { - private static final String SSCS_CASE_TYPE = "Asylum"; + private static final String IAC_CASE_TYPE = "Asylum"; @Value("${migration.jurisdiction}") private String jurisdiction; @@ -52,12 +52,12 @@ public CaseDetails fetchOne(String authorisation, String caseId) { return coreCaseDataApi.getCase(authorisation, authTokenGenerator.generate(), caseId); } - public List fetchNCases(String authorisation, int casesToFetch, long searchFrom) { + public List fetchNCases(String authorisation, int casesToFetch, long searchFrom, BoolQueryBuilder queryBuilder) { StopWatch stopWatch = StopWatch.createStarted(); List page = fetchPage(authorisation, - pageForUnsetCaseAccessManagementFieldsFieldsQuery(searchFrom, casesToFetch)); + pageForUnsetCaseFieldsFieldsQuery(searchFrom, casesToFetch, queryBuilder)); stopWatch.stop(); @@ -125,7 +125,7 @@ public SearchResult searchCases(String authorisation, SearchSourceBuilder search return coreCaseDataApi.searchCases( authorisation, authTokenGenerator.generate(), - SSCS_CASE_TYPE, + IAC_CASE_TYPE, searchBuilder.toString()); } diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/MigrationEvent.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/MigrationEvent.java new file mode 100644 index 00000000..d6e16407 --- /dev/null +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/ccd/MigrationEvent.java @@ -0,0 +1,24 @@ +package uk.gov.hmcts.reform.migration.ccd; + +import com.fasterxml.jackson.annotation.JsonEnumDefaultValue; +import com.fasterxml.jackson.annotation.JsonValue; + +public enum MigrationEvent { + MIGRATE_WORK_ALLOCATION_R3("migrateWorkAllocationR3"), + MIGRATE_CASE_FLAGS("migrateCaseFlags"), + + @JsonEnumDefaultValue + UNKNOWN_MIGRATION("unknownMigration"); + + @JsonValue + private final String id; + + MigrationEvent(String id) { + this.id = id; + } + + @Override + public String toString() { + return id; + } +} diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/queries/CcdElasticSearchQueries.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/queries/CcdElasticSearchQueries.java index e065dc7f..05dc0150 100644 --- a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/queries/CcdElasticSearchQueries.java +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/queries/CcdElasticSearchQueries.java @@ -46,6 +46,13 @@ public static SearchSourceBuilder fetchAllCaseNameInternalCasesQuery() { .sort(REFERENCE_KEYWORD, SortOrder.ASC); } + public static SearchSourceBuilder fetchAllUnmigratedCaseFlagsInternalCasesQuery() { + return SearchSourceBuilder.searchSource() + .size(1) + .query(caseFlagsInternalFieldsQuery()) + .sort(REFERENCE_KEYWORD, SortOrder.ASC); + } + public static BoolQueryBuilder missingSearchCriteriaFieldsQuery() { return QueryBuilders.boolQuery() .mustNot( @@ -60,6 +67,20 @@ public static BoolQueryBuilder missingCaseNameInternalFieldsQuery() { .should(existsQuery("data.caseNameHmctsInternal"))); } + public static BoolQueryBuilder caseFlagsInternalFieldsQuery() { + + return QueryBuilders.boolQuery() + .must(existsQuery("data.caseFlags")) + .should(QueryBuilders.boolQuery() + .should(QueryBuilders.matchQuery("data.caseFlags.value.caseFlagType", "anonymity")) + .should(QueryBuilders.matchQuery("data.caseFlags.value.caseFlagType", "detainedImmigrationAppeal")) + .minimumShouldMatch(1)) + .mustNot( + QueryBuilders.boolQuery() + .should(existsQuery("data.caseLevelFlags")) + .should(existsQuery("data.appellantLevelFlags"))); + } + public static SearchSourceBuilder pageForUnsetCaseAccessManagementFieldsFieldsQuery(int pageSize) { return SearchSourceBuilder.searchSource() .size(pageSize) @@ -76,4 +97,14 @@ public static SearchSourceBuilder pageForUnsetCaseAccessManagementFieldsFieldsQu .searchAfter(new Object[] { lastCaseId}) .sort(REFERENCE_KEYWORD, SortOrder.ASC); } + + public static SearchSourceBuilder pageForUnsetCaseFieldsFieldsQuery(Long lastCaseId, int pageSize, + BoolQueryBuilder queryBuilder) { + + return SearchSourceBuilder.searchSource() + .size(pageSize) + .query(queryBuilder) + .searchAfter(new Object[] { lastCaseId}) + .sort(REFERENCE_KEYWORD, SortOrder.ASC); + } } From d6a64d2b307b514c49fb34451fb6b33cf4214c24 Mon Sep 17 00:00:00 2001 From: ThomasKKC Date: Mon, 3 Oct 2022 11:46:45 +0100 Subject: [PATCH 2/2] RIA-6220 added correct tactical case flags for migration --- .../reform/migration/queries/CcdElasticSearchQueries.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/queries/CcdElasticSearchQueries.java b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/queries/CcdElasticSearchQueries.java index 05dc0150..cadf1dc8 100644 --- a/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/queries/CcdElasticSearchQueries.java +++ b/modules/processor/src/main/java/uk/gov/hmcts/reform/migration/queries/CcdElasticSearchQueries.java @@ -71,9 +71,14 @@ public static BoolQueryBuilder caseFlagsInternalFieldsQuery() { return QueryBuilders.boolQuery() .must(existsQuery("data.caseFlags")) - .should(QueryBuilders.boolQuery() + .must(QueryBuilders.boolQuery() .should(QueryBuilders.matchQuery("data.caseFlags.value.caseFlagType", "anonymity")) + .should(QueryBuilders.matchQuery("data.caseFlags.value.caseFlagType", "complexCase")) .should(QueryBuilders.matchQuery("data.caseFlags.value.caseFlagType", "detainedImmigrationAppeal")) + .should(QueryBuilders.matchQuery("data.caseFlags.value.caseFlagType", "foreignNationalOffender")) + .should(QueryBuilders.matchQuery("data.caseFlags.value.caseFlagType", "potentiallyViolentPerson")) + .should(QueryBuilders.matchQuery("data.caseFlags.value.caseFlagType", "unacceptableCustomerBehaviour")) + .should(QueryBuilders.matchQuery("data.caseFlags.value.caseFlagType", "unaccompaniedMinor")) .minimumShouldMatch(1)) .mustNot( QueryBuilders.boolQuery()