diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/DataState.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/DataState.java index 2f6ffd8b..37ac56d5 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/DataState.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/DataState.java @@ -1,5 +1,5 @@ package fr.insee.genesis.domain.model.surveyunit; public enum DataState { - EDITED, FORCED, INPUTED, PREVIOUS, COLLECTED + EDITED, FORCED, INPUTED, PREVIOUS, COLLECTED, FORMATTED } diff --git a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java index 9c06bad0..3246442d 100644 --- a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java +++ b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java @@ -13,7 +13,9 @@ import java.util.Comparator; import java.util.EnumMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.regex.Matcher; @@ -45,7 +47,7 @@ private DataVerifier() { * @param variablesMap VariablesMap containing definitions of each variable */ public static void verifySurveyUnits(List surveyUnitModelsList, VariablesMap variablesMap){ - List surveyUnitModelsListForced = new ArrayList<>(); // Created FORCED SU models + List surveyUnitModelsListFormatted = new ArrayList<>(); // Created FORCED SU models for(String interrogationId : getInterrogationIds(surveyUnitModelsList)) { // For each id of the list List srcSurveyUnitModelsOfInterrogationId = surveyUnitModelsList.stream().filter(element -> element.getInterrogationId().equals(interrogationId)).toList(); @@ -58,34 +60,34 @@ public static void verifySurveyUnits(List surveyUnitModelsList, //Create FORCED if any corrected variable if(!correctedCollectedVariables.isEmpty() || !correctedExternalVariables.isEmpty()){ - SurveyUnitModel newForcedSurveyUnitModel = createForcedSurveyUnitModel(surveyUnitModelsList, interrogationId, correctedCollectedVariables, correctedExternalVariables); - surveyUnitModelsListForced.add(newForcedSurveyUnitModel); + SurveyUnitModel newFormattedSurveyUnitModel = createFormattedSurveyUnitModel(surveyUnitModelsList, interrogationId, correctedCollectedVariables, correctedExternalVariables); + surveyUnitModelsListFormatted.add(newFormattedSurveyUnitModel); } } - surveyUnitModelsList.addAll(surveyUnitModelsListForced); + surveyUnitModelsList.addAll(surveyUnitModelsListFormatted); } - private static SurveyUnitModel createForcedSurveyUnitModel( + private static SurveyUnitModel createFormattedSurveyUnitModel( List surveyUnitModelsList, String interrogationId, List correctedCollectedVariables, List correctedExternalVariables ) { SurveyUnitModel sampleSurveyUnitModel = surveyUnitModelsList.stream().filter(element -> element.getInterrogationId().equals(interrogationId)).toList().getFirst(); - SurveyUnitModel newForcedSurveyUnitModel = SurveyUnitModel.builder() + SurveyUnitModel newFormattedSurveyUnitModel = SurveyUnitModel.builder() .questionnaireId(sampleSurveyUnitModel.getQuestionnaireId()) .campaignId(sampleSurveyUnitModel.getCampaignId()) .interrogationId(interrogationId) - .state(DataState.FORCED) + .state(DataState.FORMATTED) .mode(sampleSurveyUnitModel.getMode()) - .recordDate(LocalDateTime.now()) + .recordDate(LocalDateTime.now().plusSeconds(1)) // Add 1 second to avoid same recordDate as COLLECTED .fileDate(sampleSurveyUnitModel.getFileDate()) .collectedVariables(new ArrayList<>()) .externalVariables(new ArrayList<>()) .build(); for(VariableModel correctedCollectedVariable : correctedCollectedVariables){ - newForcedSurveyUnitModel.getCollectedVariables().add( + newFormattedSurveyUnitModel.getCollectedVariables().add( VariableModel.builder() .varId(correctedCollectedVariable.varId()) .value(correctedCollectedVariable.value()) @@ -97,7 +99,7 @@ private static SurveyUnitModel createForcedSurveyUnitModel( } for(VariableModel correctedExternalVariable : correctedExternalVariables){ - newForcedSurveyUnitModel.getExternalVariables().add( + newFormattedSurveyUnitModel.getExternalVariables().add( VariableModel.builder() .varId(correctedExternalVariable.varId()) .value(correctedExternalVariable.value()) @@ -107,7 +109,7 @@ private static SurveyUnitModel createForcedSurveyUnitModel( .build() ); } - return newForcedSurveyUnitModel; + return newFormattedSurveyUnitModel; } /** @@ -132,7 +134,8 @@ private static Set getInterrogationIds(List surveyUnitM * @param correctedCollectedVariables FORCED document variables */ private static void collectedVariablesManagement(List srcSurveyUnitModelsOfInterrogationId, VariablesMap variablesMap, List correctedCollectedVariables){ - Set variableNames = new HashSet<>(); + Map> variableIterations = new LinkedHashMap<>(); + List variablesToVerify = new ArrayList<>(); //Sort from more priority to less @@ -141,10 +144,7 @@ private static void collectedVariablesManagement(List srcSurvey //Get more priority variables to verify for(SurveyUnitModel srcSurveyUnitModel : sortedSurveyUnitModels){ for(VariableModel collectedVariable : srcSurveyUnitModel.getCollectedVariables()){ - if(!variableNames.contains(collectedVariable.varId())){ - variableNames.add(collectedVariable.varId()); - variablesToVerify.add(collectedVariable); - } + addIteration(collectedVariable, variableIterations, variablesToVerify); } } @@ -164,6 +164,27 @@ private static void collectedVariablesManagement(List srcSurvey } } + private static void addIteration(VariableModel variableToCheck, Map> variableIterations, List variablesToVerify) { + String varIdToCheck = variableToCheck.varId(); + Integer iterationToCheck = variableToCheck.iteration(); + + if(!variableIterations.containsKey(varIdToCheck) + || !variableIterations.get(varIdToCheck).contains(iterationToCheck)){ + List iterations = variableIterations.containsKey(varIdToCheck) ? + variableIterations.get(varIdToCheck) + : new ArrayList<>(); + if(!iterations.contains(iterationToCheck)){ + iterations.add(iterationToCheck); + } + variableIterations.put( + varIdToCheck, + iterations + ); + + variablesToVerify.add(variableToCheck); + } + } + private static VariableModel verifyVariable(VariableModel variableModel, fr.insee.bpm.metadata.model.Variable variableDefinition) { if(isParseError(variableModel.value(), variableDefinition.getType())){ return VariableModel.builder() @@ -178,7 +199,7 @@ private static VariableModel verifyVariable(VariableModel variableModel, fr.inse } private static void externalVariablesManagement(List srcSuModels, VariablesMap variablesMap, List correctedExternalVariables) { - //COLLECTED only + //External variables are in COLLECTED documents only Optional surveyUnitModelOptional = srcSuModels.stream().filter( surveyUnitModel -> surveyUnitModel.getState().equals(DataState.COLLECTED) ).findFirst(); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index dd867fdf..bf7ff8a8 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -448,7 +448,7 @@ void saveEditedTest_DocumentEdited() { } @Test - void saveEditedTest_DocumentForced() { + void saveEditedTest_DocumentFormatted() { //GIVEN surveyUnitPersistencePortStub.getMongoStub().clear(); String campaignId = CAMPAIGN_ID_WITH_DDI; @@ -495,10 +495,10 @@ void saveEditedTest_DocumentForced() { //THEN Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(2); - //FORCED document assertions + //FORMATTED document assertions Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getCampaignId()).isEqualTo(campaignId); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getQuestionnaireId()).isEqualTo(questionnaireId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getState()).isEqualTo(DataState.FORCED); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getState()).isEqualTo(DataState.FORMATTED); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getFileDate()).isNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getRecordDate()).isNotNull(); diff --git a/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java b/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java index 26c88f88..b3ae255f 100644 --- a/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java +++ b/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java @@ -75,7 +75,7 @@ void shouldHandleEmptyValuesList() { } @Test - void shouldAddForcedSurveyUnit_WhenInvalidCollectedVariable() { + void shouldAddFormattedSurveyUnit_WhenInvalidCollectedVariable() { // GIVEN // Add invalid value surveyUnits.clear(); @@ -107,13 +107,13 @@ void shouldAddForcedSurveyUnit_WhenInvalidCollectedVariable() { // WHEN DataVerifier.verifySurveyUnits(surveyUnits, variablesMap); - // THEN, check FORCED value was added + // THEN, check FORMATTED value was added Assertions.assertEquals(2, surveyUnits.size()); - Assertions.assertEquals(DataState.FORCED, surveyUnits.get(1).getState()); + Assertions.assertEquals(DataState.FORMATTED, surveyUnits.get(1).getState()); } @Test - void shouldNotAddForcedSurveyUnit_WhenAllVariablesAreValid() { + void shouldNotAddFormattedSurveyUnit_WhenAllVariablesAreValid() { // WHEN DataVerifier.verifySurveyUnits(surveyUnits, variablesMap); @@ -122,7 +122,7 @@ void shouldNotAddForcedSurveyUnit_WhenAllVariablesAreValid() { } @Test - void shouldAddForcedSurveyUnit_WhenInvalidExternalVariable() { + void shouldAddFormattedSurveyUnit_WhenInvalidExternalVariable() { //Add surveyUnit with invalid external Variable VariableModel extVar = VariableModel.builder() .varId("var2") @@ -147,12 +147,12 @@ void shouldAddForcedSurveyUnit_WhenInvalidExternalVariable() { // THEN Assertions.assertEquals(3, surveyUnits.size()); - Assertions.assertEquals(DataState.FORCED, surveyUnits.get(2).getState()); + Assertions.assertEquals(DataState.FORMATTED, surveyUnits.get(2).getState()); Assertions.assertEquals(1, surveyUnits.get(1).getExternalVariables().size()); } @Test - void shouldCorrectInvalidValuesInForcedSurveyUnit() { + void shouldCorrectInvalidValuesInFormattedSurveyUnit() { // GIVEN // ADD invalid values surveyUnits.clear(); @@ -190,11 +190,65 @@ void shouldCorrectInvalidValuesInForcedSurveyUnit() { // WHEN DataVerifier.verifySurveyUnits(surveyUnits, variablesMap); - // FORCED values added + // THEN FORMATTED values added SurveyUnitModel forcedUnit = surveyUnits.get(1); - Assertions.assertEquals(DataState.FORCED, forcedUnit.getState()); + Assertions.assertEquals(DataState.FORMATTED, forcedUnit.getState()); Assertions.assertEquals(1, forcedUnit.getCollectedVariables().size()); Assertions.assertEquals("", forcedUnit.getCollectedVariables().getFirst().value()); // Corrected values } + @Test + void shouldCorrectInvalidIterationOnFormattedSurveyUnit() { + // GIVEN + // ADD invalid values + surveyUnits.clear(); + VariableModel collectedVariable1 = VariableModel.builder() + .varId("var1") + .value("123") + .scope("loop1") + .iteration(1) + .parentId("parent1") + .build(); + VariableModel collectedVariable2 = VariableModel.builder() + .varId("var1") + .value("invalid") + .scope("loop1") + .iteration(2) + .parentId("parent1") + .build(); + VariableModel collectedVariable3 = VariableModel.builder() + .varId("var2") + .value("false") + .scope("loop2") + .iteration(1) + .parentId("parent2") + .build(); + VariableModel collectedVariable4 = VariableModel.builder() + .varId("var2") + .value("Not a boolean") + .scope("loop2") + .iteration(2) + .parentId("parent2") + .build(); + SurveyUnitModel surveyUnit = SurveyUnitModel.builder() + .interrogationId("UE1100000001") + .questionnaireId("Quest1") + .campaignId("Camp1") + .state(DataState.COLLECTED) + .collectedVariables(List.of(collectedVariable1, collectedVariable2, collectedVariable3, collectedVariable4)) + .externalVariables(List.of()) + .build(); + surveyUnits.add(surveyUnit); + + // WHEN + DataVerifier.verifySurveyUnits(surveyUnits, variablesMap); + + // THEN FORMATTED values added + Assertions.assertTrue(surveyUnits.size() > 1); + SurveyUnitModel forcedUnit = surveyUnits.get(1); + Assertions.assertEquals(DataState.FORMATTED, forcedUnit.getState()); + Assertions.assertEquals(2, forcedUnit.getCollectedVariables().size()); + Assertions.assertEquals("", forcedUnit.getCollectedVariables().getFirst().value()); // Corrected values + Assertions.assertEquals("", forcedUnit.getCollectedVariables().get(1).value()); + } }