Skip to content

Data verification produces FORMATTED data state #188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alexisszmundy merged 7 commits into from
Mar 25, 2025
Merged

Data verification produces FORMATTED data state #188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4be3be5
test: add iteration dataverifier KO test
alexisszmundy Mar 19, 2025
ebd6ac9
fix: fixed Dataverifier does not put other iterations in FORCED
alexisszmundy Mar 19, 2025
4d0f221
fix: FORCED document can have same recordDate
alexisszmundy Mar 21, 2025
731c242
feat: Dataverifier FORMATTED instead of FORCED
alexisszmundy Mar 21, 2025
1ef6679
Merge branch 'main' into devFormatedState
alexisszmundy Mar 25, 2025
09d7f9e
refactor(DataVerifier): extract local variables
alexisszmundy Mar 25, 2025
797012c
Merge branch 'main' into devFormatedState
alexisszmundy Mar 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/fr/insee/genesis/domain/model/surveyunit/DataState.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
package fr.insee.genesis.domain.model.surveyunit;

public enum DataState {
EDITED, FORCED, INPUTED, PREVIOUS, COLLECTED
EDITED, FORCED, INPUTED, PREVIOUS, COLLECTED, FORMATTED
}
55 changes: 38 additions & 17 deletions src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@
import java.util.Comparator;
import java.util.EnumMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Matcher;
Expand Down Expand Up @@ -45,7 +47,7 @@ private DataVerifier() {
* @param variablesMap VariablesMap containing definitions of each variable
*/
public static void verifySurveyUnits(List<SurveyUnitModel> surveyUnitModelsList, VariablesMap variablesMap){
List<SurveyUnitModel> surveyUnitModelsListForced = new ArrayList<>(); // Created FORCED SU models
List<SurveyUnitModel> surveyUnitModelsListFormatted = new ArrayList<>(); // Created FORCED SU models

for(String interrogationId : getInterrogationIds(surveyUnitModelsList)) { // For each id of the list
List<SurveyUnitModel> srcSurveyUnitModelsOfInterrogationId = surveyUnitModelsList.stream().filter(element -> element.getInterrogationId().equals(interrogationId)).toList();
Expand All @@ -58,34 +60,34 @@ public static void verifySurveyUnits(List<SurveyUnitModel> surveyUnitModelsList,

//Create FORCED if any corrected variable
if(!correctedCollectedVariables.isEmpty() || !correctedExternalVariables.isEmpty()){
SurveyUnitModel newForcedSurveyUnitModel = createForcedSurveyUnitModel(surveyUnitModelsList, interrogationId, correctedCollectedVariables, correctedExternalVariables);
surveyUnitModelsListForced.add(newForcedSurveyUnitModel);
SurveyUnitModel newFormattedSurveyUnitModel = createFormattedSurveyUnitModel(surveyUnitModelsList, interrogationId, correctedCollectedVariables, correctedExternalVariables);
surveyUnitModelsListFormatted.add(newFormattedSurveyUnitModel);
}
}
surveyUnitModelsList.addAll(surveyUnitModelsListForced);
surveyUnitModelsList.addAll(surveyUnitModelsListFormatted);
}

private static SurveyUnitModel createForcedSurveyUnitModel(
private static SurveyUnitModel createFormattedSurveyUnitModel(
List<SurveyUnitModel> surveyUnitModelsList,
String interrogationId,
List<VariableModel> correctedCollectedVariables,
List<VariableModel> correctedExternalVariables
) {
SurveyUnitModel sampleSurveyUnitModel = surveyUnitModelsList.stream().filter(element -> element.getInterrogationId().equals(interrogationId)).toList().getFirst();
SurveyUnitModel newForcedSurveyUnitModel = SurveyUnitModel.builder()
SurveyUnitModel newFormattedSurveyUnitModel = SurveyUnitModel.builder()
.questionnaireId(sampleSurveyUnitModel.getQuestionnaireId())
.campaignId(sampleSurveyUnitModel.getCampaignId())
.interrogationId(interrogationId)
.state(DataState.FORCED)
.state(DataState.FORMATTED)
.mode(sampleSurveyUnitModel.getMode())
.recordDate(LocalDateTime.now())
.recordDate(LocalDateTime.now().plusSeconds(1)) // Add 1 second to avoid same recordDate as COLLECTED
.fileDate(sampleSurveyUnitModel.getFileDate())
.collectedVariables(new ArrayList<>())
.externalVariables(new ArrayList<>())
.build();

for(VariableModel correctedCollectedVariable : correctedCollectedVariables){
newForcedSurveyUnitModel.getCollectedVariables().add(
newFormattedSurveyUnitModel.getCollectedVariables().add(
VariableModel.builder()
.varId(correctedCollectedVariable.varId())
.value(correctedCollectedVariable.value())
Expand All @@ -97,7 +99,7 @@ private static SurveyUnitModel createForcedSurveyUnitModel(
}

for(VariableModel correctedExternalVariable : correctedExternalVariables){
newForcedSurveyUnitModel.getExternalVariables().add(
newFormattedSurveyUnitModel.getExternalVariables().add(
VariableModel.builder()
.varId(correctedExternalVariable.varId())
.value(correctedExternalVariable.value())
Expand All @@ -107,7 +109,7 @@ private static SurveyUnitModel createForcedSurveyUnitModel(
.build()
);
}
return newForcedSurveyUnitModel;
return newFormattedSurveyUnitModel;
}

/**
Expand All @@ -132,7 +134,8 @@ private static Set<String> getInterrogationIds(List<SurveyUnitModel> surveyUnitM
* @param correctedCollectedVariables FORCED document variables
*/
private static void collectedVariablesManagement(List<SurveyUnitModel> srcSurveyUnitModelsOfInterrogationId, VariablesMap variablesMap, List<VariableModel> correctedCollectedVariables){
Set<String> variableNames = new HashSet<>();
Map<String,List<Integer>> variableIterations = new LinkedHashMap<>();

List<VariableModel> variablesToVerify = new ArrayList<>();

//Sort from more priority to less
Expand All @@ -141,10 +144,7 @@ private static void collectedVariablesManagement(List<SurveyUnitModel> srcSurvey
//Get more priority variables to verify
for(SurveyUnitModel srcSurveyUnitModel : sortedSurveyUnitModels){
for(VariableModel collectedVariable : srcSurveyUnitModel.getCollectedVariables()){
if(!variableNames.contains(collectedVariable.varId())){
variableNames.add(collectedVariable.varId());
variablesToVerify.add(collectedVariable);
}
addIteration(collectedVariable, variableIterations, variablesToVerify);
}
}

Expand All @@ -164,6 +164,27 @@ private static void collectedVariablesManagement(List<SurveyUnitModel> srcSurvey
}
}

private static void addIteration(VariableModel variableToCheck, Map<String, List<Integer>> variableIterations, List<VariableModel> variablesToVerify) {
String varIdToCheck = variableToCheck.varId();
Integer iterationToCheck = variableToCheck.iteration();

if(!variableIterations.containsKey(varIdToCheck)
|| !variableIterations.get(varIdToCheck).contains(iterationToCheck)){
List<Integer> iterations = variableIterations.containsKey(varIdToCheck) ?
variableIterations.get(varIdToCheck)
: new ArrayList<>();
if(!iterations.contains(iterationToCheck)){
iterations.add(iterationToCheck);
}
variableIterations.put(
varIdToCheck,
iterations
);

variablesToVerify.add(variableToCheck);
}
}

private static VariableModel verifyVariable(VariableModel variableModel, fr.insee.bpm.metadata.model.Variable variableDefinition) {
if(isParseError(variableModel.value(), variableDefinition.getType())){
return VariableModel.builder()
Expand All @@ -178,7 +199,7 @@ private static VariableModel verifyVariable(VariableModel variableModel, fr.inse
}

private static void externalVariablesManagement(List<SurveyUnitModel> srcSuModels, VariablesMap variablesMap, List<VariableModel> correctedExternalVariables) {
//COLLECTED only
//External variables are in COLLECTED documents only
Optional<SurveyUnitModel> surveyUnitModelOptional = srcSuModels.stream().filter(
surveyUnitModel -> surveyUnitModel.getState().equals(DataState.COLLECTED)
).findFirst();
Expand Down
6 changes: 3 additions & 3 deletions src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@ void saveEditedTest_DocumentEdited() {
}

@Test
void saveEditedTest_DocumentForced() {
void saveEditedTest_DocumentFormatted() {
//GIVEN
surveyUnitPersistencePortStub.getMongoStub().clear();
String campaignId = CAMPAIGN_ID_WITH_DDI;
Expand Down Expand Up @@ -495,10 +495,10 @@ void saveEditedTest_DocumentForced() {
//THEN
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(2);

//FORCED document assertions
//FORMATTED document assertions
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getCampaignId()).isEqualTo(campaignId);
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getQuestionnaireId()).isEqualTo(questionnaireId);
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getState()).isEqualTo(DataState.FORCED);
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getState()).isEqualTo(DataState.FORMATTED);
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getMode()).isEqualTo(Mode.WEB);
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getFileDate()).isNull();
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getRecordDate()).isNotNull();
Expand Down
72 changes: 63 additions & 9 deletions src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ void shouldHandleEmptyValuesList() {
}

@Test
void shouldAddForcedSurveyUnit_WhenInvalidCollectedVariable() {
void shouldAddFormattedSurveyUnit_WhenInvalidCollectedVariable() {
// GIVEN
// Add invalid value
surveyUnits.clear();
Expand Down Expand Up @@ -107,13 +107,13 @@ void shouldAddForcedSurveyUnit_WhenInvalidCollectedVariable() {
// WHEN
DataVerifier.verifySurveyUnits(surveyUnits, variablesMap);

// THEN, check FORCED value was added
// THEN, check FORMATTED value was added
Assertions.assertEquals(2, surveyUnits.size());
Assertions.assertEquals(DataState.FORCED, surveyUnits.get(1).getState());
Assertions.assertEquals(DataState.FORMATTED, surveyUnits.get(1).getState());
}

@Test
void shouldNotAddForcedSurveyUnit_WhenAllVariablesAreValid() {
void shouldNotAddFormattedSurveyUnit_WhenAllVariablesAreValid() {
// WHEN
DataVerifier.verifySurveyUnits(surveyUnits, variablesMap);

Expand All @@ -122,7 +122,7 @@ void shouldNotAddForcedSurveyUnit_WhenAllVariablesAreValid() {
}

@Test
void shouldAddForcedSurveyUnit_WhenInvalidExternalVariable() {
void shouldAddFormattedSurveyUnit_WhenInvalidExternalVariable() {
//Add surveyUnit with invalid external Variable
VariableModel extVar = VariableModel.builder()
.varId("var2")
Expand All @@ -147,12 +147,12 @@ void shouldAddForcedSurveyUnit_WhenInvalidExternalVariable() {

// THEN
Assertions.assertEquals(3, surveyUnits.size());
Assertions.assertEquals(DataState.FORCED, surveyUnits.get(2).getState());
Assertions.assertEquals(DataState.FORMATTED, surveyUnits.get(2).getState());
Assertions.assertEquals(1, surveyUnits.get(1).getExternalVariables().size());
}

@Test
void shouldCorrectInvalidValuesInForcedSurveyUnit() {
void shouldCorrectInvalidValuesInFormattedSurveyUnit() {
// GIVEN
// ADD invalid values
surveyUnits.clear();
Expand Down Expand Up @@ -190,11 +190,65 @@ void shouldCorrectInvalidValuesInForcedSurveyUnit() {
// WHEN
DataVerifier.verifySurveyUnits(surveyUnits, variablesMap);

// FORCED values added
// THEN FORMATTED values added
SurveyUnitModel forcedUnit = surveyUnits.get(1);
Assertions.assertEquals(DataState.FORCED, forcedUnit.getState());
Assertions.assertEquals(DataState.FORMATTED, forcedUnit.getState());
Assertions.assertEquals(1, forcedUnit.getCollectedVariables().size());
Assertions.assertEquals("", forcedUnit.getCollectedVariables().getFirst().value()); // Corrected values
}

@Test
void shouldCorrectInvalidIterationOnFormattedSurveyUnit() {
// GIVEN
// ADD invalid values
surveyUnits.clear();
VariableModel collectedVariable1 = VariableModel.builder()
.varId("var1")
.value("123")
.scope("loop1")
.iteration(1)
.parentId("parent1")
.build();
VariableModel collectedVariable2 = VariableModel.builder()
.varId("var1")
.value("invalid")
.scope("loop1")
.iteration(2)
.parentId("parent1")
.build();
VariableModel collectedVariable3 = VariableModel.builder()
.varId("var2")
.value("false")
.scope("loop2")
.iteration(1)
.parentId("parent2")
.build();
VariableModel collectedVariable4 = VariableModel.builder()
.varId("var2")
.value("Not a boolean")
.scope("loop2")
.iteration(2)
.parentId("parent2")
.build();
SurveyUnitModel surveyUnit = SurveyUnitModel.builder()
.interrogationId("UE1100000001")
.questionnaireId("Quest1")
.campaignId("Camp1")
.state(DataState.COLLECTED)
.collectedVariables(List.of(collectedVariable1, collectedVariable2, collectedVariable3, collectedVariable4))
.externalVariables(List.of())
.build();
surveyUnits.add(surveyUnit);

// WHEN
DataVerifier.verifySurveyUnits(surveyUnits, variablesMap);

// THEN FORMATTED values added
Assertions.assertTrue(surveyUnits.size() > 1);
SurveyUnitModel forcedUnit = surveyUnits.get(1);
Assertions.assertEquals(DataState.FORMATTED, forcedUnit.getState());
Assertions.assertEquals(2, forcedUnit.getCollectedVariables().size());
Assertions.assertEquals("", forcedUnit.getCollectedVariables().getFirst().value()); // Corrected values
Assertions.assertEquals("", forcedUnit.getCollectedVariables().get(1).value());
}
}
Loading