Skip to content

Commit 48df867

Browse files
alexisszmundyalexisszmundy
authored
Merge pull request #188 from InseeFr/devFormatedState
Data verification produces FORMATTED data state
2 parents 31d374c + 797012c commit 48df867

File tree

4 files changed

+105
-30
lines changed

4 files changed

+105
-30
lines changed

‎src/main/java/fr/insee/genesis/domain/model/surveyunit/DataState.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
package fr.insee.genesis.domain.model.surveyunit;
22

33
public enum DataState {
4-
EDITED, FORCED, INPUTED, PREVIOUS, COLLECTED
4+
EDITED, FORCED, INPUTED, PREVIOUS, COLLECTED, FORMATTED
55
}

‎src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java‎

Lines changed: 38 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@
1313
import java.util.Comparator;
1414
import java.util.EnumMap;
1515
import java.util.HashSet;
16+
import java.util.LinkedHashMap;
1617
import java.util.List;
18+
import java.util.Map;
1719
import java.util.Optional;
1820
import java.util.Set;
1921
import java.util.regex.Matcher;
@@ -45,7 +47,7 @@ private DataVerifier() {
4547
* @param variablesMap VariablesMap containing definitions of each variable
4648
*/
4749
public static void verifySurveyUnits(List<SurveyUnitModel> surveyUnitModelsList, VariablesMap variablesMap){
48-
List<SurveyUnitModel> surveyUnitModelsListForced = new ArrayList<>(); // Created FORCED SU models
50+
List<SurveyUnitModel> surveyUnitModelsListFormatted = new ArrayList<>(); // Created FORCED SU models
4951

5052
for(String interrogationId : getInterrogationIds(surveyUnitModelsList)) { // For each id of the list
5153
List<SurveyUnitModel> srcSurveyUnitModelsOfInterrogationId = surveyUnitModelsList.stream().filter(element -> element.getInterrogationId().equals(interrogationId)).toList();
@@ -58,34 +60,34 @@ public static void verifySurveyUnits(List<SurveyUnitModel> surveyUnitModelsList,
5860

5961
//Create FORCED if any corrected variable
6062
if(!correctedCollectedVariables.isEmpty() || !correctedExternalVariables.isEmpty()){
61-
SurveyUnitModel newForcedSurveyUnitModel = createForcedSurveyUnitModel(surveyUnitModelsList, interrogationId, correctedCollectedVariables, correctedExternalVariables);
62-
surveyUnitModelsListForced.add(newForcedSurveyUnitModel);
63+
SurveyUnitModel newFormattedSurveyUnitModel = createFormattedSurveyUnitModel(surveyUnitModelsList, interrogationId, correctedCollectedVariables, correctedExternalVariables);
64+
surveyUnitModelsListFormatted.add(newFormattedSurveyUnitModel);
6365
}
6466
}
65-
surveyUnitModelsList.addAll(surveyUnitModelsListForced);
67+
surveyUnitModelsList.addAll(surveyUnitModelsListFormatted);
6668
}
6769

68-
private static SurveyUnitModel createForcedSurveyUnitModel(
70+
private static SurveyUnitModel createFormattedSurveyUnitModel(
6971
List<SurveyUnitModel> surveyUnitModelsList,
7072
String interrogationId,
7173
List<VariableModel> correctedCollectedVariables,
7274
List<VariableModel> correctedExternalVariables
7375
) {
7476
SurveyUnitModel sampleSurveyUnitModel = surveyUnitModelsList.stream().filter(element -> element.getInterrogationId().equals(interrogationId)).toList().getFirst();
75-
SurveyUnitModel newForcedSurveyUnitModel = SurveyUnitModel.builder()
77+
SurveyUnitModel newFormattedSurveyUnitModel = SurveyUnitModel.builder()
7678
.questionnaireId(sampleSurveyUnitModel.getQuestionnaireId())
7779
.campaignId(sampleSurveyUnitModel.getCampaignId())
7880
.interrogationId(interrogationId)
79-
.state(DataState.FORCED)
81+
.state(DataState.FORMATTED)
8082
.mode(sampleSurveyUnitModel.getMode())
81-
.recordDate(LocalDateTime.now())
83+
.recordDate(LocalDateTime.now().plusSeconds(1)) // Add 1 second to avoid same recordDate as COLLECTED
8284
.fileDate(sampleSurveyUnitModel.getFileDate())
8385
.collectedVariables(new ArrayList<>())
8486
.externalVariables(new ArrayList<>())
8587
.build();
8688

8789
for(VariableModel correctedCollectedVariable : correctedCollectedVariables){
88-
newForcedSurveyUnitModel.getCollectedVariables().add(
90+
newFormattedSurveyUnitModel.getCollectedVariables().add(
8991
VariableModel.builder()
9092
.varId(correctedCollectedVariable.varId())
9193
.value(correctedCollectedVariable.value())
@@ -97,7 +99,7 @@ private static SurveyUnitModel createForcedSurveyUnitModel(
9799
}
98100

99101
for(VariableModel correctedExternalVariable : correctedExternalVariables){
100-
newForcedSurveyUnitModel.getExternalVariables().add(
102+
newFormattedSurveyUnitModel.getExternalVariables().add(
101103
VariableModel.builder()
102104
.varId(correctedExternalVariable.varId())
103105
.value(correctedExternalVariable.value())
@@ -107,7 +109,7 @@ private static SurveyUnitModel createForcedSurveyUnitModel(
107109
.build()
108110
);
109111
}
110-
return newForcedSurveyUnitModel;
112+
return newFormattedSurveyUnitModel;
111113
}
112114

113115
/**
@@ -132,7 +134,8 @@ private static Set<String> getInterrogationIds(List<SurveyUnitModel> surveyUnitM
132134
* @param correctedCollectedVariables FORCED document variables
133135
*/
134136
private static void collectedVariablesManagement(List<SurveyUnitModel> srcSurveyUnitModelsOfInterrogationId, VariablesMap variablesMap, List<VariableModel> correctedCollectedVariables){
135-
Set<String> variableNames = new HashSet<>();
137+
Map<String,List<Integer>> variableIterations = new LinkedHashMap<>();
138+
136139
List<VariableModel> variablesToVerify = new ArrayList<>();
137140

138141
//Sort from more priority to less
@@ -141,10 +144,7 @@ private static void collectedVariablesManagement(List<SurveyUnitModel> srcSurvey
141144
//Get more priority variables to verify
142145
for(SurveyUnitModel srcSurveyUnitModel : sortedSurveyUnitModels){
143146
for(VariableModel collectedVariable : srcSurveyUnitModel.getCollectedVariables()){
144-
if(!variableNames.contains(collectedVariable.varId())){
145-
variableNames.add(collectedVariable.varId());
146-
variablesToVerify.add(collectedVariable);
147-
}
147+
addIteration(collectedVariable, variableIterations, variablesToVerify);
148148
}
149149
}
150150

@@ -164,6 +164,27 @@ private static void collectedVariablesManagement(List<SurveyUnitModel> srcSurvey
164164
}
165165
}
166166

167+
private static void addIteration(VariableModel variableToCheck, Map<String, List<Integer>> variableIterations, List<VariableModel> variablesToVerify) {
168+
String varIdToCheck = variableToCheck.varId();
169+
Integer iterationToCheck = variableToCheck.iteration();
170+
171+
if(!variableIterations.containsKey(varIdToCheck)
172+
|| !variableIterations.get(varIdToCheck).contains(iterationToCheck)){
173+
List<Integer> iterations = variableIterations.containsKey(varIdToCheck) ?
174+
variableIterations.get(varIdToCheck)
175+
: new ArrayList<>();
176+
if(!iterations.contains(iterationToCheck)){
177+
iterations.add(iterationToCheck);
178+
}
179+
variableIterations.put(
180+
varIdToCheck,
181+
iterations
182+
);
183+
184+
variablesToVerify.add(variableToCheck);
185+
}
186+
}
187+
167188
private static VariableModel verifyVariable(VariableModel variableModel, fr.insee.bpm.metadata.model.Variable variableDefinition) {
168189
if(isParseError(variableModel.value(), variableDefinition.getType())){
169190
return VariableModel.builder()
@@ -178,7 +199,7 @@ private static VariableModel verifyVariable(VariableModel variableModel, fr.inse
178199
}
179200

180201
private static void externalVariablesManagement(List<SurveyUnitModel> srcSuModels, VariablesMap variablesMap, List<VariableModel> correctedExternalVariables) {
181-
//COLLECTED only
202+
//External variables are in COLLECTED documents only
182203
Optional<SurveyUnitModel> surveyUnitModelOptional = srcSuModels.stream().filter(
183204
surveyUnitModel -> surveyUnitModel.getState().equals(DataState.COLLECTED)
184205
).findFirst();

‎src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,7 @@ void saveEditedTest_DocumentEdited() {
448448
}
449449

450450
@Test
451-
void saveEditedTest_DocumentForced() {
451+
void saveEditedTest_DocumentFormatted() {
452452
//GIVEN
453453
surveyUnitPersistencePortStub.getMongoStub().clear();
454454
String campaignId = CAMPAIGN_ID_WITH_DDI;
@@ -495,10 +495,10 @@ void saveEditedTest_DocumentForced() {
495495
//THEN
496496
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(2);
497497

498-
//FORCED document assertions
498+
//FORMATTED document assertions
499499
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getCampaignId()).isEqualTo(campaignId);
500500
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getQuestionnaireId()).isEqualTo(questionnaireId);
501-
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getState()).isEqualTo(DataState.FORCED);
501+
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getState()).isEqualTo(DataState.FORMATTED);
502502
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getMode()).isEqualTo(Mode.WEB);
503503
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getFileDate()).isNull();
504504
Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getRecordDate()).isNotNull();

‎src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java‎

Lines changed: 63 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ void shouldHandleEmptyValuesList() {
7575
}
7676

7777
@Test
78-
void shouldAddForcedSurveyUnit_WhenInvalidCollectedVariable() {
78+
void shouldAddFormattedSurveyUnit_WhenInvalidCollectedVariable() {
7979
// GIVEN
8080
// Add invalid value
8181
surveyUnits.clear();
@@ -107,13 +107,13 @@ void shouldAddForcedSurveyUnit_WhenInvalidCollectedVariable() {
107107
// WHEN
108108
DataVerifier.verifySurveyUnits(surveyUnits, variablesMap);
109109

110-
// THEN, check FORCED value was added
110+
// THEN, check FORMATTED value was added
111111
Assertions.assertEquals(2, surveyUnits.size());
112-
Assertions.assertEquals(DataState.FORCED, surveyUnits.get(1).getState());
112+
Assertions.assertEquals(DataState.FORMATTED, surveyUnits.get(1).getState());
113113
}
114114

115115
@Test
116-
void shouldNotAddForcedSurveyUnit_WhenAllVariablesAreValid() {
116+
void shouldNotAddFormattedSurveyUnit_WhenAllVariablesAreValid() {
117117
// WHEN
118118
DataVerifier.verifySurveyUnits(surveyUnits, variablesMap);
119119

@@ -122,7 +122,7 @@ void shouldNotAddForcedSurveyUnit_WhenAllVariablesAreValid() {
122122
}
123123

124124
@Test
125-
void shouldAddForcedSurveyUnit_WhenInvalidExternalVariable() {
125+
void shouldAddFormattedSurveyUnit_WhenInvalidExternalVariable() {
126126
//Add surveyUnit with invalid external Variable
127127
VariableModel extVar = VariableModel.builder()
128128
.varId("var2")
@@ -147,12 +147,12 @@ void shouldAddForcedSurveyUnit_WhenInvalidExternalVariable() {
147147

148148
// THEN
149149
Assertions.assertEquals(3, surveyUnits.size());
150-
Assertions.assertEquals(DataState.FORCED, surveyUnits.get(2).getState());
150+
Assertions.assertEquals(DataState.FORMATTED, surveyUnits.get(2).getState());
151151
Assertions.assertEquals(1, surveyUnits.get(1).getExternalVariables().size());
152152
}
153153

154154
@Test
155-
void shouldCorrectInvalidValuesInForcedSurveyUnit() {
155+
void shouldCorrectInvalidValuesInFormattedSurveyUnit() {
156156
// GIVEN
157157
// ADD invalid values
158158
surveyUnits.clear();
@@ -190,11 +190,65 @@ void shouldCorrectInvalidValuesInForcedSurveyUnit() {
190190
// WHEN
191191
DataVerifier.verifySurveyUnits(surveyUnits, variablesMap);
192192

193-
// FORCED values added
193+
// THEN FORMATTED values added
194194
SurveyUnitModel forcedUnit = surveyUnits.get(1);
195-
Assertions.assertEquals(DataState.FORCED, forcedUnit.getState());
195+
Assertions.assertEquals(DataState.FORMATTED, forcedUnit.getState());
196196
Assertions.assertEquals(1, forcedUnit.getCollectedVariables().size());
197197
Assertions.assertEquals("", forcedUnit.getCollectedVariables().getFirst().value()); // Corrected values
198198
}
199199

200+
@Test
201+
void shouldCorrectInvalidIterationOnFormattedSurveyUnit() {
202+
// GIVEN
203+
// ADD invalid values
204+
surveyUnits.clear();
205+
VariableModel collectedVariable1 = VariableModel.builder()
206+
.varId("var1")
207+
.value("123")
208+
.scope("loop1")
209+
.iteration(1)
210+
.parentId("parent1")
211+
.build();
212+
VariableModel collectedVariable2 = VariableModel.builder()
213+
.varId("var1")
214+
.value("invalid")
215+
.scope("loop1")
216+
.iteration(2)
217+
.parentId("parent1")
218+
.build();
219+
VariableModel collectedVariable3 = VariableModel.builder()
220+
.varId("var2")
221+
.value("false")
222+
.scope("loop2")
223+
.iteration(1)
224+
.parentId("parent2")
225+
.build();
226+
VariableModel collectedVariable4 = VariableModel.builder()
227+
.varId("var2")
228+
.value("Not a boolean")
229+
.scope("loop2")
230+
.iteration(2)
231+
.parentId("parent2")
232+
.build();
233+
SurveyUnitModel surveyUnit = SurveyUnitModel.builder()
234+
.interrogationId("UE1100000001")
235+
.questionnaireId("Quest1")
236+
.campaignId("Camp1")
237+
.state(DataState.COLLECTED)
238+
.collectedVariables(List.of(collectedVariable1, collectedVariable2, collectedVariable3, collectedVariable4))
239+
.externalVariables(List.of())
240+
.build();
241+
surveyUnits.add(surveyUnit);
242+
243+
// WHEN
244+
DataVerifier.verifySurveyUnits(surveyUnits, variablesMap);
245+
246+
// THEN FORMATTED values added
247+
Assertions.assertTrue(surveyUnits.size() > 1);
248+
SurveyUnitModel forcedUnit = surveyUnits.get(1);
249+
Assertions.assertEquals(DataState.FORMATTED, forcedUnit.getState());
250+
Assertions.assertEquals(2, forcedUnit.getCollectedVariables().size());
251+
Assertions.assertEquals("", forcedUnit.getCollectedVariables().getFirst().value()); // Corrected values
252+
Assertions.assertEquals("", forcedUnit.getCollectedVariables().get(1).value());
253+
}
200254
}

0 commit comments

Comments
 (0)