Merge pull request #307 from InseeFr/devIdUERawProcess

Survey Unit Id process and export

Author: alexisszmundy

src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java

@@ -14,6 +14,7 @@ public class SurveyUnitSimplified {
 	private String questionnaireId;
 	private String campaignId;
 	private String interrogationId;
+	private String surveyUnitId;
 	private Mode mode;
 	private List<VariableModel> variablesUpdate;
 	private List<VariableModel> externalVariables;

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -251,6 +251,7 @@ public ResponseEntity<SurveyUnitSimplified> getLatestByInterrogationOneObject(@R
                 .questionnaireId(responses.getFirst().getQuestionnaireId())
                 .campaignId(responses.getFirst().getCampaignId())
                 .interrogationId(responses.getFirst().getInterrogationId())
+                .surveyUnitId(responses.getFirst().getIdUE())
                 .variablesUpdate(outputVariables)
                 .externalVariables(outputExternalVariables)
                 .build());
@@ -280,6 +281,7 @@ public ResponseEntity<List<SurveyUnitSimplified>> getLatestForInterrogationList(
                             .questionnaireId(responses.getFirst().getQuestionnaireId())
                             .campaignId(responses.getFirst().getCampaignId())
                             .interrogationId(responses.getFirst().getInterrogationId())
+                            .surveyUnitId(responses.getFirst().getIdUE())
                             .mode(mode)
                             .variablesUpdate(outputVariables)
                             .externalVariables(outputExternalVariables)

src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java

@@ -19,6 +19,7 @@ public class SurveyUnitModel {
 	private String questionnaireId;
 	private String campaignId;
 	private String interrogationId;
+	private String idUE;
 	private DataState state;
 	private Mode mode;
 	private String contextualId;

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java

@@ -123,7 +123,7 @@ public DataProcessResult processRawData(String campaignName, List<String> interr
                 int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize());
                 List<String> interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex);
 
-                List<LunaticJsonRawDataModel> rawData = getRawData(campaignName,mode, interrogationIdToProcess);
+                List<LunaticJsonRawDataModel> rawData = getRawData(campaignName, mode, interrogationIdToProcess);
 
                 List<SurveyUnitModel> surveyUnitModels = convertRawData(
                         rawData,
@@ -210,6 +210,7 @@ public List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawDat
                         .questionnaireId(rawData.questionnaireId())
                         .mode(rawData.mode())
                         .interrogationId(rawData.interrogationId())
+                        .idUE(rawData.idUE())
                         .contextualId(contextualId)
                         .validationDate(validationDate)
                         .isCapturedIndirectly(isCapturedIndirectly)

src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java

@@ -76,6 +76,7 @@ private static SurveyUnitModel createFormattedSurveyUnitModel(
                 .questionnaireId(sampleSurveyUnitModel.getQuestionnaireId())
                 .campaignId(sampleSurveyUnitModel.getCampaignId())
                 .interrogationId(interrogationId)
+                .idUE(sampleSurveyUnitModel.getIdUE())
                 .state(DataState.FORMATTED)
                 .mode(sampleSurveyUnitModel.getMode())
                 .recordDate(LocalDateTime.now().plusSeconds(1)) // Add 1 second to avoid same recordDate as COLLECTED

src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java

@@ -19,6 +19,7 @@ public class SurveyUnitDocument {
 	private String campaignId;
 	@Indexed
 	private String interrogationId;
+	private String idUE;
 	private String questionnaireId;
 	private String state;
 	@Indexed

src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java

@@ -68,14 +68,15 @@ void saveJsonRawDataFromStringTest() throws Exception {
         String campaignId = "SAMPLETEST-PARADATA-v1";
         String questionnaireId = "testIdQuest".toUpperCase();
         String interrogationId = "testinterrogationId";
+        String idUE = "testIdUE";
         Map<String,Object> json = JsonUtils.jsonToMap("{\"COLLECTED\": {\"testdata\": {\"COLLECTED\": [\"test\"]}}}");
 
         //WHEN
         ResponseEntity<String> response = rawResponseController.saveRawResponsesFromJsonBody(
                 campaignId
                 , questionnaireId
                 , interrogationId
-                , null
+                , idUE
                 , Mode.WEB
                 , json
         );
@@ -85,6 +86,7 @@ void saveJsonRawDataFromStringTest() throws Exception {
         Assertions.assertThat(lunaticJsonRawDataPersistanceStub.getMongoStub()).isNotEmpty().hasSize(1);
         Assertions.assertThat(lunaticJsonRawDataPersistanceStub.getMongoStub().getFirst().campaignId()).isNotNull().isEqualTo(campaignId);
         Assertions.assertThat(lunaticJsonRawDataPersistanceStub.getMongoStub().getFirst().questionnaireId()).isNotNull().isEqualTo(questionnaireId);
+        Assertions.assertThat(lunaticJsonRawDataPersistanceStub.getMongoStub().getFirst().idUE()).isNotNull().isEqualTo(idUE);
         Assertions.assertThat(lunaticJsonRawDataPersistanceStub.getMongoStub().getFirst().interrogationId()).isNotNull().isEqualTo(interrogationId);
         Assertions.assertThat(lunaticJsonRawDataPersistanceStub.getMongoStub().getFirst().mode()).isEqualTo(Mode.WEB);
 
@@ -145,9 +147,11 @@ void processJsonRawDataTest(){
         String campaignId = "SAMPLETEST-PARADATA-v2";
         String questionnaireId = campaignId + "_quest";
         String interrogationId = "testinterrogationId1";
+        String idUE = "testIdUE1";
         String varName = "AVIS_MAIL";
         String varValue = "TEST";
-        addJsonRawDataDocumentToStub(campaignId, questionnaireId, interrogationId, null, LocalDateTime.now(),varName, varValue);
+        addJsonRawDataDocumentToStub(campaignId, questionnaireId, interrogationId, idUE, null, LocalDateTime.now(),varName
+                , varValue);
 
         dataProcessingContextPersistancePortStub.getMongoStub().add(
                 DataProcessingContextMapper.INSTANCE.modelToDocument(
@@ -175,6 +179,7 @@ void processJsonRawDataTest(){
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getQuestionnaireId()).isNotNull().isEqualTo(questionnaireId);
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getMode()).isNotNull().isEqualTo(Mode.WEB);
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getInterrogationId()).isEqualTo(interrogationId);
+        Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdUE()).isEqualTo(idUE);
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getFileDate()).isNotNull();
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getRecordDate()).isNotNull();
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables()).isNotNull().isNotEmpty().hasSize(1);
@@ -205,7 +210,7 @@ void getRawResponsesFromJsonBody() {
         Instant recordDate = Instant.parse("2025-01-01T01:00:00.000Z");
         Instant processDate = Instant.parse("2025-01-02T01:00:00.000Z");
 
-        addJsonRawDataDocumentToStub(campaignId, questionnaireId, interrogationId,
+        addJsonRawDataDocumentToStub(campaignId, questionnaireId, interrogationId, null,
                 LocalDateTime.ofInstant(processDate, ZoneOffset.UTC),
                 LocalDateTime.ofInstant(recordDate, ZoneOffset.UTC),
                 varName, varValue);
@@ -236,11 +241,15 @@ private void addJsonRawDataDocumentToStub(String campaignId, String questionnair
         lunaticJsonRawDataPersistanceStub.getMongoStub().add(lunaticJsonDataDocument);
     }
 
-    private void addJsonRawDataDocumentToStub(String campaignId, String questionnaireId, String interrogationId,
-                                                                    LocalDateTime processDate,
-                                                                    LocalDateTime recordDate,
-                                                                    String variableName, String variableValue)  {
-
+    private void addJsonRawDataDocumentToStub(String campaignId,
+                                              String questionnaireId,
+                                              String interrogationId,
+                                              String idUE,
+                                              LocalDateTime processDate,
+                                              LocalDateTime recordDate,
+                                              String variableName,
+                                              String variableValue)
+    {
         Map<String, Object> jsonMap = Map.of(
                 "COLLECTED", Map.of(variableName, Map.of("COLLECTED", variableValue)),
                 "EXTERNAL", Map.of(variableName + "_EXTERNAL", variableValue + "_EXTERNAL")
@@ -251,6 +260,7 @@ private void addJsonRawDataDocumentToStub(String campaignId, String questionnair
                 .questionnaireId(questionnaireId)
                 .mode(Mode.WEB)
                 .interrogationId(interrogationId)
+                .idUE(idUE)
                 .recordDate(recordDate)
                 .processDate(processDate)
                 .data(jsonMap)

src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java

@@ -52,6 +52,7 @@ class ResponseControllerTest {
     //Constants
     static final String DEFAULT_INTERROGATION_ID = "TESTINTERROGATIONID";
     static final String DEFAULT_QUESTIONNAIRE_ID = "TESTQUESTIONNAIREID";
+    static final String DEFAULT_ID_UE = "TESTIDUE";
     static final String CAMPAIGN_ID_WITH_DDI = "SAMPLETEST-PARADATA-v1";
     static final String QUESTIONNAIRE_ID_WITH_DDI = "quest_model_famille_AD_ttp".toUpperCase();
 
@@ -155,6 +156,7 @@ void findResponsesByUEAndQuestionnaireTest() {
         Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
         Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty();
         Assertions.assertThat(response.getBody().getFirst().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID);
+        Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(DEFAULT_ID_UE);
         Assertions.assertThat(response.getBody().getFirst().getQuestionnaireId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID);
     }
 
@@ -167,6 +169,7 @@ void getLatestByUETest() {
         Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
         Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty();
         Assertions.assertThat(response.getBody().getFirst().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID);
+        Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(DEFAULT_ID_UE);
         Assertions.assertThat(response.getBody().getFirst().getQuestionnaireId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID);
         Assertions.assertThat(response.getBody().getFirst().getFileDate()).hasMonth(Month.FEBRUARY);
     }
@@ -178,6 +181,7 @@ void getLatestByUEOneObjectTest() {
         Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
         Assertions.assertThat(response.getBody()).isNotNull();
         Assertions.assertThat(response.getBody().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID);
+        Assertions.assertThat(response.getBody().getSurveyUnitId()).isEqualTo(DEFAULT_ID_UE);
         Assertions.assertThat(response.getBody().getQuestionnaireId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID);
     }
 
@@ -188,6 +192,7 @@ void getLatestForUEListTest() {
         Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
         Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty();
         Assertions.assertThat(response.getBody().getFirst().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID);
+        Assertions.assertThat(response.getBody().getFirst().getSurveyUnitId()).isEqualTo(DEFAULT_ID_UE);
     }
 
     // Perret tests

src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java

@@ -22,12 +22,13 @@ class Utils {
     //Constants
     static final String DEFAULT_INTERROGATION_ID = "TESTINTERROGATIONID";
     static final String DEFAULT_QUESTIONNAIRE_ID = "TESTQUESTIONNAIREID";
+    static final String DEFAULT_ID_UE = "TESTIDUE";
 
     static void reset(SurveyUnitPersistencePortStub surveyUnitPersistencePortStub) throws IOException {
         //MongoDB stub management
         surveyUnitPersistencePortStub.getMongoStub().clear();
 
-        addAdditionalSurveyUnitModelToMongoStub("TEST-TABLEAUX", DEFAULT_QUESTIONNAIRE_ID,
+        addAdditionalSurveyUnitModelToMongoStub("TEST-TABLEAUX", DEFAULT_QUESTIONNAIRE_ID, DEFAULT_ID_UE,
                 LocalDateTime.of(2023, 1, 1, 0, 0, 0),
                 LocalDateTime.of(2024, 1, 1, 0, 0, 0),
                 surveyUnitPersistencePortStub);
@@ -174,13 +175,13 @@ static void addAdditionalSurveyUnitModelToMongoStub(String questionnaireId, Surv
     }
 
     static void addAdditionalSurveyUnitModelToMongoStub(String campaignId, String questionnaireId, SurveyUnitPersistencePortStub surveyUnitPersistencePortStub) {
-        addAdditionalSurveyUnitModelToMongoStub(campaignId,questionnaireId,
+        addAdditionalSurveyUnitModelToMongoStub(campaignId,questionnaireId, DEFAULT_ID_UE,
                 LocalDateTime.of(2023, 2, 2, 0, 0, 0),
                 LocalDateTime.of(2024, 2, 2, 0, 0, 0),
                 surveyUnitPersistencePortStub);
     }
 
-    static void addAdditionalSurveyUnitModelToMongoStub(String campaignId, String questionnaireId,
+    static void addAdditionalSurveyUnitModelToMongoStub(String campaignId, String questionnaireId, String idUE,
                                                         LocalDateTime fileDate, LocalDateTime recordDate,
                                                         SurveyUnitPersistencePortStub surveyUnitPersistencePortStub) {
         List<VariableModel> externalVariableList = new ArrayList<>();
@@ -219,6 +220,7 @@ static void addAdditionalSurveyUnitModelToMongoStub(String campaignId, String qu
                 .campaignId(campaignId)
                 .mode(Mode.WEB)
                 .interrogationId(DEFAULT_INTERROGATION_ID)
+                .idUE(idUE)
                 .questionnaireId(questionnaireId)
                 .state(DataState.COLLECTED)
                 .fileDate(fileDate)