JSON Raw data processing

src/main/java/fr/insee/genesis/Constants.java

@@ -10,11 +10,13 @@ public class Constants {
             "(^([0-9]|[0-2][0-9]|3[0-1])[\\-\\/]([0-9]|1[0-2]|0[1-9])[\\-\\/]([0-9]{4})$)";
     public static final String FILTER_RESULT_PREFIX = "FILTER_RESULT_";
     public static final String MISSING_SUFFIX = "_MISSING";
+    public static final String MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME = "lunaticjsondata";
     private static final String[] ENO_VARIABLES = {"COMMENT_QE","COMMENT_UE","HEURE_REMPL","MIN_REMPL"};
 
     public static final String MONGODB_SCHEDULE_COLLECTION_NAME = "schedules";
     public static final String LOOP_NAME_PREFIX = "BOUCLE";
     public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses";
+    public static final String MONGODB_VARIABLETYPE_COLLECTION_NAME = "variabletypes";
     public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries";
     public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY";
     public static final String VOLUMETRY_FILE_DATE_FORMAT = "yyyy_MM_dd";

src/main/java/fr/insee/genesis/configuration/Config.java

@@ -2,12 +2,14 @@
 
 import lombok.Getter;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.context.annotation.Configuration;
 
 import java.nio.file.Path;
 
 @Configuration
 @Getter
+@EnableCaching
 public class Config {
 
 	/******************************************************/

src/main/java/fr/insee/genesis/controller/dto/rawdata/LunaticJsonRawDataUnprocessedDto.java

@@ -0,0 +1,11 @@
+package fr.insee.genesis.controller.dto.rawdata;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Builder
+@Data
+public class LunaticJsonRawDataUnprocessedDto {
+    private String campaignId;
+    private String idUE;
+}

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -10,6 +10,7 @@
 import fr.insee.genesis.controller.dto.SurveyUnitDto;
 import fr.insee.genesis.controller.dto.SurveyUnitId;
 import fr.insee.genesis.controller.dto.SurveyUnitSimplified;
+import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlDataParser;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlDataSequentialParser;
@@ -22,6 +23,7 @@
 import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
 import fr.insee.genesis.domain.ports.api.LunaticXmlRawDataApiPort;
 import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
+import fr.insee.genesis.domain.ports.api.VariableTypeApiPort;
 import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
 import fr.insee.genesis.exceptions.GenesisError;
 import fr.insee.genesis.exceptions.GenesisException;
@@ -71,6 +73,7 @@ public class ResponseController {
     private final SurveyUnitQualityService surveyUnitQualityService;
     private final LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort;
     private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
+    private final VariableTypeApiPort variableTypeApiPort;
     private final FileUtils fileUtils;
     private final ControllerUtils controllerUtils;
 
@@ -79,12 +82,14 @@ public ResponseController(SurveyUnitApiPort surveyUnitService,
                               SurveyUnitQualityService surveyUnitQualityService,
                               LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort,
                               LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort,
+                              VariableTypeApiPort variableTypeApiPort,
                               FileUtils fileUtils,
                               ControllerUtils controllerUtils) {
         this.surveyUnitService = surveyUnitService;
         this.surveyUnitQualityService = surveyUnitQualityService;
         this.lunaticXmlRawDataApiPort = lunaticXmlRawDataApiPort;
         this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
+        this.variableTypeApiPort = variableTypeApiPort;
         this.fileUtils = fileUtils;
         this.controllerUtils = controllerUtils;
     }
@@ -155,6 +160,7 @@ public ResponseEntity<Object> saveResponsesFromXmlCampaignFolder(@RequestParam("
         return ResponseEntity.internalServerError().body(errors.getFirst().getMessage());
     }
 
+    //RAW DATA XML
     @Operation(summary = "Save one file of raw responses to Genesis Database, passing its path as a parameter")
     @PutMapping(path = "/lunatic-xml/raw/save-one")
     public ResponseEntity<Object> saveRawResponsesFromXmlFile(@RequestParam("pathLunaticXml") String xmlFile,
@@ -192,17 +198,19 @@ public ResponseEntity<Object> saveRawResponsesFromXmlCampaignFolder(@RequestPara
         return ResponseEntity.ok(getSuccessMessage(isAnyDataSaved));
     }
 
-    //JSON
+    //RAW DATA JSON
     @Operation(summary = "Save lunatic json data to Genesis Database from the campaign root folder")
-    @PutMapping(path = "/lunatic-json/raw/save")
+    @PutMapping(path = "/lunatic-json/raw/save-one")
     public ResponseEntity<Object> saveRawResponsesFromJsonBody(
             @RequestParam("campaignName") String campaignName,
-            @RequestParam(value = "mode", required = false) Mode modeSpecified,
+            @RequestParam("idQuest") String idQuest,
+            @RequestParam("idUE") String idUE,
+            @RequestParam(value = "mode") Mode modeSpecified,
             @RequestBody String dataJson
     ) {
         log.info("Try to import raw lunatic JSON data for campaign: {}", campaignName);
         try {
-            lunaticJsonRawDataApiPort.saveData(campaignName, dataJson, modeSpecified);
+            lunaticJsonRawDataApiPort.saveData(campaignName, idQuest, idUE, dataJson, modeSpecified);
         }catch (JsonProcessingException jpe){
             log.error(jpe.toString());
             return ResponseEntity.badRequest().body("Invalid JSON synthax");
@@ -211,6 +219,61 @@ public ResponseEntity<Object> saveRawResponsesFromJsonBody(
         return ResponseEntity.ok(SUCCESS_MESSAGE);
     }
 
+    //GET unprocessed
+    @Operation(summary = "Get campaign id and idUE from all unprocessed raw json data")
+    @GetMapping(path = "/lunatic-json/raw/get/unprocessed")
+    public ResponseEntity<List<LunaticJsonRawDataUnprocessedDto>> getUnproccessedJsonRawData(){
+        log.info("Try to get unprocessed raw JSON datas...");
+        return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataIds());
+    }
+
+    //PROCESS
+    @Operation(summary = "Process raw data of a campaign")
+    @PostMapping(path = "/lunatic-json/raw/process")
+    public ResponseEntity<Object> processJsonRawData(
+            @RequestParam("campaignName") String campaignName,
+            @RequestParam("questionnaireId") String questionnaireId,
+            @RequestBody List<String> idUEList
+    ){
+        log.info("Try to get process raw JSON datas for campaign {} and {} idUEs", campaignName, idUEList.size());
+
+        int dataCount = 0;
+        List<GenesisError> errors = new ArrayList<>();
+
+        try {
+            List<Mode> modesList = controllerUtils.getModesList(campaignName, null);
+            for (Mode mode : modesList) {
+                //Load and save metadatas into database, throw exception if none
+                VariablesMap variablesMap = readMetadatas(campaignName, mode, errors, true);
+                if (variablesMap == null) {
+                    throw new GenesisException(400,
+                            "Error during metadata parsing for mode %s :%n%s"
+                                    .formatted(mode, errors.getLast().getMessage())
+                    );
+                }
+
+                //Save converted data
+                List<SurveyUnitModel> surveyUnitModels = lunaticJsonRawDataApiPort.parseRawData(
+                        campaignName,
+                        mode,
+                        idUEList,
+                        variablesMap
+                );
+                surveyUnitService.saveSurveyUnits(surveyUnitModels);
+
+                //Update process dates
+                lunaticJsonRawDataApiPort.updateProcessDates(surveyUnitModels);
+
+                //Save metadatas
+                variableTypeApiPort.saveMetadatas(campaignName, questionnaireId, mode, variablesMap);
+                dataCount += surveyUnitModels.size();
+            }
+            return ResponseEntity.ok("%d document(s) processed".formatted(dataCount));
+        }catch (GenesisException e){ //TODO replace with spring exception handler
+            return ResponseEntity.status(e.getStatus()).body(e.getMessage());
+        }
+    }
+
 
     //SAVE ALL
     @Operation(summary = "Save all files to Genesis Database (differential data folder only), regardless of the campaign")
@@ -246,7 +309,6 @@ public ResponseEntity<Object> saveResponsesFromAllCampaignFolders(){
         return ResponseEntity.status(209).body("Data saved with " + errors.size() + " errors");
     }
 
-
 
     //DELETE
     @Operation(summary = "Delete all responses associated with a questionnaire")

src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java

@@ -24,7 +24,7 @@ public ControllerUtils(FileUtils fileUtils) {
 	public List<Mode> getModesList(String campaign, Mode modeSpecified) throws GenesisException {
 		// If a mode is specified, we treat only this mode.
 		// If no mode is specified, we treat all modes in the campaign.
-		// If no node is specified and no specs are found, we return an error
+		// If no mode is specified and no specs are found, we return an error
 		if (modeSpecified != null){
 			return Collections.singletonList(modeSpecified);
 		}

src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java

@@ -11,6 +11,8 @@
 public record LunaticJsonDataModel(
     ObjectId id,
     String campaignId,
+    String idQuest,
+    String idUE,
     Mode mode,
     String dataJson,
     LocalDateTime recordDate,

src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java

@@ -0,0 +1,17 @@
+package fr.insee.genesis.domain.model.variabletype;
+
+import fr.insee.bpm.metadata.model.VariableType;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.Map;
+
+@Data
+@Builder
+public class VariableTypeModel {
+    private String campaignId;
+    private String questionnaireId;
+    private Mode mode;
+    private Map<String, VariableType> variables;
+}

src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java

@@ -1,8 +1,23 @@
 package fr.insee.genesis.domain.ports.api;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import fr.insee.bpm.metadata.model.VariablesMap;
+import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
+
+import java.util.List;
 
 public interface LunaticJsonRawDataApiPort {
-    void saveData(String campaignName, String dataJson, Mode mode) throws JsonProcessingException;
+    void saveData(String campaignName, String idQuest, String idUE, String dataJson, Mode mode) throws JsonProcessingException;
+    List<LunaticJsonRawDataUnprocessedDto> getUnprocessedDataIds();
+
+    List<SurveyUnitModel> parseRawData(
+            String campaignName,
+            Mode mode,
+            List<String> idUEList,
+            VariablesMap variablesMap
+    );
+
+    void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
 }

src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java

@@ -0,0 +1,8 @@
+package fr.insee.genesis.domain.ports.api;
+
+import fr.insee.bpm.metadata.model.VariablesMap;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+
+public interface VariableTypeApiPort {
+    void saveMetadatas(String campaignId, String questionnaireId, Mode mode, VariablesMap variablesMap);
+}

src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java

@@ -1,7 +1,17 @@
 package fr.insee.genesis.domain.ports.spi;
 
+import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel;
+import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonDataDocument;
+
+import java.util.List;
+import java.util.Set;
 
 public interface LunaticJsonPersistancePort {
     void save(LunaticJsonDataModel lunaticJsonDataModel);
+    List<LunaticJsonDataModel> getAllUnprocessedData();
+
+    List<LunaticJsonDataDocument> findRawData(String campaignName, Mode mode, List<String> idUEList);
+
+    void updateProcessDates(String campaignId, Set<String> idUEs);
 }

src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java

@@ -0,0 +1,12 @@
+package fr.insee.genesis.domain.ports.spi;
+
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.model.variabletype.VariableTypeModel;
+import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument;
+import org.springframework.cache.annotation.Cacheable;
+
+public interface VariableTypePersistancePort {
+    void save(VariableTypeModel variableTypeModel);
+    @Cacheable("variableTypes")
+    VariableTypeDocument find(String campaignId, String questionnaireId, Mode mode);
+}