Raw lunatic XML & JSON data saving into database

src/main/java/fr/insee/genesis/Constants.java

@@ -15,6 +15,8 @@ public class Constants {
     public static final String MONGODB_SCHEDULE_COLLECTION_NAME = "schedules";
     public static final String LOOP_NAME_PREFIX = "BOUCLE";
     public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses";
+    public static final String MONGODB_LUNATIC_XML_DATA_COLLECTION_NAME = "lunaticxmldata";
+    public static final String MONGODB_LUNATIC_JSON_DATA_COLLECTION_NAME = "lunaticjsondata";
     public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries";
     public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY";
     public static final String VOLUMETRY_FILE_DATE_FORMAT = "yyyy_MM_dd";

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -1,5 +1,6 @@
 package fr.insee.genesis.controller.rest.responses;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import fr.insee.bpm.exceptions.MetadataParserException;
 import fr.insee.bpm.metadata.model.VariablesMap;
 import fr.insee.bpm.metadata.reader.ddi.DDIReader;
@@ -18,6 +19,8 @@
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
 import fr.insee.genesis.domain.model.surveyunit.Variable;
+import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
+import fr.insee.genesis.domain.ports.api.LunaticXmlRawDataApiPort;
 import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
 import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
 import fr.insee.genesis.exceptions.GenesisError;
@@ -62,15 +65,26 @@ public class ResponseController {
 
     private static final String DDI_REGEX = "ddi[\\w,\\s-]+\\.xml";
     public static final String S_S = "%s/%s";
+    private static final String CAMPAIGN_ERROR = "Error for campaign {} : {}";
+    private static final String SUCCESS_MESSAGE = "Data saved";
     private final SurveyUnitApiPort surveyUnitService;
     private final SurveyUnitQualityService surveyUnitQualityService;
+    private final LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort;
+    private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
     private final FileUtils fileUtils;
     private final ControllerUtils controllerUtils;
 
     @Autowired
-    public ResponseController(SurveyUnitApiPort surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils, ControllerUtils controllerUtils) {
+    public ResponseController(SurveyUnitApiPort surveyUnitService,
+                              SurveyUnitQualityService surveyUnitQualityService,
+                              LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort,
+                              LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort,
+                              FileUtils fileUtils,
+                              ControllerUtils controllerUtils) {
         this.surveyUnitService = surveyUnitService;
         this.surveyUnitQualityService = surveyUnitQualityService;
+        this.lunaticXmlRawDataApiPort = lunaticXmlRawDataApiPort;
+        this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
         this.fileUtils = fileUtils;
         this.controllerUtils = controllerUtils;
     }
@@ -131,18 +145,75 @@ public ResponseEntity<Object> saveResponsesFromXmlCampaignFolder(@RequestParam("
                 //Don't stop if NoDataError thrown
                 log.warn(nde.getMessage());
             }catch (Exception e){
-                log.error("Error for campaign {} : {}", campaignName, e.toString());
+                log.error(CAMPAIGN_ERROR, campaignName, e.toString());
                 return ResponseEntity.status(500).body(e.getMessage());
             }
         }
 
         if (errors.isEmpty()){
-            return ResponseEntity.ok(isAnyDataSaved ? "Data saved" : "No data has been saved");
+            return ResponseEntity.ok(isAnyDataSaved ? SUCCESS_MESSAGE : "No data has been saved");
         }else{
             return ResponseEntity.internalServerError().body(errors.getFirst().getMessage());
         }
     }
 
+    @Operation(summary = "Save one file of raw responses to Genesis Database, passing its path as a parameter")
+    @PutMapping(path = "/lunatic-xml/raw/save-one")
+    public ResponseEntity<Object> saveRawResponsesFromXmlFile(@RequestParam("pathLunaticXml") String xmlFile,
+                                                           @RequestParam(value = "mode") Mode modeSpecified
+    )throws Exception {
+        log.info(String.format("Try to read Xml file : %s", xmlFile));
+        Path filepath = Paths.get(xmlFile);
+
+        return treatRawXmlFile(filepath, modeSpecified);
+    }
+
+    @Operation(summary = "Save multiple raw files to Genesis Database from the campaign root folder")
+    @PutMapping(path = "/lunatic-xml/raw/save-folder")
+    public ResponseEntity<Object> saveRawResponsesFromXmlCampaignFolder(@RequestParam("campaignName") String campaignName,
+                                                                     @RequestParam(value = "mode", required = false) Mode modeSpecified
+    )throws Exception {
+        boolean isAnyDataSaved = false;
+
+        log.info("Try to import raw lunatic XML data for campaign : {}", campaignName);
+
+        List<Mode> modesList = controllerUtils.getModesList(campaignName, modeSpecified);
+        for (Mode currentMode : modesList) {
+            try {
+                treatRawCampaignWithMode(campaignName, currentMode, null);
+                isAnyDataSaved = true;
+            }catch (NoDataException nde){
+                //Don't stop if NoDataError thrown
+                log.warn(nde.getMessage());
+            }catch (Exception e){
+                log.error(CAMPAIGN_ERROR, campaignName, e.toString());
+                return ResponseEntity.status(500).body(e.getMessage());
+            }
+        }
+
+        return ResponseEntity.ok(isAnyDataSaved ? SUCCESS_MESSAGE : "No data has been saved");
+    }
+
+    //JSON
+    @Operation(summary = "Save lunatic json data to Genesis Database from the campaign root folder")
+    @PutMapping(path = "/lunatic-json/raw/save")
+    public ResponseEntity<Object> saveRawResponsesFromJsonBody(
+            @RequestParam("campaignName") String campaignName,
+            @RequestParam(value = "mode", required = false) Mode modeSpecified,
+            @RequestBody String dataJson
+    ) {
+        log.info("Try to import raw lunatic JSON data for campaign : {}", campaignName);
+        try {
+            lunaticJsonRawDataApiPort.saveData(campaignName, dataJson, modeSpecified);
+        }catch (JsonProcessingException jpe){
+            log.error(jpe.toString());
+            return ResponseEntity.badRequest().body("Invalid JSON synthax");
+        }
+        log.info("Data saved for {}", campaignName);
+        return ResponseEntity.ok(SUCCESS_MESSAGE);
+    }
+
+
     //SAVE ALL
     @Operation(summary = "Save all files to Genesis Database (differential data folder only), regardless of the campaign")
     @PutMapping(path = "/lunatic-xml/save-all-campaigns")
@@ -167,12 +238,12 @@ public ResponseEntity<Object> saveResponsesFromAllCampaignFolders(){
                 log.warn(nde.getMessage());
             }
             catch (Exception e) {
-                log.warn("Error for campaign {} : {}", campaignName, e.toString());
+                log.warn(CAMPAIGN_ERROR, campaignName, e.toString());
                 errors.add(new GenesisError(e.getMessage()));
             }
         }
         if (errors.isEmpty()) {
-            return ResponseEntity.ok("Data saved");
+            return ResponseEntity.ok(SUCCESS_MESSAGE);
         } else {
             return ResponseEntity.status(209).body("Data saved with " + errors.size() + " errors");
         }
@@ -373,6 +444,43 @@ private void treatCampaignWithMode(String campaignName, Mode mode, List<GenesisE
         }
     }
 
+    private void treatRawCampaignWithMode(String campaignName,
+                                          Mode mode,
+                                          String rootDataFolder
+    ) throws IOException, ParserConfigurationException, SAXException, NoDataException {
+        log.info("Try to import data for mode : {}", mode.getModeName());
+        String dataFolder = rootDataFolder == null ?
+                fileUtils.getDataFolder(campaignName, mode.getFolder())
+                : fileUtils.getDataFolder(campaignName, mode.getFolder(), rootDataFolder);
+        List<String> dataFiles = fileUtils.listFiles(dataFolder);
+        log.info("Numbers of files to load in folder {} : {}", dataFolder, dataFiles.size());
+        if (dataFiles.isEmpty()) {
+            throw new NoDataException("No data file found in folder %s".formatted(dataFolder));
+        }
+
+        //For each XML data file
+        for (String fileName : dataFiles.stream().filter(s -> s.endsWith(".xml")).toList()) {
+            String filepathString = String.format(S_S, dataFolder, fileName);
+            Path filepath = Paths.get(filepathString);
+            //Check if file not in done folder, delete if true
+            if(isDataFileInDoneFolder(filepath, campaignName, mode.getFolder())){
+                log.warn("File {} already exists in DONE folder ! Deleting...", fileName);
+                Files.deleteIfExists(filepath);
+            }else{
+                //Read file
+                log.info("Try to read Xml file : {}", fileName);
+                ResponseEntity<Object> response = treatRawXmlFile(filepath, mode);
+
+                log.debug("File {} saved", fileName);
+                if(response.getStatusCode() == HttpStatus.OK){
+                    fileUtils.moveDataFile(campaignName, mode.getFolder(), filepath);
+                }else{
+                    log.error("Error on file {}", fileName);
+                }
+            }
+        }
+    }
+
     private VariablesMap readMetadatas(String campaignName, Mode mode, List<GenesisError> errors, boolean withDDI) {
         VariablesMap variablesMap;
         if(withDDI){
@@ -431,6 +539,26 @@ private ResponseEntity<Object> treatXmlFileWithMemory(Path filepath, Mode modeSp
         return ResponseEntity.ok().build();
     }
 
+    private ResponseEntity<Object> treatRawXmlFile(Path filepath, Mode modeSpecified) throws IOException,
+            ParserConfigurationException, SAXException {
+        LunaticXmlCampaign campaign;
+        // DOM method
+        LunaticXmlDataParser parser = new LunaticXmlDataParser();
+        try {
+            campaign = parser.parseDataFile(filepath);
+        } catch (GenesisException e) {
+            log.error(e.toString());
+            return ResponseEntity.status(e.getStatus()).body(e.getMessage());
+        }
+
+        log.debug("Begin saving raw xml data file {}", filepath);
+        lunaticXmlRawDataApiPort.saveData(campaign, modeSpecified);
+        log.debug(SUCCESS_MESSAGE);
+
+        log.info("File {} treated" , filepath.getFileName());
+        return ResponseEntity.ok().build();
+    }
+
     private ResponseEntity<Object> treatXmlFileSequentially(Path filepath, Mode modeSpecified, VariablesMap variablesMap) throws IOException, XMLStreamException {
         LunaticXmlCampaign campaign;
         //Sequential method

src/main/java/fr/insee/genesis/controller/sources/json/LunaticJsonDataResponse.java

@@ -9,7 +9,7 @@
 public class LunaticJsonDataResponse {
 
     @JsonProperty("EXTERNAL")
-    private LunaticJSonExternalVariables externalVariables;
+    private LunaticJsonExternalVariables externalVariables;
 
     @JsonProperty("COLLECTED")
     private LunaticJsonCollectedVariables collectedVariables;

src/main/java/fr/insee/genesis/controller/sources/json/LunaticJSonExternalVariables.java → src/main/java/fr/insee/genesis/controller/sources/json/LunaticJsonExternalVariables.java

@@ -7,7 +7,7 @@
 import java.util.Map;
 
 @Data
-public class LunaticJSonExternalVariables {
+public class LunaticJsonExternalVariables {
     private Map<String, String> externalMap = new HashMap<>();
 
     @JsonAnySetter

src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java

@@ -0,0 +1,19 @@
+package fr.insee.genesis.domain.model.surveyunit.rawdata;
+
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import lombok.Builder;
+import lombok.Data;
+import org.bson.types.ObjectId;
+
+import java.time.LocalDateTime;
+
+@Data
+@Builder
+public class LunaticJsonDataModel {
+    private ObjectId id;
+    private String campaignId;
+    private Mode mode;
+    private String dataJson;
+    private LocalDateTime recordDate;
+    private LocalDateTime processDate;
+}

src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticXmlDataModel.java

@@ -0,0 +1,18 @@
+package fr.insee.genesis.domain.model.surveyunit.rawdata;
+
+import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import lombok.Builder;
+import lombok.Data;
+import org.bson.types.ObjectId;
+
+import java.time.LocalDateTime;
+
+@Data
+@Builder
+public class LunaticXmlDataModel{
+    private ObjectId id;
+    private Mode mode;
+    private LunaticXmlCampaign data;
+    private LocalDateTime recordDate;
+}

src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java

@@ -0,0 +1,8 @@
+package fr.insee.genesis.domain.ports.api;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+
+public interface LunaticJsonRawDataApiPort {
+    void saveData(String campaignName, String dataJson, Mode mode) throws JsonProcessingException;
+}

src/main/java/fr/insee/genesis/domain/ports/api/LunaticXmlRawDataApiPort.java

@@ -0,0 +1,8 @@
+package fr.insee.genesis.domain.ports.api;
+
+import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+
+public interface LunaticXmlRawDataApiPort {
+    void saveData(LunaticXmlCampaign data, Mode mode);
+}

src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonPersistancePort.java

@@ -0,0 +1,7 @@
+package fr.insee.genesis.domain.ports.spi;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel;
+
+public interface LunaticJsonPersistancePort {
+    void save(LunaticJsonDataModel lunaticJsonDataModel);
+}

src/main/java/fr/insee/genesis/domain/ports/spi/LunaticXmlPersistancePort.java

@@ -0,0 +1,9 @@
+package fr.insee.genesis.domain.ports.spi;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel;
+
+import java.util.List;
+
+public interface LunaticXmlPersistancePort {
+    void save(LunaticXmlDataModel lunaticXmlDataModels);
+}

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java

@@ -0,0 +1,34 @@
+package fr.insee.genesis.domain.service.rawdata;
+
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonDataModel;
+import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
+import fr.insee.genesis.domain.ports.spi.LunaticJsonPersistancePort;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Service;
+
+import java.time.LocalDateTime;
+
+@Service
+public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort {
+    @Qualifier("lunaticJsonMongoAdapter")
+    private final LunaticJsonPersistancePort lunaticJsonPersistancePort;
+
+    @Autowired
+    public LunaticJsonRawDataService(LunaticJsonPersistancePort lunaticJsonPersistancePort) {
+        this.lunaticJsonPersistancePort = lunaticJsonPersistancePort;
+    }
+
+    @Override
+    public void saveData(String campaignName, String dataJson, Mode mode){
+        LunaticJsonDataModel lunaticJsonDataModel = LunaticJsonDataModel.builder()
+                .campaignId(campaignName)
+                .mode(mode)
+                .dataJson(dataJson)
+                .recordDate(LocalDateTime.now())
+                .build();
+
+        lunaticJsonPersistancePort.save(lunaticJsonDataModel);
+    }
+}

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticXmlRawDataService.java

@@ -0,0 +1,34 @@
+package fr.insee.genesis.domain.service.rawdata;
+
+import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticXmlDataModel;
+import fr.insee.genesis.domain.ports.api.LunaticXmlRawDataApiPort;
+import fr.insee.genesis.domain.ports.spi.LunaticXmlPersistancePort;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Service;
+
+import java.time.LocalDateTime;
+
+@Service
+public class LunaticXmlRawDataService implements LunaticXmlRawDataApiPort {
+    @Qualifier("lunaticXmlMongoAdapter")
+    private final LunaticXmlPersistancePort lunaticXmlPersistancePort;
+
+    @Autowired
+    public LunaticXmlRawDataService(LunaticXmlPersistancePort lunaticXmlPersistancePort) {
+        this.lunaticXmlPersistancePort = lunaticXmlPersistancePort;
+    }
+
+    @Override
+    public void saveData(LunaticXmlCampaign campaign, Mode mode) {
+        LunaticXmlDataModel lunaticXmlDataModel = LunaticXmlDataModel.builder()
+                .mode(mode)
+                .data(campaign)
+                .recordDate(LocalDateTime.now())
+                .build();
+
+        lunaticXmlPersistancePort.save(lunaticXmlDataModel);
+    }
+}