Init raw response controller

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -0,0 +1,53 @@
+package fr.insee.genesis.controller.rest.responses;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+@RequestMapping(path = "/responses/raw" )
+@Controller
+@Tag(name = "Response services")
+@Slf4j
+public class RawResponseController {
+
+
+    private static final String SUCCESS_MESSAGE = "Data saved";
+    private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
+
+
+    public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort) {
+        this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
+    }
+
+    //SAVE
+    @Operation(summary = "Save lunatic json data to Genesis Database from the campaign name")
+    @PutMapping(path = "lunatic-json/save")
+    public ResponseEntity<Object> saveRawResponsesFromJsonBody(
+            @RequestParam("campaignName") String campaignName,
+            @RequestParam("interrogationId") String interrogationId,
+            @RequestParam(value = "surveyUnitId", required = false) String idUE,
+            @RequestParam("questionnaireId") String questionnaireId,
+            @RequestParam(value = "mode", required = false) Mode modeSpecified,
+            @RequestBody String dataJson
+    ) {
+        log.info("Try to import raw lunatic JSON data for campaign: {}", campaignName);
+        try {
+            lunaticJsonRawDataApiPort.saveData(campaignName, interrogationId, idUE, questionnaireId, modeSpecified, dataJson);
+        }catch (JsonProcessingException jpe ){
+            log.error(jpe.toString());
+            return ResponseEntity.badRequest().body("Invalid JSON syntax");
+        }
+        log.info("Data saved for {}", campaignName);
+        return ResponseEntity.ok(SUCCESS_MESSAGE);
+    }
+
+}

src/main/java/fr/insee/genesis/controller/services/MetadataService.java

@@ -0,0 +1,83 @@
+package fr.insee.genesis.controller.services;
+
+import fr.insee.bpm.exceptions.MetadataParserException;
+import fr.insee.bpm.metadata.model.VariablesMap;
+import fr.insee.bpm.metadata.reader.ddi.DDIReader;
+import fr.insee.bpm.metadata.reader.lunatic.LunaticReader;
+import fr.insee.genesis.exceptions.GenesisError;
+import fr.insee.genesis.infrastructure.utils.FileUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+@Slf4j
+@Service
+public class MetadataService {
+
+    private static final String DDI_FILE_PATTERN = "ddi[\\w,\\s-]+\\.xml";
+    private static final String LUNATIC_FILE_PATTERN = "lunatic[\\w,\\s-]+\\.json";
+
+    /**
+     * Parse metadata file, either DDI or Lunatic, depending on the withDDI flag.
+     *
+     * @param metadataFilePath path to the metadata file
+     * @param withDDI          true for DDI parsing, false for Lunatic parsing
+     * @return VariablesMap or null if an error occurs
+     */
+    public VariablesMap parseMetadata(String metadataFilePath, boolean withDDI) {
+        try {
+            log.info("Try to read {} file: {}", withDDI ? "DDI" : "Lunatic", metadataFilePath);
+            if (withDDI) {
+                return DDIReader.getMetadataFromDDI(
+                        Path.of(metadataFilePath).toFile().toURI().toURL().toString(),
+                        new FileInputStream(metadataFilePath)).getVariables();
+            } else {
+                return LunaticReader.getMetadataFromLunatic(
+                        new FileInputStream(metadataFilePath)).getVariables();
+            }
+        } catch (MetadataParserException | IOException e) {
+            log.error("Error reading metadata file", e);
+            return null;
+        }
+    }
+
+    /**
+     * By folder defined by campaignName
+     * Attempt to parse DDI metadata first; if it fails, fall back to Lunatic parsing.
+     *
+     * @param campaignName name of the campaign
+     * @param modeName     mode associated with the data
+     * @param fileUtils    utility for file operations
+     * @param errors       list to populate with errors
+     * @return VariablesMap or null if parsing fails
+     */
+    public VariablesMap readMetadatas(String campaignName, String modeName, FileUtils fileUtils, List<GenesisError> errors) {
+
+        Path ddiFilePath = null;
+        VariablesMap variablesMap = null;
+        try {
+            ddiFilePath = fileUtils.findFile(String.format("%s/%s", fileUtils.getSpecFolder(campaignName), modeName), DDI_FILE_PATTERN);
+            variablesMap = parseMetadata(ddiFilePath.toString(), true);
+
+        } catch (RuntimeException e) {
+            //DDI file not found and already log - Go to next step
+        } catch (IOException e) {
+            log.warn("No DDI File found for {}, {} mode. Will try to use Lunatic...", campaignName, modeName);
+        }
+        if(variablesMap == null ){
+            log.warn("DDI not found or error occurred. Trying Lunatic metadata...for {}, {} mode", campaignName, modeName);
+            try {
+                Path lunaticFilePath = fileUtils.findFile(String.format("%s/%s", fileUtils.getSpecFolder(campaignName), modeName), LUNATIC_FILE_PATTERN);
+                return parseMetadata(lunaticFilePath.toString(), false);
+            } catch (Exception ex) {
+                log.error("Error reading Lunatic metadata file", ex);
+                errors.add(new GenesisError(ex.toString()));
+                return null;
+            }
+        }
+        return variablesMap;
+    }
+}

src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/LunaticJsonDataModel.java

@@ -11,6 +11,9 @@
 public record LunaticJsonDataModel(
     ObjectId id,
     String campaignId,
+    String interrogationId,
+    String idUE,
+    String questionnaireId,
     Mode mode,
     String dataJson,
     LocalDateTime recordDate,

src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java

@@ -4,5 +4,5 @@
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 
 public interface LunaticJsonRawDataApiPort {
-    void saveData(String campaignName, String dataJson, Mode mode) throws JsonProcessingException;
+    void saveData(String campaignName, String interrogationId, String idUE, String questionnaireId, Mode mode, String jsonData) throws JsonProcessingException;
 }

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java

@@ -25,12 +25,15 @@ public LunaticJsonRawDataService(LunaticJsonPersistancePort lunaticJsonPersistan
     }
 
     @Override
-    public void saveData(String campaignName, String dataJson, Mode mode) throws JsonParseException {
+    public void saveData(String campaignName, String interrogationId, String idUE, String questionnaireId, Mode mode, String dataJson) throws JsonParseException {
         if(!isJsonValid(dataJson)){
-            throw new JsonParseException("Invalid JSON synthax");
+            throw new JsonParseException("Invalid JSON syntax");
         }
         LunaticJsonDataModel lunaticJsonDataModel = LunaticJsonDataModel.builder()
                 .campaignId(campaignName)
+                .questionnaireId(questionnaireId)
+                .interrogationId(interrogationId)
+                .idUE(idUE)
                 .mode(mode)
                 .dataJson(dataJson)
                 .recordDate(LocalDateTime.now())

src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java

@@ -57,7 +57,6 @@ public List<SurveyUnitModel> findByInterrogationId(String questionnaireId) {
 	@Override
 	public List<SurveyUnitModel> findByInterrogationIdsAndQuestionnaireId(List<SurveyUnitModel> questionnaireIds, String questionnaireId) {
 		List<SurveyUnitDocument> surveyUnits= new ArrayList<>();
-		// TODO: 18-10-2023 : find a way to do this in one query
 		questionnaireIds.forEach(su -> {
 			List<SurveyUnitDocument> docs = mongoRepository.findByInterrogationIdAndQuestionnaireId(su.getInterrogationId(), questionnaireId);
 			surveyUnits.addAll(docs);

src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonDataDocument.java

@@ -19,6 +19,9 @@ public class LunaticJsonDataDocument {
     @Id
     private ObjectId id;
     private String campaignId;
+    private String interrogationId;
+    private String idUE;
+    private String questionnaireId;
     private Mode mode;
     private Map<String, Object> data;
     private LocalDateTime recordDate;

src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java

@@ -3,7 +3,6 @@
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
 import fr.insee.genesis.infrastructure.document.surveyunit.SurveyUnitDocument;
 import org.mapstruct.Mapper;
-import org.mapstruct.Mapping;
 import org.mapstruct.factory.Mappers;
 
 import java.util.List;

src/main/resources/application.properties

@@ -7,6 +7,7 @@ spring.profiles.active=local
 # Global configuration
 #--------------------------------------------------------------------------
 fr.insee.genesis.authentication = OIDC
+logging.level.org.springframework.security=DEBUG
 
 #--------------------------------------------------------------------------
 # Configuration for springdoc / swagger

src/test/java/cucumber/functional_tests/MainDefinitions.java

@@ -10,6 +10,7 @@
 import fr.insee.genesis.controller.dto.VariableQualityToolDto;
 import fr.insee.genesis.controller.dto.VariableStateDto;
 import fr.insee.genesis.controller.rest.responses.ResponseController;
+import fr.insee.genesis.controller.services.MetadataService;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlDataParser;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit;
@@ -19,15 +20,11 @@
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
 import fr.insee.genesis.domain.model.surveyunit.VariableModel;
-import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService;
-import fr.insee.genesis.domain.service.rawdata.LunaticXmlRawDataService;
 import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
 import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService;
 import fr.insee.genesis.exceptions.GenesisException;
 import fr.insee.genesis.infrastructure.utils.FileUtils;
 import fr.insee.genesis.stubs.ConfigStub;
-import fr.insee.genesis.stubs.LunaticJsonPersistanceStub;
-import fr.insee.genesis.stubs.LunaticXmlPersistanceStub;
 import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
 import io.cucumber.java.After;
 import io.cucumber.java.Before;
@@ -64,11 +61,10 @@ public class MainDefinitions {
     ResponseController responseController = new ResponseController(
             new SurveyUnitService(surveyUnitPersistence),
             surveyUnitQualityService,
-            new LunaticXmlRawDataService(new LunaticXmlPersistanceStub()),
-            new LunaticJsonRawDataService(new LunaticJsonPersistanceStub()),
             new FileUtils(config),
             new ControllerUtils(new FileUtils(config)),
-            new AuthUtils(config)
+            new AuthUtils(config),
+            new MetadataService()
         );
 
     List<SurveyUnitModel> surveyUnitModels;
@@ -123,7 +119,7 @@ public void get_models(String fileName, String ddiName) throws IOException, Pars
 
     @When("We save data from that directory")
     public void get_su_models_from_folder() throws Exception {
-        responseController.saveResponsesFromXmlCampaignFolder(this.inDirectory.getFileName().toString(), null, true);
+        responseController.saveResponsesFromXmlCampaignFolder(this.inDirectory.getFileName().toString(), null);
     }
 
     @When("We delete that directory")

src/test/java/RunCucumberTest.java → src/test/java/fr/insee/genesis/RunCucumberTest.java

@@ -1,3 +1,5 @@
+package fr.insee.genesis;
+
 import io.cucumber.junit.Cucumber;
 import io.cucumber.junit.CucumberOptions;
 import org.junit.runner.RunWith;