feat: new endpoints to save raw response using the modele filiere library

loichenninger · loichenninger · commit 21dccf4b2b20 · 2025-11-12T10:46:00.000+01:00
diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties
@@ -0,0 +1,2 @@
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar
diff --git a/pom.xml b/pom.xml
@@ -98,7 +98,16 @@
             <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
             <version>${springdoc.version}</version>
         </dependency>
-
+        <dependency>
+            <groupId>fr.insee</groupId>
+            <artifactId>modelefiliere</artifactId>
+            <version>add-processed-response-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>com.networknt</groupId>
+            <artifactId>json-schema-validator</artifactId>
+            <version>1.5.9</version>
+        </dependency>
         <!-- XML libraries -->
         <!-- XML-XSLT with Saxon -->
         <dependency>
@@ -107,13 +116,6 @@
             <version>12.9</version>
         </dependency>
 
-        <!-- JSON -->
-        <dependency>
-            <groupId>com.networknt</groupId>
-            <artifactId>json-schema-validator</artifactId>
-            <version>2.0.0</version>
-        </dependency>
-
         <!-- generate implementation auto -->
         <dependency>
             <groupId>org.mapstruct</groupId>
diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java
@@ -1,18 +1,22 @@
 package fr.insee.genesis.controller.rest.responses;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.networknt.schema.Error;
-import com.networknt.schema.Schema;
-import com.networknt.schema.SchemaRegistry;
-import com.networknt.schema.dialect.Dialects;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
+import fr.insee.genesis.controller.utils.ExtendedJsonNormalizer;
+import fr.insee.genesis.controller.utils.JsonSchemaValidator;
+import fr.insee.genesis.controller.utils.SchemaType;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
 import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
 import fr.insee.genesis.exceptions.GenesisError;
 import fr.insee.genesis.exceptions.GenesisException;
+import fr.insee.genesis.exceptions.SchemaValidationException;
+import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository;
+import fr.insee.modelefiliere.RawResponseDto;
 import io.swagger.v3.oas.annotations.Hidden;
 import io.swagger.v3.oas.annotations.Operation;
 import lombok.extern.slf4j.Slf4j;
@@ -30,9 +34,9 @@
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
+import java.io.IOException;
 import java.time.Instant;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
@@ -42,20 +46,22 @@
 
 @Slf4j
 @Controller
-@RequestMapping(path = "/responses/raw")
 public class RawResponseController {
 
     private static final String SUCCESS_MESSAGE = "Interrogation %s saved";
     private static final String PARTITION_ID = "partitionId";
     private static final String INTERROGATION_ID = "interrogationId";
     private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
+    private final RawResponseInputRepository rawRepository;
 
-    public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort) {
+
+    public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, RawResponseInputRepository rawRepository) {
         this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
+        this.rawRepository = rawRepository;
     }
 
     @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database")
-    @PutMapping(path = "/lunatic-json/save")
+    @PutMapping(path = "/responses/raw/lunatic-json/save")
     @PreAuthorize("hasRole('COLLECT_PLATFORM')")
     public ResponseEntity<String> saveRawResponsesFromJsonBody(
 
@@ -86,16 +92,18 @@ public ResponseEntity<String> saveRawResponsesFromJsonBody(
         return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, interrogationId));
     }
 
-    @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " +
-            "schema validation)")
-    @PutMapping(path = "/lunatic-json")
+/*    @Operation(summary = "Deprecated")
+    @PutMapping(path="/lunatic-json")
     @PreAuthorize("hasRole('COLLECT_PLATFORM')")
-    public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidation(
+    // Check version when merging
+    @Deprecated(since="1.13.0", forRemoval=true)
+    public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidationDeprecated(
             @RequestBody Map<String, Object> body
     ) {
+
         SchemaRegistry schemaRegistry = SchemaRegistry.withDialect(Dialects.getDraft202012(), SchemaRegistry.Builder::build);
         Schema jsonSchema = schemaRegistry
-                .getSchema(RawResponseController.class.getResourceAsStream("/jsonSchemas/RawResponse.json")
+                .getSchema(RawResponseController.class.getResourceAsStream("/modele-filiere-spec/RawResponse.json")
         );
         try {
             if (jsonSchema == null) {
@@ -134,19 +142,48 @@ public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidation(
         log.info("Data saved for interrogationId {} and partition {}", body.get(INTERROGATION_ID).toString(),
                 body.get(PARTITION_ID).toString());
         return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, body.get(INTERROGATION_ID).toString()));
+    }*/
+
+    @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " +
+            "schema validation)")
+    @PutMapping(path="/raw-responses")
+    @PreAuthorize("hasRole('COLLECT_PLATFORM')")
+    public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidation(
+            @RequestBody Map<String, Object> body
+    ) {
+        ObjectMapper objectMapperLocal = new ObjectMapper();
+        objectMapperLocal.registerModule(new JavaTimeModule());
+        objectMapperLocal.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);   // ISO-8601
+        objectMapperLocal.disable(DeserializationFeature.ADJUST_DATES_TO_CONTEXT_TIME_ZONE);
+
+        try {
+            RawResponseDto rawResponseDto = JsonSchemaValidator.readAndValidateFromClasspath(
+                    ExtendedJsonNormalizer.normalize(new ObjectMapper().readTree(
+                            new ObjectMapper().writeValueAsString(body))),
+                    SchemaType.RAW_RESPONSE.getSchemaFileName(),
+                    RawResponseDto.class,
+                    objectMapperLocal
+            );
+            rawRepository.saveAsRawJson(rawResponseDto);
+        } catch (SchemaValidationException | IOException e) {
+            return ResponseEntity.status(400).body(e.toString());
+        }
+        return ResponseEntity.ok("Change this when ready");
+    }
+
     }
 
     //GET unprocessed
     @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data")
-    @GetMapping(path = "/lunatic-json/get/unprocessed")
+    @GetMapping(path = "/responses/raw/lunatic-json/get/unprocessed")
     @PreAuthorize("hasRole('SCHEDULER')")
     public ResponseEntity<List<LunaticJsonRawDataUnprocessedDto>> getUnproccessedJsonRawData() {
         log.info("Try to get unprocessed raw JSON datas...");
         return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataIds());
     }
 
     @Hidden
-    @GetMapping(path = "/lunatic-json/get/by-interrogation-mode-and-campaign")
+    @GetMapping(path = "/responses/raw/lunatic-json/get/by-interrogation-mode-and-campaign")
     @PreAuthorize("hasRole('ADMIN')")
     public ResponseEntity<LunaticJsonRawDataModel> getJsonRawData(
             @RequestParam(INTERROGATION_ID) String interrogationId,
@@ -159,7 +196,7 @@ public ResponseEntity<LunaticJsonRawDataModel> getJsonRawData(
 
     //PROCESS
     @Operation(summary = "Process raw data of a campaign")
-    @PostMapping(path = "/lunatic-json/process")
+    @PostMapping(path = "/responses/raw/lunatic-json/process")
     @PreAuthorize("hasRole('SCHEDULER')")
     public ResponseEntity<String> processJsonRawData(
             @RequestParam("campaignName") String campaignName,
@@ -181,7 +218,7 @@ public ResponseEntity<String> processJsonRawData(
     }
 
     @Operation(summary = "Get processed data ids from last n hours (default 24h)")
-    @GetMapping(path = "/lunatic-json/processed/ids")
+    @GetMapping(path = "/responses/raw/lunatic-json/processed/ids")
     @PreAuthorize("hasRole('ADMIN')")
     public ResponseEntity<Map<String, List<String>>> getProcessedDataIdsSinceHours(
             @RequestParam("questionnaireId") String questionnaireId,
@@ -193,7 +230,7 @@ public ResponseEntity<Map<String, List<String>>> getProcessedDataIdsSinceHours(
     }
 
     @Operation(summary = "Get lunatic JSON data from one campaign in Genesis Database, filtered by start and end dates")
-    @GetMapping(path = "/lunatic-json/{campaignId}")
+    @GetMapping(path = "/responses/raw/lunatic-json/{campaignId}")
     @PreAuthorize("hasRole('USER_BATCH_GENERIC')")
     public ResponseEntity<PagedModel<LunaticJsonRawDataModel>> getRawResponsesFromJsonBody(
             @PathVariable String campaignId,
diff --git a/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java b/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java
@@ -0,0 +1,53 @@
+package fr.insee.genesis.controller.utils;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.*;
+
+import java.util.Iterator;
+import java.util.Map;
+public class ExtendedJsonNormalizer {
+
+    private ExtendedJsonNormalizer() {}
+
+    /**
+     * Recursively converts Mongo Extended JSON objects into simple types expected by the schema:
+     *  - {"$date": "..."}  -> TextNode("...")
+     *
+     * Leaves all other values untouched. Returns a structural copy of the node (does not mutate the original).
+     */
+
+    public static JsonNode normalize(JsonNode node) {
+        if (node == null || node.isNull() || node.isMissingNode()) return node;
+
+        if (node.isObject()) {
+            ObjectNode obj = (ObjectNode) node;
+
+            if (obj.size() == 1) {
+                if (obj.has("$date") && obj.get("$date").isTextual()) {
+                    return TextNode.valueOf(obj.get("$date").asText());
+                }
+//                if (obj.has("$oid") && obj.get("$oid").isTextual()) {
+//                    return TextNode.valueOf(obj.get("$oid").asText());
+//                }
+            }
+
+            ObjectNode copy = obj.objectNode();
+            Iterator<Map.Entry<String, JsonNode>> it = obj.fields();
+            while (it.hasNext()) {
+                Map.Entry<String, JsonNode> e = it.next();
+                copy.set(e.getKey(), normalize(e.getValue()));
+            }
+            return copy;
+        }
+
+        if (node.isArray()) {
+            ArrayNode src = (ArrayNode) node;
+            ArrayNode dst = src.arrayNode();
+            for (JsonNode child : src) {
+                dst.add(normalize(child));
+            }
+            return dst;
+        }
+        return node;
+    }
+}
diff --git a/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java b/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java
@@ -0,0 +1,64 @@
+package fr.insee.genesis.controller.utils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.networknt.schema.JsonSchema;
+import com.networknt.schema.JsonSchemaFactory;
+import com.networknt.schema.SpecVersion;
+import com.networknt.schema.ValidationMessage;
+import fr.insee.genesis.exceptions.SchemaValidationException;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+@Slf4j
+public class JsonSchemaValidator {
+
+    private JsonSchemaValidator() {}
+
+    public static <T> T readAndValidate(JsonNode root,
+                                        JsonSchema schema,
+                                        Class<T> targetType,
+                                        ObjectMapper mapper)
+            throws SchemaValidationException, JsonProcessingException {
+        ensureValid(root, schema);
+        return mapper.treeToValue(root, targetType);
+    }
+
+    public static <T> T readAndValidateFromClasspath(JsonNode root,
+                                                     String schemaResourcePath,
+                                                     Class<T> targetType,
+                                                     ObjectMapper mapper)
+            throws SchemaValidationException, IOException {
+        JsonSchema schema = loadSchemaFromClasspath(schemaResourcePath, mapper);
+        return readAndValidate(root, schema, targetType, mapper);
+    }
+
+    public static JsonSchema loadSchemaFromClasspath(String resourcePath, ObjectMapper mapper) throws IOException {
+        if (resourcePath == null || resourcePath.isBlank()) {
+            throw new IOException("Schema resource path is null or blank");
+        }
+        String cp = resourcePath.startsWith("/") ? resourcePath.substring(1) : resourcePath;
+        try (InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(cp)) {
+            if (in == null) throw new IOException("Schema not found on classpath: " + resourcePath);
+            JsonNode schemaNode = mapper.readTree(in);
+            return JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012).getSchema(schemaNode);
+        }
+    }
+
+    private static void ensureValid(JsonNode root, JsonSchema schema) throws SchemaValidationException {
+        log.info("Schéma Apply : " + schema.getSchemaNode().get("title"));
+        java.util.Set<ValidationMessage> errors = schema.validate(root);
+        if (!errors.isEmpty()) {
+            String formatted = errors.stream()
+                    .sorted(java.util.Comparator.comparing(ValidationMessage::getEvaluationPath))
+                    .map(err -> err.getMessage())
+                    .collect(java.util.stream.Collectors.joining("\n"));
+            throw new SchemaValidationException(
+                    "Uploaded JSON is not correct according to the json-schema:\n" + formatted, errors);
+        }
+        log.info("Schema-compliant JSON : " + schema.getSchemaNode().get("title"));
+    }
+}
diff --git a/src/main/java/fr/insee/genesis/controller/utils/SchemaType.java b/src/main/java/fr/insee/genesis/controller/utils/SchemaType.java
@@ -0,0 +1,26 @@
+package fr.insee.genesis.controller.utils;
+
+import lombok.Getter;
+
+@Getter
+public enum SchemaType {
+
+ //   PROCESS_MESSAGE(Names.PROCESS_MESSAGE),
+    INTERROGATION(Names.INTERROGATION),
+    RAW_RESPONSE(Names.RAW_RESPONSE);
+
+    public static class Names {
+//        public static final String PROCESS_MESSAGE = "/modele-filiere-spec/Command.json";
+        public static final String INTERROGATION = "/modele-filiere-spec/Interrogation.json";
+        public static final String RAW_RESPONSE = "/modele-filiere-spec/RawResponse.json";
+
+        private Names() {
+
+        }
+    }
+    private final String schemaFileName;
+
+    SchemaType(String schemaFileName) {
+        this.schemaFileName = schemaFileName;
+    }
+}
diff --git a/src/main/java/fr/insee/genesis/exceptions/SchemaValidationException.java b/src/main/java/fr/insee/genesis/exceptions/SchemaValidationException.java
@@ -0,0 +1,16 @@
+package fr.insee.genesis.exceptions;
+
+import com.networknt.schema.ValidationMessage;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+import java.util.Set;
+
+@Getter
+@AllArgsConstructor
+public class SchemaValidationException extends Exception{
+
+    private final String message;
+
+    private Set<ValidationMessage> errors;
+}
diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java
diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java
diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip`
	`2`	`+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar`