Coverage and v2 fro json schema Validation

pom.xml

@@ -27,7 +27,9 @@
         <sonar.language>java</sonar.language>
         <sonar.sourceEncoding>UTF-8</sonar.sourceEncoding>
         <sonar.exclusions>
-            src/main/java/fr/insee/genesis/configuration/**/*.java
+            **/*MapperImpl.class, <!-- Generated mappers by mapstruct -->
+            src/main/java/fr/insee/genesis/configuration/**/*.java,
+            src/main/java/fr/insee/genesis/exceptions/*.java
         </sonar.exclusions>
         <skipSurefireReport>true</skipSurefireReport>
         <!-- Pi Test-->
@@ -109,7 +111,7 @@
         <dependency>
             <groupId>com.networknt</groupId>
             <artifactId>json-schema-validator</artifactId>
-            <version>1.5.9</version>
+            <version>2.0.0</version>
         </dependency>
 
         <!-- generate implementation auto -->
@@ -180,6 +182,8 @@
                     <configuration>
                         <excludes>
                             <exclude>src/main/java/fr/insee/genesis/configuration/**/*</exclude>
+                            <exclude>**/*MapperImpl.class</exclude>
+                            <exclude>src/main/java/fr/insee/genesis/exceptions/*</exclude>
                         </excludes>
                     </configuration>
                     <executions>

src/main/java/fr/insee/genesis/controller/rest/UtilsController.java

@@ -4,6 +4,7 @@
 import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
 import fr.insee.genesis.domain.service.volumetry.VolumetryLogService;
 import fr.insee.genesis.domain.utils.XMLSplitter;
+import fr.insee.genesis.exceptions.GenesisException;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import lombok.extern.slf4j.Slf4j;
@@ -44,7 +45,7 @@ public ResponseEntity<Object> saveResponsesFromXmlFile(@RequestParam("inputFolde
 														   @RequestParam("outputFolder") String outputFolder,
 														   @RequestParam("filename") String filename,
 														   @RequestParam("nbResponsesByFile") int nbSU)
-			throws XMLStreamException, IOException {
+            throws XMLStreamException, IOException, GenesisException {
 		XMLSplitter.split(inputFolder, filename, outputFolder, "SurveyUnit", nbSU);
 		return ResponseEntity.ok("File split");
 	}

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -2,10 +2,10 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.networknt.schema.JsonSchema;
-import com.networknt.schema.JsonSchemaFactory;
-import com.networknt.schema.SpecVersion;
-import com.networknt.schema.ValidationMessage;
+import com.networknt.schema.Error;
+import com.networknt.schema.Schema;
+import com.networknt.schema.SchemaRegistry;
+import com.networknt.schema.dialect.Dialects;
 import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
@@ -38,7 +38,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -94,14 +93,15 @@ public ResponseEntity<String> saveRawResponsesFromJsonBody(
     public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidation(
             @RequestBody Map<String, Object> body
     ) {
-        JsonSchema jsonSchema = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7).getSchema(
-                RawResponseController.class.getResourceAsStream("/jsonSchemas/RawResponse.json")
+        SchemaRegistry schemaRegistry = SchemaRegistry.withDialect(Dialects.getDraft202012(), SchemaRegistry.Builder::build);
+        Schema jsonSchema = schemaRegistry
+                .getSchema(RawResponseController.class.getResourceAsStream("/jsonSchemas/RawResponse.json")
         );
         try {
             if (jsonSchema == null) {
                 throw new GenesisException(500, "No RawResponse json schema has been found");
             }
-            Set<ValidationMessage> errors = jsonSchema.validate(
+            List<Error> errors = jsonSchema.validate(
                     new ObjectMapper().readTree(
                             new ObjectMapper().writeValueAsString(body)
                     )
@@ -209,10 +209,10 @@ public ResponseEntity<PagedModel<LunaticJsonRawDataModel>> getRawResponsesFromJs
         return ResponseEntity.status(HttpStatus.OK).body(new PagedModel<>(rawResponses));
     }
 
-    private void validate(Set<ValidationMessage> errors) throws GenesisException {
+    private void validate(List<Error> errors) throws GenesisException {
         if (!errors.isEmpty()) {
             String errorMessage = errors.stream()
-                    .map(ValidationMessage::getMessage)
+                    .map(Error::getMessage)
                     .collect(Collectors.joining(System.lineSeparator() + " - "));
 
             throw new GenesisException(

src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java

@@ -156,7 +156,7 @@ public List<List<SurveyUnitModel>> findLatestByIdAndByQuestionnaireIdAndModeOrde
         List<List<SurveyUnitModel>> listLatestUpdatesbyVariables = new ArrayList<>();
 
         //1) QUERY
-        // => conversion of "List<InterrogationId>" -> "List<String>" for query using lamda
+        // => conversion of "List<InterrogationId>" -> "List<String>" for query using lambda
         List<String> queryInParam = interrogationIds.stream().map(InterrogationId::getInterrogationId).toList();
 
         //Get !!!all versions!!! of a set of "interrogationIds"

src/main/java/fr/insee/genesis/domain/utils/XMLSplitter.java

@@ -1,14 +1,8 @@
 package fr.insee.genesis.domain.utils;
 
+import fr.insee.genesis.exceptions.GenesisException;
 import lombok.experimental.UtilityClass;
 
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
 import javax.xml.XMLConstants;
 import javax.xml.stream.XMLEventFactory;
 import javax.xml.stream.XMLEventReader;
@@ -20,12 +14,17 @@
 import javax.xml.stream.events.StartDocument;
 import javax.xml.stream.events.StartElement;
 import javax.xml.stream.events.XMLEvent;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 @UtilityClass
 public class XMLSplitter {
 
 	// We use StAX in this class to deal with memory issues on huge XML files
-	public static void split(String inputfolder, String xmlfile, String outputFolder, String condition, int nbElementsByFile) throws XMLStreamException, IOException  {
+	public static void split(String inputfolder, String xmlfile, String outputFolder, String condition, int nbElementsByFile) throws XMLStreamException, IOException, GenesisException {
 
 		String xmlResource = inputfolder + xmlfile;
 		List<XMLEvent> header = getHeader(xmlResource, condition);
@@ -34,42 +33,45 @@ public static void split(String inputfolder, String xmlfile, String outputFolder
 		XMLInputFactory xif = XMLInputFactory.newInstance();
 		xif.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
 		xif.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
-		XMLEventReader xer = xif.createXMLEventReader(new FileReader(xmlResource));
-		StartElement rootStartElement = xer.nextTag().asStartElement();
-		StartDocument startDocument = xef.createStartDocument();
-		EndDocument endDocument = xef.createEndDocument();
-
-		XMLOutputFactory xof = XMLOutputFactory.newFactory();
-		int fileCount = 1;
-		while(xer.hasNext() && !xer.peek().isEndDocument()) {
-			XMLEvent xmlEvent = xer.nextEvent();
-
-			if (isStartElementWithName(condition, xmlEvent)) {
-				// Create a file for the fragment, the name is derived from the value of the id attribute
-				FileWriter fileWriter = new FileWriter(outputFolder + "split" + fileCount + ".xml");
-
-				// A StAX XMLEventWriter will be used to write the XML fragment
-				XMLEventWriter xew = xof.createXMLEventWriter(fileWriter);
-				xew.add(startDocument);
-
-				// Add the elements which are common to all split files
-				addHeadersToNewFile(header, xew);
-
-				// Write the XMLEvents that are part of SurveyUnit element
-				xew.add(xmlEvent);
-				xmlEvent = xer.nextEvent();
-				int nbResponses = 1;
-				// We loop until we reach the end tag Survey units indicating the near end of the document
-				iterateOnSurveyUnits(condition, nbElementsByFile, xer, xmlEvent, xew, nbResponses);
-
-				// Write the file, close everything we opened and update the file's counter
-				xew.add(xef.createEndElement(rootStartElement.getName(), null));
-				xew.add(endDocument);
-				fileWriter.close();
-
-				fileCount++;
-
+        try (FileReader fr = new FileReader(xmlResource)){
+            XMLEventReader xer = xif.createXMLEventReader(fr);
+
+		    StartElement rootStartElement = xer.nextTag().asStartElement();
+		    StartDocument startDocument = xef.createStartDocument();
+		    EndDocument endDocument = xef.createEndDocument();
+
+		    XMLOutputFactory xof = XMLOutputFactory.newFactory();
+		    int fileCount = 1;
+		    while(xer.hasNext() && !xer.peek().isEndDocument()) {
+		    	XMLEvent xmlEvent = xer.nextEvent();
+
+		    	if (isStartElementWithName(condition, xmlEvent)) {
+                    // Create a file for the fragment, the name is derived from the value of the id attribute
+                    FileWriter fileWriter = new FileWriter(outputFolder + "split" + fileCount + ".xml");
+
+                    // A StAX XMLEventWriter will be used to write the XML fragment
+                    XMLEventWriter xew = xof.createXMLEventWriter(fileWriter);
+                    xew.add(startDocument);
+
+                    // Add the elements which are common to all split files
+                    addHeadersToNewFile(header, xew);
+
+                    // Write the XMLEvents that are part of SurveyUnit element
+                    xew.add(xmlEvent);
+                    xmlEvent = xer.nextEvent();
+                    int nbResponses = 1;
+                    // We loop until we reach the end tag Survey units indicating the near end of the document
+                    iterateOnSurveyUnits(condition, nbElementsByFile, xer, xmlEvent, xew, nbResponses);
+
+                    // Write the file, close everything we opened and update the file's counter
+                    xew.add(xef.createEndElement(rootStartElement.getName(), null));
+                    xew.add(endDocument);
+                    fileWriter.close();
+
+                    fileCount++;
+                }
 			}
+            xer.close();
 		}
 	}
 
@@ -103,33 +105,37 @@ private static boolean isEndElementWithName(XMLEvent xmlEvent, String condition)
 		return xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().getLocalPart().equals(condition);
 	}
 
-	private static List<XMLEvent> getHeader(String xmlResource, String condition) throws FileNotFoundException, XMLStreamException {
+	private static List<XMLEvent> getHeader(String xmlResource, String condition) throws XMLStreamException, GenesisException {
 		XMLInputFactory xif = XMLInputFactory.newInstance();
 		xif.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
 		xif.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
-		XMLEventReader xer = xif.createXMLEventReader(new FileReader(xmlResource));
-
-		List<XMLEvent> cachedXMLEvents = new ArrayList<>();
-		while(xer.hasNext() && !xer.peek().isEndDocument()) {
-			XMLEvent xmlEvent = xer.nextTag();
-			if (!xmlEvent.isStartElement()) {
-				break;
-			}
-			StartElement breakStartElement = xmlEvent.asStartElement();
-
-			cachedXMLEvents.add(breakStartElement);
-			xmlEvent = xer.nextEvent();
-			while (!(xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().equals(breakStartElement.getName()))) {
-				if (isStartElementWithName(condition, xmlEvent)) {
-					xer.close();
-					return cachedXMLEvents;
-				}
-				cachedXMLEvents.add(xmlEvent);
-				xmlEvent = xer.nextEvent();
-			}
-		}
-		xer.close();
-		return List.of();
+        try (FileReader fr = new FileReader(xmlResource)) {
+            XMLEventReader xer = xif.createXMLEventReader(fr);
+
+            List<XMLEvent> cachedXMLEvents = new ArrayList<>();
+            while (xer.hasNext() && !xer.peek().isEndDocument()) {
+                XMLEvent xmlEvent = xer.nextTag();
+                if (!xmlEvent.isStartElement()) {
+                    break;
+                }
+                StartElement breakStartElement = xmlEvent.asStartElement();
+
+                cachedXMLEvents.add(breakStartElement);
+                xmlEvent = xer.nextEvent();
+                while (!(xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().equals(breakStartElement.getName()))) {
+                    if (isStartElementWithName(condition, xmlEvent)) {
+                        xer.close();
+                        return cachedXMLEvents;
+                    }
+                    cachedXMLEvents.add(xmlEvent);
+                    xmlEvent = xer.nextEvent();
+                }
+            }
+            xer.close();
+        } catch (IOException e) {
+            throw new GenesisException(500,e.getMessage());
+        }
+        return List.of();
 	}
 
 }

src/test/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDtoTest.java

@@ -0,0 +1,49 @@
+package fr.insee.genesis.controller.mappers;
+
+import fr.insee.genesis.controller.dto.ScheduleDto;
+import fr.insee.genesis.domain.model.context.DataProcessingContextModel;
+import org.bson.types.ObjectId;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.time.LocalDateTime.now;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+class DataProcessingContextMapperDtoTest {
+	@Test
+	void dataProcessingContextToScheduleDto() {
+		DataProcessingContextMapperDto d = new DataProcessingContextMapperDto();
+		DataProcessingContextModel dataProcessingContext = new DataProcessingContextModel();
+
+		ScheduleDto actual = d.dataProcessingContextToScheduleDto(dataProcessingContext);
+        assertNotNull(actual);
+		assertNull( actual.lastExecution());
+        assertNull( actual.surveyName());
+        assertNull( actual.kraftwerkExecutionScheduleList());
+
+        DataProcessingContextMapperDto d2 = new DataProcessingContextMapperDto();
+        DataProcessingContextModel dataProcessingContext2 = new DataProcessingContextModel();
+        dataProcessingContext2.setId(new ObjectId());
+        dataProcessingContext2.setWithReview(true);
+        dataProcessingContext2.setLastExecution(now());
+
+        ScheduleDto actual2 = d2.dataProcessingContextToScheduleDto(dataProcessingContext2);
+        assertNotNull(actual2);
+        assertNotNull( actual2.lastExecution());
+        assertNull( actual2.surveyName());
+	}
+
+	@Test
+	void dataProcessingContextListToScheduleDtoList() {
+		DataProcessingContextMapperDto d = new DataProcessingContextMapperDto();
+		List<DataProcessingContextModel> contexts = new ArrayList<>();
+		List<ScheduleDto> expected = new ArrayList<>();
+		List<ScheduleDto> actual = d.dataProcessingContextListToScheduleDtoList(contexts);
+
+		assertEquals(expected, actual);
+	}
+}

src/test/java/fr/insee/genesis/controller/sources/json/LunaticJsonCollectedVariablesTest.java

@@ -0,0 +1,13 @@
+package fr.insee.genesis.controller.sources.json;
+
+import org.junit.jupiter.api.Test;
+
+class LunaticJsonCollectedVariablesTest {
+	@Test
+	void setVariables() {
+		LunaticJsonCollectedVariables l = new LunaticJsonCollectedVariables();
+		String variable = "abc";
+		LunaticJsonVariableData value = new LunaticJsonVariableData();
+		l.setVariables(variable, value);
+	}
+}

src/test/java/fr/insee/genesis/controller/sources/json/LunaticJsonExternalVariablesTest.java

@@ -0,0 +1,13 @@
+package fr.insee.genesis.controller.sources.json;
+
+import org.junit.jupiter.api.Test;
+
+class LunaticJsonExternalVariablesTest {
+	@Test
+	void setVariables() {
+		LunaticJsonExternalVariables l = new LunaticJsonExternalVariables();
+		String variable = "abc";
+		String value = "abc";
+		l.setVariables(variable, value);
+	}
+}