Skip to content

Commit 3328dfc

Browse files
alicelaalicela
authored
Merge pull request #324 from InseeFr/devCoverage
Coverage and v2 fro json schema Validation
2 parents d2569cd + dcd0451 commit 3328dfc

File tree

12 files changed

+717
-81
lines changed

12 files changed

+717
-81
lines changed

‎pom.xml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@
2727
<sonar.language>java</sonar.language>
2828
<sonar.sourceEncoding>UTF-8</sonar.sourceEncoding>
2929
<sonar.exclusions>
30-
src/main/java/fr/insee/genesis/configuration/**/*.java
30+
**/*MapperImpl.class, <!-- Generated mappers by mapstruct -->
31+
src/main/java/fr/insee/genesis/configuration/**/*.java,
32+
src/main/java/fr/insee/genesis/exceptions/*.java
3133
</sonar.exclusions>
3234
<skipSurefireReport>true</skipSurefireReport>
3335
<!-- Pi Test-->
@@ -109,7 +111,7 @@
109111
<dependency>
110112
<groupId>com.networknt</groupId>
111113
<artifactId>json-schema-validator</artifactId>
112-
<version>1.5.9</version>
114+
<version>2.0.0</version>
113115
</dependency>
114116

115117
<!-- generate implementation auto -->
@@ -180,6 +182,8 @@
180182
<configuration>
181183
<excludes>
182184
<exclude>src/main/java/fr/insee/genesis/configuration/**/*</exclude>
185+
<exclude>**/*MapperImpl.class</exclude>
186+
<exclude>src/main/java/fr/insee/genesis/exceptions/*</exclude>
183187
</excludes>
184188
</configuration>
185189
<executions>

‎src/main/java/fr/insee/genesis/controller/rest/UtilsController.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
55
import fr.insee.genesis.domain.service.volumetry.VolumetryLogService;
66
import fr.insee.genesis.domain.utils.XMLSplitter;
7+
import fr.insee.genesis.exceptions.GenesisException;
78
import io.swagger.v3.oas.annotations.Operation;
89
import io.swagger.v3.oas.annotations.tags.Tag;
910
import lombok.extern.slf4j.Slf4j;
@@ -44,7 +45,7 @@ public ResponseEntity<Object> saveResponsesFromXmlFile(@RequestParam("inputFolde
4445
@RequestParam("outputFolder") String outputFolder,
4546
@RequestParam("filename") String filename,
4647
@RequestParam("nbResponsesByFile") int nbSU)
47-
throws XMLStreamException, IOException {
48+
throws XMLStreamException, IOException, GenesisException {
4849
XMLSplitter.split(inputFolder, filename, outputFolder, "SurveyUnit", nbSU);
4950
return ResponseEntity.ok("File split");
5051
}

‎src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22

33
import com.fasterxml.jackson.core.JsonProcessingException;
44
import com.fasterxml.jackson.databind.ObjectMapper;
5-
import com.networknt.schema.JsonSchema;
6-
import com.networknt.schema.JsonSchemaFactory;
7-
import com.networknt.schema.SpecVersion;
8-
import com.networknt.schema.ValidationMessage;
5+
import com.networknt.schema.Error;
6+
import com.networknt.schema.Schema;
7+
import com.networknt.schema.SchemaRegistry;
8+
import com.networknt.schema.dialect.Dialects;
99
import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
1010
import fr.insee.genesis.domain.model.surveyunit.Mode;
1111
import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
@@ -38,7 +38,6 @@
3838
import java.util.ArrayList;
3939
import java.util.List;
4040
import java.util.Map;
41-
import java.util.Set;
4241
import java.util.stream.Collectors;
4342

4443
@Slf4j
@@ -94,14 +93,15 @@ public ResponseEntity<String> saveRawResponsesFromJsonBody(
9493
public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidation(
9594
@RequestBody Map<String, Object> body
9695
) {
97-
JsonSchema jsonSchema = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7).getSchema(
98-
RawResponseController.class.getResourceAsStream("/jsonSchemas/RawResponse.json")
96+
SchemaRegistry schemaRegistry = SchemaRegistry.withDialect(Dialects.getDraft202012(), SchemaRegistry.Builder::build);
97+
Schema jsonSchema = schemaRegistry
98+
.getSchema(RawResponseController.class.getResourceAsStream("/jsonSchemas/RawResponse.json")
9999
);
100100
try {
101101
if (jsonSchema == null) {
102102
throw new GenesisException(500, "No RawResponse json schema has been found");
103103
}
104-
Set<ValidationMessage> errors = jsonSchema.validate(
104+
List<Error> errors = jsonSchema.validate(
105105
new ObjectMapper().readTree(
106106
new ObjectMapper().writeValueAsString(body)
107107
)
@@ -209,10 +209,10 @@ public ResponseEntity<PagedModel<LunaticJsonRawDataModel>> getRawResponsesFromJs
209209
return ResponseEntity.status(HttpStatus.OK).body(new PagedModel<>(rawResponses));
210210
}
211211

212-
private void validate(Set<ValidationMessage> errors) throws GenesisException {
212+
private void validate(List<Error> errors) throws GenesisException {
213213
if (!errors.isEmpty()) {
214214
String errorMessage = errors.stream()
215-
.map(ValidationMessage::getMessage)
215+
.map(Error::getMessage)
216216
.collect(Collectors.joining(System.lineSeparator() + " - "));
217217

218218
throw new GenesisException(

‎src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ public List<List<SurveyUnitModel>> findLatestByIdAndByQuestionnaireIdAndModeOrde
156156
List<List<SurveyUnitModel>> listLatestUpdatesbyVariables = new ArrayList<>();
157157

158158
//1) QUERY
159-
// => conversion of "List<InterrogationId>" -> "List<String>" for query using lamda
159+
// => conversion of "List<InterrogationId>" -> "List<String>" for query using lambda
160160
List<String> queryInParam = interrogationIds.stream().map(InterrogationId::getInterrogationId).toList();
161161

162162
//Get !!!all versions!!! of a set of "interrogationIds"

‎src/main/java/fr/insee/genesis/domain/utils/XMLSplitter.java‎

Lines changed: 73 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,8 @@
11
package fr.insee.genesis.domain.utils;
22

3+
import fr.insee.genesis.exceptions.GenesisException;
34
import lombok.experimental.UtilityClass;
45

5-
import java.io.FileNotFoundException;
6-
import java.io.FileReader;
7-
import java.io.FileWriter;
8-
import java.io.IOException;
9-
import java.util.ArrayList;
10-
import java.util.List;
11-
126
import javax.xml.XMLConstants;
137
import javax.xml.stream.XMLEventFactory;
148
import javax.xml.stream.XMLEventReader;
@@ -20,12 +14,17 @@
2014
import javax.xml.stream.events.StartDocument;
2115
import javax.xml.stream.events.StartElement;
2216
import javax.xml.stream.events.XMLEvent;
17+
import java.io.FileReader;
18+
import java.io.FileWriter;
19+
import java.io.IOException;
20+
import java.util.ArrayList;
21+
import java.util.List;
2322

2423
@UtilityClass
2524
public class XMLSplitter {
2625

2726
// We use StAX in this class to deal with memory issues on huge XML files
28-
public static void split(String inputfolder, String xmlfile, String outputFolder, String condition, int nbElementsByFile) throws XMLStreamException, IOException {
27+
public static void split(String inputfolder, String xmlfile, String outputFolder, String condition, int nbElementsByFile) throws XMLStreamException, IOException, GenesisException {
2928

3029
String xmlResource = inputfolder + xmlfile;
3130
List<XMLEvent> header = getHeader(xmlResource, condition);
@@ -34,42 +33,45 @@ public static void split(String inputfolder, String xmlfile, String outputFolder
3433
XMLInputFactory xif = XMLInputFactory.newInstance();
3534
xif.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
3635
xif.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
37-
XMLEventReader xer = xif.createXMLEventReader(new FileReader(xmlResource));
38-
StartElement rootStartElement = xer.nextTag().asStartElement();
39-
StartDocument startDocument = xef.createStartDocument();
40-
EndDocument endDocument = xef.createEndDocument();
41-
42-
XMLOutputFactory xof = XMLOutputFactory.newFactory();
43-
int fileCount = 1;
44-
while(xer.hasNext() && !xer.peek().isEndDocument()) {
45-
XMLEvent xmlEvent = xer.nextEvent();
46-
47-
if (isStartElementWithName(condition, xmlEvent)) {
48-
// Create a file for the fragment, the name is derived from the value of the id attribute
49-
FileWriter fileWriter = new FileWriter(outputFolder + "split" + fileCount + ".xml");
50-
51-
// A StAX XMLEventWriter will be used to write the XML fragment
52-
XMLEventWriter xew = xof.createXMLEventWriter(fileWriter);
53-
xew.add(startDocument);
54-
55-
// Add the elements which are common to all split files
56-
addHeadersToNewFile(header, xew);
57-
58-
// Write the XMLEvents that are part of SurveyUnit element
59-
xew.add(xmlEvent);
60-
xmlEvent = xer.nextEvent();
61-
int nbResponses = 1;
62-
// We loop until we reach the end tag Survey units indicating the near end of the document
63-
iterateOnSurveyUnits(condition, nbElementsByFile, xer, xmlEvent, xew, nbResponses);
64-
65-
// Write the file, close everything we opened and update the file's counter
66-
xew.add(xef.createEndElement(rootStartElement.getName(), null));
67-
xew.add(endDocument);
68-
fileWriter.close();
69-
70-
fileCount++;
71-
36+
try (FileReader fr = new FileReader(xmlResource)){
37+
XMLEventReader xer = xif.createXMLEventReader(fr);
38+
39+
StartElement rootStartElement = xer.nextTag().asStartElement();
40+
StartDocument startDocument = xef.createStartDocument();
41+
EndDocument endDocument = xef.createEndDocument();
42+
43+
XMLOutputFactory xof = XMLOutputFactory.newFactory();
44+
int fileCount = 1;
45+
while(xer.hasNext() && !xer.peek().isEndDocument()) {
46+
XMLEvent xmlEvent = xer.nextEvent();
47+
48+
if (isStartElementWithName(condition, xmlEvent)) {
49+
// Create a file for the fragment, the name is derived from the value of the id attribute
50+
FileWriter fileWriter = new FileWriter(outputFolder + "split" + fileCount + ".xml");
51+
52+
// A StAX XMLEventWriter will be used to write the XML fragment
53+
XMLEventWriter xew = xof.createXMLEventWriter(fileWriter);
54+
xew.add(startDocument);
55+
56+
// Add the elements which are common to all split files
57+
addHeadersToNewFile(header, xew);
58+
59+
// Write the XMLEvents that are part of SurveyUnit element
60+
xew.add(xmlEvent);
61+
xmlEvent = xer.nextEvent();
62+
int nbResponses = 1;
63+
// We loop until we reach the end tag Survey units indicating the near end of the document
64+
iterateOnSurveyUnits(condition, nbElementsByFile, xer, xmlEvent, xew, nbResponses);
65+
66+
// Write the file, close everything we opened and update the file's counter
67+
xew.add(xef.createEndElement(rootStartElement.getName(), null));
68+
xew.add(endDocument);
69+
fileWriter.close();
70+
71+
fileCount++;
72+
}
7273
}
74+
xer.close();
7375
}
7476
}
7577

@@ -103,33 +105,37 @@ private static boolean isEndElementWithName(XMLEvent xmlEvent, String condition)
103105
return xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().getLocalPart().equals(condition);
104106
}
105107

106-
private static List<XMLEvent> getHeader(String xmlResource, String condition) throws FileNotFoundException, XMLStreamException {
108+
private static List<XMLEvent> getHeader(String xmlResource, String condition) throws XMLStreamException, GenesisException {
107109
XMLInputFactory xif = XMLInputFactory.newInstance();
108110
xif.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
109111
xif.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
110-
XMLEventReader xer = xif.createXMLEventReader(new FileReader(xmlResource));
111-
112-
List<XMLEvent> cachedXMLEvents = new ArrayList<>();
113-
while(xer.hasNext() && !xer.peek().isEndDocument()) {
114-
XMLEvent xmlEvent = xer.nextTag();
115-
if (!xmlEvent.isStartElement()) {
116-
break;
117-
}
118-
StartElement breakStartElement = xmlEvent.asStartElement();
119-
120-
cachedXMLEvents.add(breakStartElement);
121-
xmlEvent = xer.nextEvent();
122-
while (!(xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().equals(breakStartElement.getName()))) {
123-
if (isStartElementWithName(condition, xmlEvent)) {
124-
xer.close();
125-
return cachedXMLEvents;
126-
}
127-
cachedXMLEvents.add(xmlEvent);
128-
xmlEvent = xer.nextEvent();
129-
}
130-
}
131-
xer.close();
132-
return List.of();
112+
try (FileReader fr = new FileReader(xmlResource)) {
113+
XMLEventReader xer = xif.createXMLEventReader(fr);
114+
115+
List<XMLEvent> cachedXMLEvents = new ArrayList<>();
116+
while (xer.hasNext() && !xer.peek().isEndDocument()) {
117+
XMLEvent xmlEvent = xer.nextTag();
118+
if (!xmlEvent.isStartElement()) {
119+
break;
120+
}
121+
StartElement breakStartElement = xmlEvent.asStartElement();
122+
123+
cachedXMLEvents.add(breakStartElement);
124+
xmlEvent = xer.nextEvent();
125+
while (!(xmlEvent.isEndElement() && xmlEvent.asEndElement().getName().equals(breakStartElement.getName()))) {
126+
if (isStartElementWithName(condition, xmlEvent)) {
127+
xer.close();
128+
return cachedXMLEvents;
129+
}
130+
cachedXMLEvents.add(xmlEvent);
131+
xmlEvent = xer.nextEvent();
132+
}
133+
}
134+
xer.close();
135+
} catch (IOException e) {
136+
throw new GenesisException(500,e.getMessage());
137+
}
138+
return List.of();
133139
}
134140

135141
}

‎src/test/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDtoTest.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package fr.insee.genesis.controller.mappers;
2+
3+
import fr.insee.genesis.controller.dto.ScheduleDto;
4+
import fr.insee.genesis.domain.model.context.DataProcessingContextModel;
5+
import org.bson.types.ObjectId;
6+
import org.junit.jupiter.api.Test;
7+
8+
import java.util.ArrayList;
9+
import java.util.List;
10+
11+
import static java.time.LocalDateTime.now;
12+
import static org.junit.jupiter.api.Assertions.assertEquals;
13+
import static org.junit.jupiter.api.Assertions.assertNotNull;
14+
import static org.junit.jupiter.api.Assertions.assertNull;
15+
16+
class DataProcessingContextMapperDtoTest {
17+
@Test
18+
void dataProcessingContextToScheduleDto() {
19+
DataProcessingContextMapperDto d = new DataProcessingContextMapperDto();
20+
DataProcessingContextModel dataProcessingContext = new DataProcessingContextModel();
21+
22+
ScheduleDto actual = d.dataProcessingContextToScheduleDto(dataProcessingContext);
23+
assertNotNull(actual);
24+
assertNull( actual.lastExecution());
25+
assertNull( actual.surveyName());
26+
assertNull( actual.kraftwerkExecutionScheduleList());
27+
28+
DataProcessingContextMapperDto d2 = new DataProcessingContextMapperDto();
29+
DataProcessingContextModel dataProcessingContext2 = new DataProcessingContextModel();
30+
dataProcessingContext2.setId(new ObjectId());
31+
dataProcessingContext2.setWithReview(true);
32+
dataProcessingContext2.setLastExecution(now());
33+
34+
ScheduleDto actual2 = d2.dataProcessingContextToScheduleDto(dataProcessingContext2);
35+
assertNotNull(actual2);
36+
assertNotNull( actual2.lastExecution());
37+
assertNull( actual2.surveyName());
38+
}
39+
40+
@Test
41+
void dataProcessingContextListToScheduleDtoList() {
42+
DataProcessingContextMapperDto d = new DataProcessingContextMapperDto();
43+
List<DataProcessingContextModel> contexts = new ArrayList<>();
44+
List<ScheduleDto> expected = new ArrayList<>();
45+
List<ScheduleDto> actual = d.dataProcessingContextListToScheduleDtoList(contexts);
46+
47+
assertEquals(expected, actual);
48+
}
49+
}

‎src/test/java/fr/insee/genesis/controller/sources/json/LunaticJsonCollectedVariablesTest.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package fr.insee.genesis.controller.sources.json;
2+
3+
import org.junit.jupiter.api.Test;
4+
5+
class LunaticJsonCollectedVariablesTest {
6+
@Test
7+
void setVariables() {
8+
LunaticJsonCollectedVariables l = new LunaticJsonCollectedVariables();
9+
String variable = "abc";
10+
LunaticJsonVariableData value = new LunaticJsonVariableData();
11+
l.setVariables(variable, value);
12+
}
13+
}

‎src/test/java/fr/insee/genesis/controller/sources/json/LunaticJsonExternalVariablesTest.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package fr.insee.genesis.controller.sources.json;
2+
3+
import org.junit.jupiter.api.Test;
4+
5+
class LunaticJsonExternalVariablesTest {
6+
@Test
7+
void setVariables() {
8+
LunaticJsonExternalVariables l = new LunaticJsonExternalVariables();
9+
String variable = "abc";
10+
String value = "abc";
11+
l.setVariables(variable, value);
12+
}
13+
}

0 commit comments

Comments
 (0)