Skip to content

Commit 1df9784

Browse files
loichenningerloichenninger
authored
Merge pull request #193 from InseeFr/devRawEndpoints
Add raw data services
2 parents 48df867 + f60ce69 commit 1df9784

File tree

15 files changed

+357
-16
lines changed

15 files changed

+357
-16
lines changed

‎src/main/java/fr/insee/genesis/Constants.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@ public class Constants {
1616
public static final String MONGODB_SCHEDULE_COLLECTION_NAME = "schedules";
1717
public static final String LOOP_NAME_PREFIX = "BOUCLE";
1818
public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses";
19+
public static final String MONGODB_RESPONSE_RAW_COLLECTION_NAME = "lunaticjsondata";
1920
public static final String MONGODB_VARIABLETYPE_COLLECTION_NAME = "variabletypes";
2021
public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries";
2122
public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY";
23+
public static final String VOLUMETRY_RAW_FILE_SUFFIX = "_RAW_VOLUMETRY";
2224
public static final String VOLUMETRY_FILE_DATE_FORMAT = "yyyy_MM_dd";
2325
public static final int VOLUMETRY_FILE_EXPIRATION_DAYS = 30;
2426

‎src/main/java/fr/insee/genesis/controller/rest/UtilsController.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package fr.insee.genesis.controller.rest;
22

3+
import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
34
import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
45
import fr.insee.genesis.domain.service.volumetry.VolumetryLogService;
56
import fr.insee.genesis.domain.utils.XMLSplitter;
@@ -23,14 +24,15 @@ public class UtilsController {
2324

2425
private final VolumetryLogService volumetryLogService;
2526
private final SurveyUnitApiPort surveyUnitService;
27+
private final LunaticJsonRawDataApiPort lunaticJsonRawDataService;
2628

2729

2830

29-
public UtilsController(SurveyUnitApiPort surveyUnitService,VolumetryLogService volumetryLogService) {
31+
public UtilsController(SurveyUnitApiPort surveyUnitService, VolumetryLogService volumetryLogService, LunaticJsonRawDataApiPort lunaticJsonRawDataService) {
3032
this.surveyUnitService = surveyUnitService;
3133
this.volumetryLogService = volumetryLogService;
32-
33-
}
34+
this.lunaticJsonRawDataService = lunaticJsonRawDataService;
35+
}
3436

3537

3638

@@ -51,6 +53,7 @@ public ResponseEntity<Object> saveResponsesFromXmlFile(@RequestParam("inputFolde
5153
@PreAuthorize("hasRole('SCHEDULER')")
5254
public ResponseEntity<Object> saveVolumetry() throws IOException {
5355
volumetryLogService.writeVolumetries(surveyUnitService);
56+
volumetryLogService.writeRawDataVolumetries(lunaticJsonRawDataService);
5457
volumetryLogService.cleanOldFiles();
5558
return ResponseEntity.ok("Volumetric saved");
5659
}

‎src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import fr.insee.genesis.domain.model.surveyunit.Mode;
55
import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
66
import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService;
7+
import io.swagger.v3.oas.annotations.Hidden;
78
import io.swagger.v3.oas.annotations.Operation;
89
import lombok.extern.slf4j.Slf4j;
910
import org.springframework.http.ResponseEntity;
@@ -86,6 +87,18 @@ public ResponseEntity<List<LunaticJsonRawDataUnprocessedDto>> getUnproccessedJso
8687
return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataIds());
8788
}
8889

90+
@Hidden
91+
@GetMapping(path= "/lunatic-json/get/by-interrogation-mode-and-campaign")
92+
@PreAuthorize("hasRole('ADMIN')")
93+
public ResponseEntity<LunaticJsonRawDataModel> getJsonRawData(
94+
@RequestParam("interrogationId") String interrogationId,
95+
@RequestParam("campaignName") String campaignName,
96+
@RequestParam(value = "mode") Mode modeSpecified
97+
){
98+
List<LunaticJsonRawDataModel> data = lunaticJsonRawDataApiPort.getRawData(campaignName,modeSpecified,List.of(interrogationId));
99+
return ResponseEntity.ok(data.getFirst());
100+
}
101+
89102
//PROCESS
90103
/* @Operation(summary = "Process raw data of a campaign")
91104
@PostMapping(path = "/lunatic-json/process")

‎src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import fr.insee.genesis.exceptions.GenesisException;
2929
import fr.insee.genesis.exceptions.NoDataException;
3030
import fr.insee.genesis.infrastructure.utils.FileUtils;
31+
import io.swagger.v3.oas.annotations.Hidden;
3132
import io.swagger.v3.oas.annotations.Operation;
3233
import io.swagger.v3.oas.annotations.tags.Tag;
3334
import lombok.extern.slf4j.Slf4j;
@@ -225,6 +226,7 @@ public ResponseEntity<SurveyUnitQualityToolDto> findResponsesByInterrogationAndQ
225226
return ResponseEntity.ok(responseQualityTool);
226227
}
227228

229+
@Hidden
228230
@Operation(summary = "Retrieve all responses (for all interrogations) of one questionnaire")
229231
@GetMapping(path = "/by-questionnaire")
230232
@PreAuthorize("hasRole('ADMIN')")

‎src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
88

99
import java.util.List;
10+
import java.util.Set;
1011

1112
public interface LunaticJsonRawDataApiPort {
1213

@@ -21,4 +22,6 @@ List<SurveyUnitModel> parseRawData(
2122
VariablesMap variablesMap
2223
);
2324
void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
25+
Set<String> findDistinctQuestionnaireIds();
26+
long countResponsesByQuestionnaireId(String campaignId);
2427
}

‎src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,6 @@ public interface LunaticJsonRawDataPersistencePort {
1212
List<LunaticJsonRawDataModel> findRawData(String campaignName, Mode mode, List<String> interrogationIdList);
1313
List<LunaticJsonRawDataModel> getAllUnprocessedData();
1414
void updateProcessDates(String campaignId, Set<String> interrogationIds);
15-
15+
Set<String> findDistinctQuestionnaireIds();
16+
long countResponsesByQuestionnaireId(String questionnaireId);
1617
}

‎src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,16 @@ public void updateProcessDates(List<SurveyUnitModel> surveyUnitModels) {
268268
}
269269
}
270270

271+
@Override
272+
public Set<String> findDistinctQuestionnaireIds() {
273+
return lunaticJsonRawDataPersistencePort.findDistinctQuestionnaireIds();
274+
}
275+
276+
@Override
277+
public long countResponsesByQuestionnaireId(String campaignId) {
278+
return lunaticJsonRawDataPersistencePort.countResponsesByQuestionnaireId(campaignId);
279+
}
280+
271281
/*
272282
*//**
273283
* Parse collected variables from raw data JSON

‎src/main/java/fr/insee/genesis/domain/service/volumetry/VolumetryLogService.java‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import fr.insee.genesis.Constants;
44
import fr.insee.genesis.configuration.Config;
55
import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
6+
import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
67
import lombok.extern.slf4j.Slf4j;
78
import org.springframework.beans.factory.annotation.Autowired;
89
import org.springframework.stereotype.Service;
@@ -47,17 +48,40 @@ public void writeVolumetries(SurveyUnitApiPort surveyUnitApiPort) throws IOExcep
4748

4849
Files.writeString(logFilePath, line, StandardOpenOption.APPEND);
4950
}
51+
}
5052

53+
public void writeRawDataVolumetries(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort) throws IOException {
54+
Path logFilePath = Path.of(config.getLogFolder()).resolve(Constants.VOLUMETRY_FOLDER_NAME)
55+
.resolve(
56+
LocalDate.now().format(DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT))
57+
+ Constants.VOLUMETRY_RAW_FILE_SUFFIX + ".csv");
58+
Files.createDirectories(logFilePath.getParent());
59+
//Overwrite log file with header if exists
60+
if (Files.exists(logFilePath)){
61+
Files.delete(logFilePath);
62+
}
63+
Files.writeString(logFilePath, "questionnaire;volumetry\n");
64+
65+
//Write lines
66+
Set<String> questionnaires = lunaticJsonRawDataApiPort.findDistinctQuestionnaireIds();
67+
for (String questionnaireId : questionnaires) {
68+
long countResult = lunaticJsonRawDataApiPort.countResponsesByQuestionnaireId(questionnaireId);
69+
70+
String line = questionnaireId + ";" + countResult + "\n";
71+
72+
Files.writeString(logFilePath, line, StandardOpenOption.APPEND);
73+
}
5174
}
5275

5376
public void cleanOldFiles() throws IOException {
5477
try (Stream<Path> pathStream = Files.walk(Path.of(config.getLogFolder()).resolve(Constants.VOLUMETRY_FOLDER_NAME))){
5578
for (Path logFilePath : pathStream.filter(path -> path.getFileName().toString().endsWith(".csv")).toList()){
5679
//If older than x months
57-
if (LocalDate.parse(
58-
logFilePath.getFileName().toString().replace(Constants.VOLUMETRY_FILE_SUFFIX + ".csv", ""),
59-
DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT)
60-
).isBefore(LocalDate.now().minusDays(Constants.VOLUMETRY_FILE_EXPIRATION_DAYS))
80+
//Extract date
81+
String datePart = logFilePath.getFileName().toString().split("_VOLUMETRY\\.csv")[0] // Delete common suffix
82+
.replace("_RAW", ""); // Delete "_RAW" if present
83+
if (LocalDate.parse(datePart, DateTimeFormatter.ofPattern(Constants.VOLUMETRY_FILE_DATE_FORMAT))
84+
.isBefore(LocalDate.now().minusDays(Constants.VOLUMETRY_FILE_EXPIRATION_DAYS))
6185
) {
6286
Files.deleteIfExists(logFilePath);
6387
log.info("Deleted {}", logFilePath);

‎src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.springframework.stereotype.Service;
1717

1818
import java.time.LocalDateTime;
19+
import java.util.HashSet;
1920
import java.util.List;
2021
import java.util.Set;
2122

@@ -44,8 +45,7 @@ public List<LunaticJsonRawDataModel> getAllUnprocessedData() {
4445

4546
@Override
4647
public List<LunaticJsonRawDataModel> findRawData(String campaignName, Mode mode, List<String> interrogationIdList) {
47-
List<LunaticJsonRawDataDocument> rawDataDocs = repository.findModesByCampaignIdAndByModeAndinterrogationIdIninterrogationIdList(campaignName, mode, interrogationIdList);
48-
return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs);
48+
List<LunaticJsonRawDataDocument> rawDataDocs = repository.findModesByCampaignIdAndByModeAndinterrogationIdIninterrogationIdList(campaignName, mode, interrogationIdList);return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs);
4949
}
5050

5151
@Override
@@ -56,4 +56,20 @@ public void updateProcessDates(String campaignId, Set<String> interrogationIds)
5656
, Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME
5757
);
5858
}
59+
60+
@Override
61+
public Set<String> findDistinctQuestionnaireIds() {
62+
Set<String> questionnaireIds = new HashSet<>();
63+
for(String questionnaireId : mongoTemplate.getCollection(Constants.MONGODB_RESPONSE_RAW_COLLECTION_NAME).distinct(
64+
"questionnaireId",
65+
String.class)){
66+
questionnaireIds.add(questionnaireId);
67+
}
68+
return questionnaireIds;
69+
}
70+
71+
@Override
72+
public long countResponsesByQuestionnaireId(String questionnaireId) {
73+
return repository.countByQuestionnaireId(questionnaireId);
74+
}
5975
}

‎src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,7 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository<LunaticJso
1717
List<Mode> findModesByCampaignId(String campaignId);
1818
@Query(value = "{ 'campaignId' : ?0, 'mode' : ?1, 'interrogationId': {$in: ?2} }")
1919
List<LunaticJsonRawDataDocument> findModesByCampaignIdAndByModeAndinterrogationIdIninterrogationIdList(String campaignName, Mode mode, List<String> interrogationIdList);
20+
long countByQuestionnaireId(String questionnaireId);
21+
22+
2023
}

0 commit comments

Comments
 (0)