Skip to content

Commit 5b14413

Browse files
loichenningerloichenninger
committed
Merge branch 'devRawDataProcessing' into develop
2 parents f4f805e + 7f8db99 commit 5b14413

File tree

63 files changed

+184105
-707
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

63 files changed

+184105
-707
lines changed

‎qodana.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#-------------------------------------------------------------------------------#
2+
# Qodana analysis is configured by qodana.yaml file #
3+
# https://www.jetbrains.com/help/qodana/qodana-yaml.html #
4+
#-------------------------------------------------------------------------------#
5+
version: "1.0"
6+
7+
#Specify inspection profile for code analysis
8+
profile:
9+
name: qodana.starter
10+
11+
#Enable inspections
12+
#include:
13+
# - name: <SomeEnabledInspectionId>
14+
15+
#Disable inspections
16+
#exclude:
17+
# - name: <SomeDisabledInspectionId>
18+
# paths:
19+
# - <path/where/not/run/inspection>
20+
21+
projectJDK: 21 #(Applied in CI/CD pipeline)
22+
23+
#Execute shell command before Qodana execution (Applied in CI/CD pipeline)
24+
#bootstrap: sh ./prepare-qodana.sh
25+
26+
#Install IDE plugins before Qodana execution (Applied in CI/CD pipeline)
27+
#plugins:
28+
# - id: <plugin.id> #(plugin id can be found at https://plugins.jetbrains.com)
29+
30+
#Specify Qodana linter for analysis (Applied in CI/CD pipeline)
31+
linter: jetbrains/qodana-jvm-community:latest

‎src/main/java/fr/insee/genesis/Constants.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@ public class Constants {
1010
"(^([0-9]|[0-2][0-9]|3[0-1])[\\-\\/]([0-9]|1[0-2]|0[1-9])[\\-\\/]([0-9]{4})$)";
1111
public static final String FILTER_RESULT_PREFIX = "FILTER_RESULT_";
1212
public static final String MISSING_SUFFIX = "_MISSING";
13+
public static final String MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME = "lunaticjsondata";
1314
private static final String[] ENO_VARIABLES = {"COMMENT_QE","COMMENT_UE","HEURE_REMPL","MIN_REMPL"};
1415

1516
public static final String MONGODB_SCHEDULE_COLLECTION_NAME = "schedules";
1617
public static final String LOOP_NAME_PREFIX = "BOUCLE";
1718
public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses";
19+
public static final String MONGODB_VARIABLETYPE_COLLECTION_NAME = "variabletypes";
1820
public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries";
1921
public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY";
2022
public static final String VOLUMETRY_FILE_DATE_FORMAT = "yyyy_MM_dd";

‎src/main/java/fr/insee/genesis/configuration/Config.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@
22

33
import lombok.Getter;
44
import org.springframework.beans.factory.annotation.Value;
5+
import org.springframework.cache.annotation.EnableCaching;
56
import org.springframework.context.annotation.Configuration;
67

78
import java.nio.file.Path;
89

910
@Configuration
1011
@Getter
12+
@EnableCaching
1113
public class Config {
1214

1315
/******************************************************/

‎src/main/java/fr/insee/genesis/controller/dto/rawdata/LunaticJsonRawDataUnprocessedDto.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package fr.insee.genesis.controller.dto.rawdata;
2+
3+
import lombok.Builder;
4+
5+
@Builder
6+
public record LunaticJsonRawDataUnprocessedDto(String campaignId, String interrogationId){}

‎src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
package fr.insee.genesis.controller.rest.responses;
2+
3+
import fr.insee.bpm.metadata.model.VariablesMap;
4+
import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
5+
import fr.insee.genesis.controller.services.MetadataService;
6+
import fr.insee.genesis.controller.utils.ControllerUtils;
7+
import fr.insee.genesis.domain.model.surveyunit.DataState;
8+
import fr.insee.genesis.domain.model.surveyunit.Mode;
9+
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
10+
import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
11+
import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
12+
import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
13+
import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService;
14+
import fr.insee.genesis.exceptions.GenesisError;
15+
import fr.insee.genesis.exceptions.GenesisException;
16+
import fr.insee.genesis.infrastructure.utils.FileUtils;
17+
import io.swagger.v3.oas.annotations.Operation;
18+
import lombok.extern.slf4j.Slf4j;
19+
import org.springframework.http.ResponseEntity;
20+
import org.springframework.stereotype.Controller;
21+
import org.springframework.web.bind.annotation.GetMapping;
22+
import org.springframework.web.bind.annotation.PostMapping;
23+
import org.springframework.web.bind.annotation.PutMapping;
24+
import org.springframework.web.bind.annotation.RequestBody;
25+
import org.springframework.web.bind.annotation.RequestMapping;
26+
import org.springframework.web.bind.annotation.RequestParam;
27+
28+
import java.time.LocalDateTime;
29+
import java.util.ArrayList;
30+
import java.util.List;
31+
import java.util.Map;
32+
33+
@Slf4j
34+
@Controller
35+
@RequestMapping(path = "/responses/raw" )
36+
public class RawResponseController {
37+
38+
private static final String SUCCESS_MESSAGE = "Interrogation %s saved";
39+
private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
40+
private final ControllerUtils controllerUtils;
41+
private final MetadataService metadataService;
42+
private final SurveyUnitService surveyUnitService;
43+
private final SurveyUnitQualityService surveyUnitQualityService;
44+
private final FileUtils fileUtils;
45+
46+
public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, ControllerUtils controllerUtils, MetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils) {
47+
this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
48+
this.controllerUtils = controllerUtils;
49+
this.metadataService = metadataService;
50+
this.surveyUnitService = surveyUnitService;
51+
this.surveyUnitQualityService = surveyUnitQualityService;
52+
this.fileUtils = fileUtils;
53+
}
54+
55+
@Operation(summary = "Save lunatic json data from one interrogation in Genesis Database")
56+
@PutMapping(path = "/lunatic-json/save")
57+
public ResponseEntity<Object> saveRawResponsesFromJsonBody(
58+
@RequestParam("campaignName") String campaignName,
59+
@RequestParam("questionnaireId") String questionnaireId,
60+
@RequestParam("interrogationId") String interrogationId,
61+
@RequestParam(value = "surveyUnitId", required = false) String idUE,
62+
@RequestParam(value = "mode") Mode modeSpecified,
63+
@RequestBody Map<String, Object> dataJson
64+
) {
65+
log.info("Try to save interrogationId {} for campaign {}",interrogationId,campaignName);
66+
LunaticJsonRawDataModel rawData = LunaticJsonRawDataModel.builder()
67+
.campaignId(campaignName)
68+
.questionnaireId(questionnaireId)
69+
.interrogationId(interrogationId)
70+
.idUE(idUE)
71+
.mode(modeSpecified)
72+
.data(dataJson)
73+
.recordDate(LocalDateTime.now())
74+
.build();
75+
try {
76+
lunaticJsonRawDataApiPort.save(rawData);
77+
} catch (Exception e){
78+
return ResponseEntity.status(500).body("Unexpected error");
79+
}
80+
log.info("Data saved for interrogationId {} and campaign {}",interrogationId, campaignName);
81+
// Collect platform prefer code 201 in case of success
82+
return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE,interrogationId));
83+
}
84+
85+
//GET unprocessed
86+
@Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data")
87+
@GetMapping(path = "/lunatic-json/get/unprocessed")
88+
public ResponseEntity<List<LunaticJsonRawDataUnprocessedDto>> getUnproccessedJsonRawData(){
89+
log.info("Try to get unprocessed raw JSON datas...");
90+
return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataIds());
91+
}
92+
93+
//PROCESS
94+
@Operation(summary = "Process raw data of a campaign")
95+
@PostMapping(path = "/lunatic-json/process")
96+
public ResponseEntity<Object> processJsonRawData(
97+
@RequestParam("campaignName") String campaignName,
98+
@RequestParam("questionnaireId") String questionnaireId,
99+
@RequestBody List<String> interrogationIdList
100+
){
101+
log.info("Try to process raw JSON datas for campaign {} and {} interrogationIds", campaignName, interrogationIdList.size());
102+
103+
int dataCount = 0;
104+
int forcedDataCount = 0;
105+
List<GenesisError> errors = new ArrayList<>();
106+
107+
try {
108+
List<Mode> modesList = controllerUtils.getModesList(campaignName, null);
109+
for (Mode mode : modesList) {
110+
//Load and save metadatas into database, throw exception if none
111+
VariablesMap variablesMap = metadataService.readMetadatas(campaignName, mode.getModeName(), fileUtils,
112+
errors);
113+
if (variablesMap == null) {
114+
throw new GenesisException(400,
115+
"Error during metadata parsing for mode %s :%n%s"
116+
.formatted(mode, errors.getLast().getMessage())
117+
);
118+
}
119+
120+
List<LunaticJsonRawDataModel> rawData = lunaticJsonRawDataApiPort.getRawData(campaignName,mode,interrogationIdList);
121+
//Save converted data
122+
List<SurveyUnitModel> surveyUnitModels = lunaticJsonRawDataApiPort.convertRawData(
123+
rawData,
124+
variablesMap
125+
);
126+
127+
surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap);
128+
surveyUnitService.saveSurveyUnits(surveyUnitModels);
129+
130+
//Update process dates
131+
lunaticJsonRawDataApiPort.updateProcessDates(surveyUnitModels);
132+
133+
//Save metadatas
134+
//TODO Enable when mapping problem solved for get metadatas step
135+
//variableTypeApiPort.saveMetadatas(campaignName, questionnaireId, mode, variablesMap);
136+
137+
//Increment data count
138+
dataCount += surveyUnitModels.size();
139+
forcedDataCount += surveyUnitModels.stream().filter(
140+
surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORCED)
141+
).toList().size();
142+
}
143+
return forcedDataCount == 0 ?
144+
ResponseEntity.ok("%d document(s) processed".formatted(dataCount))
145+
: ResponseEntity.ok("%d document(s) processed, including %d FORCED after data verification"
146+
.formatted(dataCount, forcedDataCount));
147+
}catch (GenesisException e){ //TODO replace with spring exception handler
148+
return ResponseEntity.status(e.getStatus()).body(e.getMessage());
149+
}
150+
}
151+
152+
}

0 commit comments

Comments
 (0)