Skip to content

Commit 3e5e8cc

Browse files
alicelaalicela
authored
Merge pull request #322 from InseeFr/devSonar
Dev sonar
2 parents 1132f9b + 34433ac commit 3e5e8cc

23 files changed

+664
-522
lines changed

‎Dockerfile‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@ ENV JAVA_TOOL_OPTIONS_DEFAULT \
88
-XX:MaxRAMPercentage=75
99

1010
# Setup a non-root user context (security)
11-
RUN addgroup -g 1000 tomcatgroup
12-
RUN adduser -D -s / -u 1000 tomcatuser -G tomcatgroup
13-
RUN mkdir /opt/app/temp-files
14-
RUN chown -R 1000:1000 /opt/app
11+
RUN addgroup -g 1000 tomcatgroup \
12+
&& adduser -D -s / -u 1000 tomcatuser -G tomcatgroup \
13+
&& mkdir /opt/app/temp-files \
14+
&& chown -R 1000:1000 /opt/app
1515

1616
USER 1000
1717

‎src/main/java/fr/insee/genesis/controller/rest/CommonApiResponse.java‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,11 @@
11
package fr.insee.genesis.controller.rest;
22

33
import io.swagger.v3.oas.annotations.responses.ApiResponse;
4-
import io.swagger.v3.oas.annotations.responses.ApiResponses;
54

6-
@ApiResponses({
7-
@ApiResponse(responseCode = "400", description = "Invalid request parameters"),
8-
@ApiResponse(responseCode = "401", description = "Unauthorized"),
9-
@ApiResponse(responseCode = "403", description = "Forbidden"),
10-
@ApiResponse(responseCode = "404", description = "Requested resource not found"),
11-
@ApiResponse(responseCode = "500", description = "Internal server error")
12-
})
5+
@ApiResponse(responseCode = "400", description = "Invalid request parameters")
6+
@ApiResponse(responseCode = "401", description = "Unauthorized")
7+
@ApiResponse(responseCode = "403", description = "Forbidden")
8+
@ApiResponse(responseCode = "404", description = "Requested resource not found")
9+
@ApiResponse(responseCode = "500", description = "Internal server error")
1310
public interface CommonApiResponse {
1411
}

‎src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -102,11 +102,6 @@ private MetadataModel readMetadatas(String campaignName, String modeName, FileUt
102102
}
103103
}
104104

105-
/* // Adding Eno variables if necessary
106-
// For review : not sure if this the best way to do it
107-
for (String enoVar : Arrays.stream(Constants.getEnoVariables()).toList()){
108-
variablesMap.putVariable(new Variable(enoVar, ));
109-
}*/
110105
if(!errors.isEmpty()){
111106
throw new GenesisException(404, errors.getLast().getMessage());
112107
}

‎src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java‎

Lines changed: 92 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -139,25 +139,14 @@ public DataProcessResult processRawData(String campaignName, List<String> interr
139139

140140
//Increment data count
141141
dataCount += surveyUnitModels.size();
142-
formattedDataCount += surveyUnitModels.stream().filter(
143-
surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)
144-
).toList().size();
142+
formattedDataCount += surveyUnitModels.stream()
143+
.filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED))
144+
.toList()
145+
.size();
145146

146147
//Send processed ids grouped by questionnaire (if review activated)
147148
if(dataProcessingContext != null && dataProcessingContext.isWithReview()) {
148-
try {
149-
ResponseEntity<Object> response =
150-
surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels));
151-
152-
if (response.getStatusCode().is2xxSuccessful()) {
153-
log.info("Successfully sent {} ids to quality tool", getProcessedIdsMap(surveyUnitModels).keySet().size());
154-
}else{
155-
log.warn("Survey unit quality tool responded non-2xx code {} and body {}",
156-
response.getStatusCode(), response.getBody());
157-
}
158-
}catch (IOException e){
159-
log.error("Error during Perret call request building : {}", e.toString());
160-
}
149+
sendProcessedIdsToQualityTool(surveyUnitModels);
161150
}
162151

163152
//Remove processed ids from list
@@ -169,6 +158,22 @@ public DataProcessResult processRawData(String campaignName, List<String> interr
169158
return new DataProcessResult(dataCount, formattedDataCount);
170159
}
171160

161+
private void sendProcessedIdsToQualityTool(List<SurveyUnitModel> surveyUnitModels) {
162+
try {
163+
ResponseEntity<Object> response =
164+
surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels));
165+
166+
if (response.getStatusCode().is2xxSuccessful()) {
167+
log.info("Successfully sent {} ids to quality tool", getProcessedIdsMap(surveyUnitModels).size());
168+
}else{
169+
log.warn("Survey unit quality tool responded non-2xx code {} and body {}",
170+
response.getStatusCode(), response.getBody());
171+
}
172+
}catch (IOException e){
173+
log.error("Error during Perret call request building : {}", e.toString());
174+
}
175+
}
176+
172177
private Map<String, Set<String>> getProcessedIdsMap(List<SurveyUnitModel> surveyUnitModels) {
173178
Map<String, Set<String>> processedInterrogationIdsPerQuestionnaire = new HashMap<>();
174179
surveyUnitModels.forEach(model ->
@@ -186,24 +191,12 @@ public List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawDat
186191
//For each possible data state (we receive COLLECTED or EDITED)
187192
for(DataState dataState : List.of(DataState.COLLECTED,DataState.EDITED)){
188193
for (LunaticJsonRawDataModel rawData : rawDataList) {
189-
RawDataModelType rawDataModelType =
190-
rawData.data().containsKey("data") ?
191-
RawDataModelType.FILIERE :
192-
RawDataModelType.DEFAULT;
194+
RawDataModelType rawDataModelType = getRawDataModelType(rawData);
193195

194196
//Get optional fields
195-
String contextualId = null;
196-
Boolean isCapturedIndirectly = null;
197-
LocalDateTime validationDate = null;
198-
try{
199-
contextualId = rawData.data().get("contextualId") == null ? null : rawData.data().get("contextualId").toString();
200-
isCapturedIndirectly = rawData.data().get("isCapturedIndirectly") == null ? null :
201-
Boolean.parseBoolean(rawData.data().get("isCapturedIndirectly").toString());
202-
validationDate = rawData.data().get("validationDate") == null ? null :
203-
LocalDateTime.parse(rawData.data().get("validationDate").toString());
204-
}catch(Exception e){
205-
log.warn("Exception during optional fields parsing : %s".formatted(e.toString()));
206-
}
197+
String contextualId = getContextualId(rawData);
198+
Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawData);
199+
LocalDateTime validationDate = getValidationDate(rawData);
207200

208201
SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder()
209202
.campaignId(rawData.campaignId())
@@ -225,17 +218,18 @@ public List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawDat
225218
convertRawDataCollectedVariables(rawData, surveyUnitModel, dataState, rawDataModelType, variablesMap);
226219

227220
//External variables conversion into COLLECTED document
228-
if(dataState.equals(DataState.COLLECTED)){
221+
if(dataState == DataState.COLLECTED){
229222
convertRawDataExternalVariables(rawData, surveyUnitModel, rawDataModelType, variablesMap);
230223
}
231224

232-
if(surveyUnitModel.getCollectedVariables().isEmpty()
233-
&& surveyUnitModel.getExternalVariables().isEmpty()
234-
){
225+
boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty()
226+
&& surveyUnitModel.getExternalVariables().isEmpty();
227+
228+
if(hasNoVariable){
235229
if(surveyUnitModel.getState() == DataState.COLLECTED){
236230
log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawData.interrogationId());
237231
}
238-
continue;
232+
continue;// don't add suModel
239233
}
240234
surveyUnitModels.add(surveyUnitModel);
241235
}
@@ -244,6 +238,42 @@ public List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawDat
244238
return surveyUnitModels;
245239
}
246240

241+
private static RawDataModelType getRawDataModelType(LunaticJsonRawDataModel rawData) {
242+
return rawData.data().containsKey("data") ?
243+
RawDataModelType.FILIERE :
244+
RawDataModelType.DEFAULT;
245+
}
246+
247+
private static LocalDateTime getValidationDate(LunaticJsonRawDataModel rawData) {
248+
try{
249+
return rawData.data().get("validationDate") == null ? null :
250+
LocalDateTime.parse(rawData.data().get("validationDate").toString());
251+
}catch(Exception e){
252+
log.warn("Exception when parsing validation date : {}}",e.toString());
253+
return null;
254+
}
255+
}
256+
257+
private static Boolean getIsCapturedIndirectly(LunaticJsonRawDataModel rawData) {
258+
try{
259+
return rawData.data().get("isCapturedIndirectly") == null ? null :
260+
Boolean.parseBoolean(rawData.data().get("isCapturedIndirectly").toString());
261+
}catch(Exception e){
262+
log.warn("Exception when parsing isCapturedIndirectly : {}}",e.toString());
263+
return Boolean.FALSE;
264+
}
265+
266+
}
267+
268+
private static String getContextualId(LunaticJsonRawDataModel rawData) {
269+
try{
270+
return rawData.data().get("contextualId") == null ? null : rawData.data().get("contextualId").toString();
271+
}catch(Exception e){
272+
log.warn("Exception when parsing contextual id : {}}",e.toString());
273+
return null;
274+
}
275+
}
276+
247277
@Override
248278
public List<LunaticJsonRawDataUnprocessedDto> getUnprocessedDataIds() {
249279
List<LunaticJsonRawDataUnprocessedDto> dtos = new ArrayList<>();
@@ -333,22 +363,34 @@ private void convertRawDataCollectedVariables(
333363

334364
}
335365

336-
private static void convertToCollectedVar(SurveyUnitModel dstSurveyUnitModel, DataState dataState, VariablesMap variablesMap, Map<String, Object> collectedMap) {
337-
for(Map.Entry<String, Object> collectedVariable : collectedMap.entrySet()) {
366+
private static void convertToCollectedVar(
367+
SurveyUnitModel dstSurveyUnitModel,
368+
DataState dataState,
369+
VariablesMap variablesMap,
370+
Map<String, Object> collectedMap
371+
) {
372+
final String stateKey = dataState.toString();
373+
final var dest = dstSurveyUnitModel.getCollectedVariables();
338374

339-
//Skip if collected variable does not have state
340-
if(!JsonUtils.asMap(collectedVariable.getValue()).containsKey(dataState.toString())){
341-
continue;
342-
}
375+
for (Map.Entry<String, Object> collectedVariable : collectedMap.entrySet()) {
376+
// Map for this variable (COLLECTED/EDITED -> value)
377+
Map<String, Object> states = JsonUtils.asMap(collectedVariable.getValue());
343378

344-
//Value
345-
Object valuesForState = JsonUtils.asMap(collectedVariable.getValue()).get(dataState.toString());
346-
if (valuesForState != null) {
347-
if (valuesForState instanceof List<?>) {
348-
convertListVar(valuesForState, collectedVariable, variablesMap, dstSurveyUnitModel.getCollectedVariables());
349-
continue;
379+
// nothing if no state
380+
if (states.containsKey(stateKey)) {
381+
Object value = states.get(stateKey);
382+
383+
// liste ?
384+
if (value instanceof List<?>) {
385+
// on garde exactement ta signature existante
386+
convertListVar(value, collectedVariable, variablesMap, dest);
387+
}
388+
389+
// scalaire non null ?
390+
if (value != null && !(value instanceof List<?>)) {
391+
// idem: on garde convertOneVar(entry, String, ...)
392+
convertOneVar(collectedVariable, String.valueOf(value), variablesMap, 1, dest);
350393
}
351-
convertOneVar(collectedVariable, valuesForState.toString(), variablesMap, 1, dstSurveyUnitModel.getCollectedVariables());
352394
}
353395
}
354396
}

‎src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java‎

Lines changed: 22 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ public List<InterrogationId> findDistinctInterrogationIdsByQuestionnaireIdAndDat
289289

290290
/**
291291
* @author Adrien Marchal
292-
* Calculations made to establish the data a worker will be responsible of, among the whole data to be processed.
292+
* Calculations made to establish the data a worker will be responsible for, among the whole data to be processed.
293293
* (needed for distributed process / horizontal scaling)
294294
*/
295295
@Override
@@ -544,18 +544,16 @@ private void extractVariables(SurveyUnitModel surveyUnitModel,
544544
for (VariableModel collectedVariable : surveyUnitModel.getCollectedVariables()) {
545545
VarIdScopeTuple loopIdTuple = new VarIdScopeTuple(collectedVariable.varId(), collectedVariable.scope(),
546546
collectedVariable.iteration());
547-
VariableDto variableDto = collectedVariableMap.get(loopIdTuple);
548-
549-
//Create variable into map if not exists
550-
if (variableDto == null) {
551-
variableDto = VariableDto.builder()
552-
.variableName(collectedVariable.varId())
553-
.scope(collectedVariable.scope())
554-
.iteration(collectedVariable.iteration())
555-
.variableStateDtoList(new ArrayList<>())
556-
.build();
557-
collectedVariableMap.put(loopIdTuple, variableDto);
558-
}
547+
548+
VariableDto variableDto = collectedVariableMap.computeIfAbsent(loopIdTuple, k ->
549+
VariableDto.builder()
550+
.variableName(collectedVariable.varId())
551+
.scope(collectedVariable.scope())
552+
.iteration(collectedVariable.iteration())
553+
.variableStateDtoList(new ArrayList<>())
554+
.build()
555+
);
556+
559557
//Extract variable state
560558
if (isMostRecentForSameState(surveyUnitModel, variableDto)) {
561559
variableDto.getVariableStateDtoList().add(
@@ -577,18 +575,17 @@ private void extractVariables(SurveyUnitModel surveyUnitModel,
577575
}
578576
for(VariableModel externalVariable : surveyUnitModel.getExternalVariables()){
579577
VarIdScopeTuple loopIdTuple = new VarIdScopeTuple(externalVariable.varId(), externalVariable.scope(), externalVariable.iteration());
580-
VariableDto variableDto = externalVariableMap.get(loopIdTuple);
581-
582-
//Create variable into map if not exists
583-
if(variableDto == null){
584-
variableDto = VariableDto.builder()
585-
.variableName(externalVariable.varId())
586-
.scope(externalVariable.scope())
587-
.iteration(externalVariable.iteration())
588-
.variableStateDtoList(new ArrayList<>())
589-
.build();
590-
externalVariableMap.put(loopIdTuple, variableDto);
591-
}
578+
579+
VariableDto variableDto = externalVariableMap.computeIfAbsent(loopIdTuple, k ->
580+
VariableDto.builder()
581+
.variableName(externalVariable.varId())
582+
.scope(externalVariable.scope())
583+
.iteration(externalVariable.iteration())
584+
.variableStateDtoList(new ArrayList<>())
585+
.build()
586+
);
587+
588+
592589
//Extract variable state
593590
if(isMostRecentForSameState(surveyUnitModel, variableDto)){
594591
variableDto.getVariableStateDtoList().add(

‎src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
import fr.insee.genesis.configuration.Config;
44
import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort;
5-
import fr.insee.genesis.infrastructure.utils.http.OidcService;
65
import fr.insee.genesis.infrastructure.utils.http.HttpUtils;
6+
import fr.insee.genesis.infrastructure.utils.http.OidcService;
77
import lombok.extern.slf4j.Slf4j;
88
import org.springframework.beans.factory.annotation.Autowired;
99
import org.springframework.beans.factory.annotation.Qualifier;
@@ -20,11 +20,14 @@
2020
@Slf4j
2121
public class SurveyQualityToolPerretAdapter implements SurveyUnitQualityToolPort {
2222

23-
@Autowired
24-
Config config;
23+
private final Config config;
24+
private final OidcService oidcService;
2525

2626
@Autowired
27-
OidcService oidcService;
27+
public SurveyQualityToolPerretAdapter( Config config,OidcService oidcService) {
28+
this.config = config;
29+
this.oidcService = oidcService;
30+
}
2831

2932

3033
@Override

0 commit comments

Comments
 (0)