Dev sonar

Dockerfile

@@ -8,10 +8,10 @@ ENV JAVA_TOOL_OPTIONS_DEFAULT \
     -XX:MaxRAMPercentage=75
 
 # Setup a non-root user context (security)
-RUN addgroup -g 1000 tomcatgroup
-RUN adduser -D -s / -u 1000 tomcatuser -G tomcatgroup
-RUN mkdir /opt/app/temp-files
-RUN chown -R 1000:1000 /opt/app
+RUN addgroup -g 1000 tomcatgroup \
+&& adduser -D -s / -u 1000 tomcatuser -G tomcatgroup \
+&& mkdir /opt/app/temp-files \
+&& chown -R 1000:1000 /opt/app
 
 USER 1000
 

src/main/java/fr/insee/genesis/controller/rest/CommonApiResponse.java

@@ -1,14 +1,11 @@
 package fr.insee.genesis.controller.rest;
 
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.responses.ApiResponses;
 
-@ApiResponses({
-        @ApiResponse(responseCode = "400", description = "Invalid request parameters"),
-        @ApiResponse(responseCode = "401", description = "Unauthorized"),
-        @ApiResponse(responseCode = "403", description = "Forbidden"),
-        @ApiResponse(responseCode = "404", description = "Requested resource not found"),
-        @ApiResponse(responseCode = "500", description = "Internal server error")
-})
+@ApiResponse(responseCode = "400", description = "Invalid request parameters")
+@ApiResponse(responseCode = "401", description = "Unauthorized")
+@ApiResponse(responseCode = "403", description = "Forbidden")
+@ApiResponse(responseCode = "404", description = "Requested resource not found")
+@ApiResponse(responseCode = "500", description = "Internal server error")
 public interface CommonApiResponse {
 }

src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java

@@ -102,11 +102,6 @@ private MetadataModel readMetadatas(String campaignName, String modeName, FileUt
             }
         }
 
-/*        // Adding Eno variables if necessary
-        // For review : not sure if this the best way to do it
-        for (String enoVar : Arrays.stream(Constants.getEnoVariables()).toList()){
-            variablesMap.putVariable(new Variable(enoVar, ));
-        }*/
         if(!errors.isEmpty()){
             throw new GenesisException(404, errors.getLast().getMessage());
         }

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java

@@ -139,25 +139,14 @@ public DataProcessResult processRawData(String campaignName, List<String> interr
 
                 //Increment data count
                 dataCount += surveyUnitModels.size();
-                formattedDataCount += surveyUnitModels.stream().filter(
-                        surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)
-                ).toList().size();
+                formattedDataCount += surveyUnitModels.stream()
+                        .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED))
+                        .toList()
+                        .size();
 
                 //Send processed ids grouped by questionnaire (if review activated)
                 if(dataProcessingContext != null && dataProcessingContext.isWithReview()) {
-                    try {
-                        ResponseEntity<Object> response =
-                                surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels));
-
-                    if (response.getStatusCode().is2xxSuccessful()) {
-                        log.info("Successfully sent {} ids to quality tool", getProcessedIdsMap(surveyUnitModels).keySet().size());
-                    }else{
-                        log.warn("Survey unit quality tool responded non-2xx code {} and body {}",
-                                response.getStatusCode(), response.getBody());
-                    }
-                    }catch (IOException e){
-                        log.error("Error during Perret call request building : {}", e.toString());
-                    }
+                    sendProcessedIdsToQualityTool(surveyUnitModels);
                 }
 
                 //Remove processed ids from list
@@ -169,6 +158,22 @@ public DataProcessResult processRawData(String campaignName, List<String> interr
         return new DataProcessResult(dataCount, formattedDataCount);
     }
 
+    private void sendProcessedIdsToQualityTool(List<SurveyUnitModel> surveyUnitModels) {
+        try {
+            ResponseEntity<Object> response =
+                    surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels));
+
+        if (response.getStatusCode().is2xxSuccessful()) {
+            log.info("Successfully sent {} ids to quality tool", getProcessedIdsMap(surveyUnitModels).size());
+        }else{
+            log.warn("Survey unit quality tool responded non-2xx code {} and body {}",
+                    response.getStatusCode(), response.getBody());
+        }
+        }catch (IOException e){
+            log.error("Error during Perret call request building : {}", e.toString());
+        }
+    }
+
     private Map<String, Set<String>> getProcessedIdsMap(List<SurveyUnitModel> surveyUnitModels) {
         Map<String, Set<String>> processedInterrogationIdsPerQuestionnaire = new HashMap<>();
         surveyUnitModels.forEach(model ->
@@ -186,24 +191,12 @@ public List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawDat
         //For each possible data state (we receive COLLECTED or EDITED)
         for(DataState dataState : List.of(DataState.COLLECTED,DataState.EDITED)){
             for (LunaticJsonRawDataModel rawData : rawDataList) {
-                RawDataModelType rawDataModelType =
-                        rawData.data().containsKey("data") ?
-                                RawDataModelType.FILIERE :
-                                RawDataModelType.DEFAULT;
+                RawDataModelType rawDataModelType = getRawDataModelType(rawData);
 
                 //Get optional fields
-                String contextualId = null;
-                Boolean isCapturedIndirectly = null;
-                LocalDateTime validationDate = null;
-                try{
-                    contextualId = rawData.data().get("contextualId") == null ? null : rawData.data().get("contextualId").toString();
-                    isCapturedIndirectly = rawData.data().get("isCapturedIndirectly") == null ? null :
-                            Boolean.parseBoolean(rawData.data().get("isCapturedIndirectly").toString());
-                    validationDate = rawData.data().get("validationDate") == null ? null :
-                            LocalDateTime.parse(rawData.data().get("validationDate").toString());
-                }catch(Exception e){
-                    log.warn("Exception during optional fields parsing : %s".formatted(e.toString()));
-                }
+                String contextualId = getContextualId(rawData);
+                Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawData);
+                LocalDateTime validationDate = getValidationDate(rawData);
 
                 SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder()
                         .campaignId(rawData.campaignId())
@@ -225,17 +218,18 @@ public List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawDat
                 convertRawDataCollectedVariables(rawData, surveyUnitModel, dataState, rawDataModelType, variablesMap);
 
                 //External variables conversion into COLLECTED document
-                if(dataState.equals(DataState.COLLECTED)){
+                if(dataState == DataState.COLLECTED){
                     convertRawDataExternalVariables(rawData, surveyUnitModel, rawDataModelType, variablesMap);
                 }
 
-                if(surveyUnitModel.getCollectedVariables().isEmpty()
-                        && surveyUnitModel.getExternalVariables().isEmpty()
-                ){
+                boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty()
+                        && surveyUnitModel.getExternalVariables().isEmpty();
+
+                if(hasNoVariable){
                     if(surveyUnitModel.getState() == DataState.COLLECTED){
                         log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawData.interrogationId());
                     }
-                    continue;
+                    continue;// don't add suModel
                 }
                 surveyUnitModels.add(surveyUnitModel);
             }
@@ -244,6 +238,42 @@ public List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawDat
         return surveyUnitModels;
     }
 
+    private static RawDataModelType getRawDataModelType(LunaticJsonRawDataModel rawData) {
+        return rawData.data().containsKey("data") ?
+                RawDataModelType.FILIERE :
+                RawDataModelType.DEFAULT;
+    }
+
+    private static LocalDateTime getValidationDate(LunaticJsonRawDataModel rawData) {
+        try{
+            return rawData.data().get("validationDate") == null ? null :
+                LocalDateTime.parse(rawData.data().get("validationDate").toString());
+        }catch(Exception e){
+            log.warn("Exception when parsing validation date : {}}",e.toString());
+            return null;
+        }
+    }
+
+    private static Boolean getIsCapturedIndirectly(LunaticJsonRawDataModel rawData) {
+        try{
+            return rawData.data().get("isCapturedIndirectly") == null ? null :
+                    Boolean.parseBoolean(rawData.data().get("isCapturedIndirectly").toString());
+        }catch(Exception e){
+            log.warn("Exception when parsing isCapturedIndirectly : {}}",e.toString());
+            return Boolean.FALSE;
+        }
+
+    }
+
+    private static String getContextualId(LunaticJsonRawDataModel rawData) {
+        try{
+            return rawData.data().get("contextualId") == null ? null : rawData.data().get("contextualId").toString();
+        }catch(Exception e){
+            log.warn("Exception when parsing contextual id : {}}",e.toString());
+            return null;
+        }
+    }
+
     @Override
     public List<LunaticJsonRawDataUnprocessedDto> getUnprocessedDataIds() {
         List<LunaticJsonRawDataUnprocessedDto> dtos = new ArrayList<>();
@@ -333,22 +363,34 @@ private void convertRawDataCollectedVariables(
 
     }
 
-    private static void convertToCollectedVar(SurveyUnitModel dstSurveyUnitModel, DataState dataState, VariablesMap variablesMap, Map<String, Object> collectedMap) {
-        for(Map.Entry<String, Object> collectedVariable : collectedMap.entrySet()) {
+    private static void convertToCollectedVar(
+            SurveyUnitModel dstSurveyUnitModel,
+            DataState dataState,
+            VariablesMap variablesMap,
+            Map<String, Object> collectedMap
+    ) {
+        final String stateKey = dataState.toString();
+        final var dest = dstSurveyUnitModel.getCollectedVariables();
 
-            //Skip if collected variable does not have state
-            if(!JsonUtils.asMap(collectedVariable.getValue()).containsKey(dataState.toString())){
-                continue;
-            }
+        for (Map.Entry<String, Object> collectedVariable : collectedMap.entrySet()) {
+            // Map for this variable (COLLECTED/EDITED -> value)
+            Map<String, Object> states = JsonUtils.asMap(collectedVariable.getValue());
 
-            //Value
-            Object valuesForState = JsonUtils.asMap(collectedVariable.getValue()).get(dataState.toString());
-            if (valuesForState != null) {
-                if (valuesForState instanceof List<?>) {
-                    convertListVar(valuesForState, collectedVariable, variablesMap, dstSurveyUnitModel.getCollectedVariables());
-                    continue;
+            // nothing if no state
+            if (states.containsKey(stateKey)) {
+                Object value = states.get(stateKey);
+
+                // liste ?
+                if (value instanceof List<?>) {
+                    // on garde exactement ta signature existante
+                    convertListVar(value, collectedVariable, variablesMap, dest);
+                }
+
+                // scalaire non null ?
+                if (value != null && !(value instanceof List<?>)) {
+                    // idem: on garde convertOneVar(entry, String, ...)
+                    convertOneVar(collectedVariable, String.valueOf(value), variablesMap, 1, dest);
                 }
-                convertOneVar(collectedVariable, valuesForState.toString(), variablesMap, 1, dstSurveyUnitModel.getCollectedVariables());
             }
         }
     }

src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java

@@ -289,7 +289,7 @@ public List<InterrogationId> findDistinctInterrogationIdsByQuestionnaireIdAndDat
 
     /**
      * @author Adrien Marchal
-     * Calculations made to establish the data a worker will be responsible of, among the whole data to be processed.
+     * Calculations made to establish the data a worker will be responsible for, among the whole data to be processed.
      * (needed for distributed process / horizontal scaling)
      */
     @Override
@@ -544,18 +544,16 @@ private void extractVariables(SurveyUnitModel surveyUnitModel,
         for (VariableModel collectedVariable : surveyUnitModel.getCollectedVariables()) {
             VarIdScopeTuple loopIdTuple = new VarIdScopeTuple(collectedVariable.varId(), collectedVariable.scope(),
                     collectedVariable.iteration());
-            VariableDto variableDto = collectedVariableMap.get(loopIdTuple);
-
-            //Create variable into map if not exists
-            if (variableDto == null) {
-                variableDto = VariableDto.builder()
-                        .variableName(collectedVariable.varId())
-                        .scope(collectedVariable.scope())
-                        .iteration(collectedVariable.iteration())
-                        .variableStateDtoList(new ArrayList<>())
-                        .build();
-                collectedVariableMap.put(loopIdTuple, variableDto);
-            }
+
+            VariableDto variableDto = collectedVariableMap.computeIfAbsent(loopIdTuple, k ->
+                    VariableDto.builder()
+                            .variableName(collectedVariable.varId())
+                            .scope(collectedVariable.scope())
+                            .iteration(collectedVariable.iteration())
+                            .variableStateDtoList(new ArrayList<>())
+                            .build()
+            );
+
             //Extract variable state
             if (isMostRecentForSameState(surveyUnitModel, variableDto)) {
                 variableDto.getVariableStateDtoList().add(
@@ -577,18 +575,17 @@ private void extractVariables(SurveyUnitModel surveyUnitModel,
         }
         for(VariableModel externalVariable : surveyUnitModel.getExternalVariables()){
             VarIdScopeTuple loopIdTuple = new VarIdScopeTuple(externalVariable.varId(), externalVariable.scope(), externalVariable.iteration());
-            VariableDto variableDto = externalVariableMap.get(loopIdTuple);
-
-            //Create variable into map if not exists
-            if(variableDto == null){
-                variableDto = VariableDto.builder()
-                        .variableName(externalVariable.varId())
-                        .scope(externalVariable.scope())
-                        .iteration(externalVariable.iteration())
-                        .variableStateDtoList(new ArrayList<>())
-                        .build();
-                externalVariableMap.put(loopIdTuple, variableDto);
-            }
+
+            VariableDto variableDto = externalVariableMap.computeIfAbsent(loopIdTuple, k ->
+                    VariableDto.builder()
+                            .variableName(externalVariable.varId())
+                            .scope(externalVariable.scope())
+                            .iteration(externalVariable.iteration())
+                            .variableStateDtoList(new ArrayList<>())
+                            .build()
+            );
+
+
             //Extract variable state
             if(isMostRecentForSameState(surveyUnitModel, variableDto)){
                 variableDto.getVariableStateDtoList().add(

src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java

@@ -2,8 +2,8 @@
 
 import fr.insee.genesis.configuration.Config;
 import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort;
-import fr.insee.genesis.infrastructure.utils.http.OidcService;
 import fr.insee.genesis.infrastructure.utils.http.HttpUtils;
+import fr.insee.genesis.infrastructure.utils.http.OidcService;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -20,11 +20,14 @@
 @Slf4j
 public class SurveyQualityToolPerretAdapter implements SurveyUnitQualityToolPort {
 
-    @Autowired
-    Config config;
+    private final Config config;
+    private final OidcService oidcService;
 
     @Autowired
-    OidcService oidcService;
+    public SurveyQualityToolPerretAdapter( Config config,OidcService oidcService) {
+        this.config = config;
+        this.oidcService = oidcService;
+    }
 
 
     @Override