Variable state edited writing service

.github/workflows/maven.yml

@@ -42,7 +42,7 @@ jobs:
         cd ..
 
     - name: Build Genesis with Maven
-      run: mvn -B package --file pom.xml
+      run: mvn clean package -B --file pom.xml
 
     # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive
     - name: Update dependency graph

src/main/java/fr/insee/genesis/Constants.java

@@ -47,8 +47,8 @@ public class Constants {
 
 
     // Data extraction parameters
-    public static final int BATCH_SIZE = 100; //TODO Adapt to avoid OutOfMemoryException
-    public static final String DIFFRENTIAL_DATA_FOLDER_NAME = "differential/data";
+    public static final int BATCH_SIZE = 100; //Adapt to avoid OutOfMemoryException
+    public static final String DIFFERENTIAL_DATA_FOLDER_NAME = "differential/data";
 
     // Kraftwerk service path parameters
     public static final String KRAFTWERK_MAIN_ENDPOINT = "";

src/main/java/fr/insee/genesis/configuration/Config.java

@@ -20,6 +20,9 @@ public class Config {
 	@Value("${fr.insee.genesis.sourcefolder.specifications}")
 	private String specFolderSource;
 
+	@Value("${fr.insee.genesis.authentication}")
+	private String authType;
+
 	@Value("${fr.insee.genesis.oidc.auth-server-url}")
 	private String authServerUrl;
 

src/main/java/fr/insee/genesis/controller/dto/SurveyUnitInputDto.java

@@ -0,0 +1,19 @@
+package fr.insee.genesis.controller.dto;
+
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Builder
+@Data
+public class SurveyUnitInputDto {
+
+    private String surveyUnitId;
+    private String campaignId;
+    private Mode mode;
+    private String idQuestionnaire;
+    private List<VariableInputDto> collectedVariables;
+
+}

src/main/java/fr/insee/genesis/controller/dto/VariableInputDto.java

@@ -0,0 +1,17 @@
+package fr.insee.genesis.controller.dto;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+@Builder
+@Data
+public class VariableInputDto {
+    private String variableName;
+    private String idLoop;
+
+    @JsonProperty("newVariableState")
+    private VariableStateInputDto variableStateInputDto;
+}

src/main/java/fr/insee/genesis/controller/dto/VariableStateDto.java

@@ -14,6 +14,6 @@ public class VariableStateDto {
     private boolean active;
     private String value;
 
-    @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "dd/MM/yyyy hh:mm:ss.SSS")
+    @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "dd/MM/yyyy HH:mm:ss.SSS")
     private LocalDateTime date;
 }

src/main/java/fr/insee/genesis/controller/dto/VariableStateInputDto.java

@@ -0,0 +1,15 @@
+package fr.insee.genesis.controller.dto;
+
+import fr.insee.genesis.domain.model.surveyunit.DataState;
+import lombok.Builder;
+import lombok.Data;
+
+@Builder
+@Data
+public class VariableStateInputDto {
+
+    private DataState state;
+
+    private String value;
+
+}

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -9,11 +9,13 @@
 import fr.insee.genesis.controller.adapter.LunaticXmlAdapter;
 import fr.insee.genesis.controller.dto.SurveyUnitDto;
 import fr.insee.genesis.controller.dto.SurveyUnitId;
+import fr.insee.genesis.controller.dto.SurveyUnitInputDto;
 import fr.insee.genesis.controller.dto.SurveyUnitSimplified;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlDataParser;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlDataSequentialParser;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit;
+import fr.insee.genesis.controller.utils.AuthUtils;
 import fr.insee.genesis.controller.utils.ControllerUtils;
 import fr.insee.genesis.domain.model.surveyunit.CollectedVariable;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
@@ -54,6 +56,7 @@
 import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 @RequestMapping(path = "/responses" )
@@ -73,20 +76,24 @@ public class ResponseController {
     private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
     private final FileUtils fileUtils;
     private final ControllerUtils controllerUtils;
+    private final AuthUtils authUtils;
 
 
     public ResponseController(SurveyUnitApiPort surveyUnitService,
                               SurveyUnitQualityService surveyUnitQualityService,
                               LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort,
                               LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort,
                               FileUtils fileUtils,
-                              ControllerUtils controllerUtils) {
+                              ControllerUtils controllerUtils,
+                              AuthUtils authUtils
+    ) {
         this.surveyUnitService = surveyUnitService;
         this.surveyUnitQualityService = surveyUnitQualityService;
         this.lunaticXmlRawDataApiPort = lunaticXmlRawDataApiPort;
         this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
         this.fileUtils = fileUtils;
         this.controllerUtils = controllerUtils;
+        this.authUtils = authUtils;
     }
 
     //SAVE
@@ -230,7 +237,7 @@ public ResponseEntity<Object> saveResponsesFromAllCampaignFolders(){
             try {
                 List<Mode> modesList = controllerUtils.getModesList(campaignName, null); //modeSpecified null = all modes
                 for (Mode currentMode : modesList) {
-                    processCampaignWithMode(campaignName, currentMode, errors, Constants.DIFFRENTIAL_DATA_FOLDER_NAME);
+                    processCampaignWithMode(campaignName, currentMode, errors, Constants.DIFFERENTIAL_DATA_FOLDER_NAME);
                 }
             }catch (NoDataException nde){
                 log.warn(nde.getMessage());
@@ -360,6 +367,62 @@ public ResponseEntity<List<SurveyUnitSimplified>> getLatestForUEList(@RequestPar
         return ResponseEntity.ok(results);
     }
 
+    @Operation(summary = "Save edited variables",
+            description = "Save edited variables document into database")
+    @PostMapping(path = "/save-edited")
+    public ResponseEntity<Object> saveEditedVariables(
+            @RequestBody SurveyUnitInputDto surveyUnitInputDto
+    ){
+        //Parse metadata
+        //Try to look for DDI first, if no DDI found looks for lunatic components
+        List<GenesisError> errors = new ArrayList<>();
+        VariablesMap variablesMap = readMetadatas(surveyUnitInputDto.getCampaignId(), surveyUnitInputDto.getMode(), errors, true);
+        if(variablesMap == null){
+            log.warn("Can't find DDI, trying with lunatic...");
+            variablesMap = readMetadatas(surveyUnitInputDto.getCampaignId(), surveyUnitInputDto.getMode(), errors, false);
+            if(variablesMap == null){
+                return ResponseEntity.status(404).body(errors.getLast().getMessage());
+            }
+        }
+
+        //Check if input edited variables are in metadatas
+        List<String> absentCollectedVariableNames =
+                surveyUnitQualityService.checkVariablesPresentInMetadata(surveyUnitInputDto.getCollectedVariables(),
+                variablesMap);
+//        List<String> absentExternalVariableNames =
+//                surveyUnitQualityService.checkVariablesPresentInMetadata(surveyUnitInputDto.getExternalVariables(),
+//                        variablesMap);
+        if (!absentCollectedVariableNames.isEmpty()) {
+            String absentVariables = String.join("\n", absentCollectedVariableNames);
+            return ResponseEntity.badRequest().body(
+                    String.format("The following variables are absent in metadatas : %n%s", absentVariables)
+            );
+        }
+
+        //Fetch user identifier from OIDC token
+        String userIdentifier = authUtils.getIDEP();
+
+
+        //Create surveyUnitModel for each STATE received (Quality tool could send variables with another STATE than EDITED)
+        List<SurveyUnitModel> surveyUnitModels;
+        try{
+            surveyUnitModels = surveyUnitService.parseEditedVariables(
+                    surveyUnitInputDto,
+                    userIdentifier,
+                    variablesMap
+            );
+        }catch (GenesisException e){
+           return ResponseEntity.status(e.getStatus()).body(e.getMessage());
+        }
+
+        //Check data with dataverifier (might create a FORCED document)
+        surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap);
+
+        //Save documents
+        surveyUnitService.saveSurveyUnits(surveyUnitModels);
+        return ResponseEntity.ok(SUCCESS_MESSAGE);
+    }
+
     //Utilities
 
     /**
@@ -579,10 +642,9 @@ private ResponseEntity<Object> processXmlFileSequentially(Path filepath, Mode mo
                 su = parser.readNextSurveyUnit();
             }
 
-            log.info("Saved {} survey units updates", suCount);
+            log.info("Saved {} survey units updates from Xml file {}", suCount,  filepath.getFileName());
         }
 
-        log.info("File {} processed", filepath.getFileName());
         return ResponseEntity.ok().build();
     }
 

src/main/java/fr/insee/genesis/controller/utils/AuthUtils.java

@@ -0,0 +1,33 @@
+package fr.insee.genesis.controller.utils;
+
+import fr.insee.genesis.configuration.Config;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Component;
+
+import java.util.Optional;
+
+@Component
+public class AuthUtils {
+    private final Config config;
+
+    @Autowired
+    public AuthUtils(Config config){
+        this.config = config;
+    }
+
+    /**
+     * Extract IDEP from OIDC token
+     * @return the IDEP from the OIDC if the app uses OIDC auth, null otherwise
+     */
+    public String getIDEP() {
+        if ("OIDC".equals(config.getAuthType())) {
+            return Optional.ofNullable(SecurityContextHolder.getContext().getAuthentication())
+                    .map(Authentication::getName)
+                    .map(name -> name.substring(name.lastIndexOf(":") + 1))
+                    .orElse(null);
+        }
+        return null;
+    }
+}

src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java

@@ -29,6 +29,8 @@ public class SurveyUnitModel {
 	private List<CollectedVariable> collectedVariables;
 	private List<Variable> externalVariables;
 
+	private String modifiedBy;
+
 	public SurveyUnitModel(String idUE, Mode mode) {
 		this.idUE = idUE;
 		this.mode = mode;

src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java

@@ -1,11 +1,14 @@
 package fr.insee.genesis.domain.ports.api;
 
+import fr.insee.bpm.metadata.model.VariablesMap;
 import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire;
 import fr.insee.genesis.controller.dto.SurveyUnitDto;
+import fr.insee.genesis.controller.dto.SurveyUnitInputDto;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign;
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
 import fr.insee.genesis.controller.dto.SurveyUnitId;
+import fr.insee.genesis.exceptions.GenesisException;
 
 import java.util.List;
 import java.util.Set;
@@ -48,4 +51,8 @@ public interface SurveyUnitApiPort {
     List<CampaignWithQuestionnaire> findCampaignsWithQuestionnaires();
 
     List<QuestionnaireWithCampaign> findQuestionnairesWithCampaigns();
-}
+
+    List<SurveyUnitModel> parseEditedVariables(SurveyUnitInputDto surveyUnitInputDto,
+                                         String userIdentifier,
+                                         VariablesMap variablesMap) throws GenesisException;
+}

src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitQualityService.java

@@ -1,18 +1,39 @@
 package fr.insee.genesis.domain.service.surveyunit;
 
+import fr.insee.genesis.controller.dto.VariableDto;
+import fr.insee.genesis.controller.dto.VariableInputDto;
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
 import fr.insee.genesis.domain.utils.DataVerifier;
 import fr.insee.bpm.metadata.model.VariablesMap;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
 import java.util.List;
 
 /**
  * This service is used to ensure data quality processes
  */
 @Service
 public class SurveyUnitQualityService {
-    public void verifySurveyUnits(List<SurveyUnitModel> suDtos, VariablesMap variablesMap) {
-        DataVerifier.verifySurveyUnits(suDtos,variablesMap);
+    public void verifySurveyUnits(List<SurveyUnitModel> surveyUnitModels, VariablesMap variablesMap) {
+        DataVerifier.verifySurveyUnits(surveyUnitModels,variablesMap);
+    }
+
+    /**
+     * Checks
+     * @param variableInputDtos input variables to check
+     * @param variablesMap BPM Questionnaire metadata to use for checking
+     * @return A list of variables that are <strong>absent</strong> in metadata
+     */
+    public List<String> checkVariablesPresentInMetadata(List<VariableInputDto> variableInputDtos, VariablesMap variablesMap) {
+        List<String> absentVariableNames = new ArrayList<>();
+
+        for(VariableInputDto variableInputDto : variableInputDtos){
+            if(!variablesMap.hasVariable(variableInputDto.getVariableName())){
+                absentVariableNames.add(variableInputDto.getVariableName());
+            }
+        }
+
+        return absentVariableNames;
     }
 }