Skip to content

Commit a527a3e

Browse files
loichenningerloichenninger
authored
Merge pull request #145 from InseeFr/devEditedVariableWrite
Variable state edited writing service
2 parents b9b8d5b + e984033 commit a527a3e

File tree

18 files changed

+495
-23
lines changed

18 files changed

+495
-23
lines changed

‎.github/workflows/maven.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ jobs:
4242
cd ..
4343
4444
- name: Build Genesis with Maven
45-
run: mvn -B package --file pom.xml
45+
run: mvn clean package -B --file pom.xml
4646

4747
# Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive
4848
- name: Update dependency graph

‎src/main/java/fr/insee/genesis/Constants.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ public class Constants {
4747

4848

4949
// Data extraction parameters
50-
public static final int BATCH_SIZE = 100; //TODO Adapt to avoid OutOfMemoryException
51-
public static final String DIFFRENTIAL_DATA_FOLDER_NAME = "differential/data";
50+
public static final int BATCH_SIZE = 100; //Adapt to avoid OutOfMemoryException
51+
public static final String DIFFERENTIAL_DATA_FOLDER_NAME = "differential/data";
5252

5353
// Kraftwerk service path parameters
5454
public static final String KRAFTWERK_MAIN_ENDPOINT = "";

‎src/main/java/fr/insee/genesis/configuration/Config.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ public class Config {
2020
@Value("${fr.insee.genesis.sourcefolder.specifications}")
2121
private String specFolderSource;
2222

23+
@Value("${fr.insee.genesis.authentication}")
24+
private String authType;
25+
2326
@Value("${fr.insee.genesis.oidc.auth-server-url}")
2427
private String authServerUrl;
2528

‎src/main/java/fr/insee/genesis/controller/dto/SurveyUnitInputDto.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package fr.insee.genesis.controller.dto;
2+
3+
import fr.insee.genesis.domain.model.surveyunit.Mode;
4+
import lombok.Builder;
5+
import lombok.Data;
6+
7+
import java.util.List;
8+
9+
@Builder
10+
@Data
11+
public class SurveyUnitInputDto {
12+
13+
private String surveyUnitId;
14+
private String campaignId;
15+
private Mode mode;
16+
private String idQuestionnaire;
17+
private List<VariableInputDto> collectedVariables;
18+
19+
}

‎src/main/java/fr/insee/genesis/controller/dto/VariableInputDto.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package fr.insee.genesis.controller.dto;
2+
3+
import com.fasterxml.jackson.annotation.JsonProperty;
4+
import lombok.Builder;
5+
import lombok.Data;
6+
7+
import java.util.List;
8+
9+
@Builder
10+
@Data
11+
public class VariableInputDto {
12+
private String variableName;
13+
private String idLoop;
14+
15+
@JsonProperty("newVariableState")
16+
private VariableStateInputDto variableStateInputDto;
17+
}

‎src/main/java/fr/insee/genesis/controller/dto/VariableStateDto.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,6 @@ public class VariableStateDto {
1414
private boolean active;
1515
private String value;
1616

17-
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "dd/MM/yyyy hh:mm:ss.SSS")
17+
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "dd/MM/yyyy HH:mm:ss.SSS")
1818
private LocalDateTime date;
1919
}

‎src/main/java/fr/insee/genesis/controller/dto/VariableStateInputDto.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package fr.insee.genesis.controller.dto;
2+
3+
import fr.insee.genesis.domain.model.surveyunit.DataState;
4+
import lombok.Builder;
5+
import lombok.Data;
6+
7+
@Builder
8+
@Data
9+
public class VariableStateInputDto {
10+
11+
private DataState state;
12+
13+
private String value;
14+
15+
}

‎src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java‎

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@
99
import fr.insee.genesis.controller.adapter.LunaticXmlAdapter;
1010
import fr.insee.genesis.controller.dto.SurveyUnitDto;
1111
import fr.insee.genesis.controller.dto.SurveyUnitId;
12+
import fr.insee.genesis.controller.dto.SurveyUnitInputDto;
1213
import fr.insee.genesis.controller.dto.SurveyUnitSimplified;
1314
import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign;
1415
import fr.insee.genesis.controller.sources.xml.LunaticXmlDataParser;
1516
import fr.insee.genesis.controller.sources.xml.LunaticXmlDataSequentialParser;
1617
import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit;
18+
import fr.insee.genesis.controller.utils.AuthUtils;
1719
import fr.insee.genesis.controller.utils.ControllerUtils;
1820
import fr.insee.genesis.domain.model.surveyunit.CollectedVariable;
1921
import fr.insee.genesis.domain.model.surveyunit.Mode;
@@ -54,6 +56,7 @@
5456
import java.time.LocalDateTime;
5557
import java.util.ArrayList;
5658
import java.util.List;
59+
import java.util.stream.Collectors;
5760
import java.util.stream.Stream;
5861

5962
@RequestMapping(path = "/responses" )
@@ -73,20 +76,24 @@ public class ResponseController {
7376
private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
7477
private final FileUtils fileUtils;
7578
private final ControllerUtils controllerUtils;
79+
private final AuthUtils authUtils;
7680

7781

7882
public ResponseController(SurveyUnitApiPort surveyUnitService,
7983
SurveyUnitQualityService surveyUnitQualityService,
8084
LunaticXmlRawDataApiPort lunaticXmlRawDataApiPort,
8185
LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort,
8286
FileUtils fileUtils,
83-
ControllerUtils controllerUtils) {
87+
ControllerUtils controllerUtils,
88+
AuthUtils authUtils
89+
) {
8490
this.surveyUnitService = surveyUnitService;
8591
this.surveyUnitQualityService = surveyUnitQualityService;
8692
this.lunaticXmlRawDataApiPort = lunaticXmlRawDataApiPort;
8793
this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
8894
this.fileUtils = fileUtils;
8995
this.controllerUtils = controllerUtils;
96+
this.authUtils = authUtils;
9097
}
9198

9299
//SAVE
@@ -230,7 +237,7 @@ public ResponseEntity<Object> saveResponsesFromAllCampaignFolders(){
230237
try {
231238
List<Mode> modesList = controllerUtils.getModesList(campaignName, null); //modeSpecified null = all modes
232239
for (Mode currentMode : modesList) {
233-
processCampaignWithMode(campaignName, currentMode, errors, Constants.DIFFRENTIAL_DATA_FOLDER_NAME);
240+
processCampaignWithMode(campaignName, currentMode, errors, Constants.DIFFERENTIAL_DATA_FOLDER_NAME);
234241
}
235242
}catch (NoDataException nde){
236243
log.warn(nde.getMessage());
@@ -360,6 +367,62 @@ public ResponseEntity<List<SurveyUnitSimplified>> getLatestForUEList(@RequestPar
360367
return ResponseEntity.ok(results);
361368
}
362369

370+
@Operation(summary = "Save edited variables",
371+
description = "Save edited variables document into database")
372+
@PostMapping(path = "/save-edited")
373+
public ResponseEntity<Object> saveEditedVariables(
374+
@RequestBody SurveyUnitInputDto surveyUnitInputDto
375+
){
376+
//Parse metadata
377+
//Try to look for DDI first, if no DDI found looks for lunatic components
378+
List<GenesisError> errors = new ArrayList<>();
379+
VariablesMap variablesMap = readMetadatas(surveyUnitInputDto.getCampaignId(), surveyUnitInputDto.getMode(), errors, true);
380+
if(variablesMap == null){
381+
log.warn("Can't find DDI, trying with lunatic...");
382+
variablesMap = readMetadatas(surveyUnitInputDto.getCampaignId(), surveyUnitInputDto.getMode(), errors, false);
383+
if(variablesMap == null){
384+
return ResponseEntity.status(404).body(errors.getLast().getMessage());
385+
}
386+
}
387+
388+
//Check if input edited variables are in metadatas
389+
List<String> absentCollectedVariableNames =
390+
surveyUnitQualityService.checkVariablesPresentInMetadata(surveyUnitInputDto.getCollectedVariables(),
391+
variablesMap);
392+
// List<String> absentExternalVariableNames =
393+
// surveyUnitQualityService.checkVariablesPresentInMetadata(surveyUnitInputDto.getExternalVariables(),
394+
// variablesMap);
395+
if (!absentCollectedVariableNames.isEmpty()) {
396+
String absentVariables = String.join("\n", absentCollectedVariableNames);
397+
return ResponseEntity.badRequest().body(
398+
String.format("The following variables are absent in metadatas : %n%s", absentVariables)
399+
);
400+
}
401+
402+
//Fetch user identifier from OIDC token
403+
String userIdentifier = authUtils.getIDEP();
404+
405+
406+
//Create surveyUnitModel for each STATE received (Quality tool could send variables with another STATE than EDITED)
407+
List<SurveyUnitModel> surveyUnitModels;
408+
try{
409+
surveyUnitModels = surveyUnitService.parseEditedVariables(
410+
surveyUnitInputDto,
411+
userIdentifier,
412+
variablesMap
413+
);
414+
}catch (GenesisException e){
415+
return ResponseEntity.status(e.getStatus()).body(e.getMessage());
416+
}
417+
418+
//Check data with dataverifier (might create a FORCED document)
419+
surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap);
420+
421+
//Save documents
422+
surveyUnitService.saveSurveyUnits(surveyUnitModels);
423+
return ResponseEntity.ok(SUCCESS_MESSAGE);
424+
}
425+
363426
//Utilities
364427

365428
/**
@@ -579,10 +642,9 @@ private ResponseEntity<Object> processXmlFileSequentially(Path filepath, Mode mo
579642
su = parser.readNextSurveyUnit();
580643
}
581644

582-
log.info("Saved {} survey units updates", suCount);
645+
log.info("Saved {} survey units updates from Xml file {}", suCount, filepath.getFileName());
583646
}
584647

585-
log.info("File {} processed", filepath.getFileName());
586648
return ResponseEntity.ok().build();
587649
}
588650

‎src/main/java/fr/insee/genesis/controller/utils/AuthUtils.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package fr.insee.genesis.controller.utils;
2+
3+
import fr.insee.genesis.configuration.Config;
4+
import org.springframework.beans.factory.annotation.Autowired;
5+
import org.springframework.security.core.Authentication;
6+
import org.springframework.security.core.context.SecurityContextHolder;
7+
import org.springframework.stereotype.Component;
8+
9+
import java.util.Optional;
10+
11+
@Component
12+
public class AuthUtils {
13+
private final Config config;
14+
15+
@Autowired
16+
public AuthUtils(Config config){
17+
this.config = config;
18+
}
19+
20+
/**
21+
* Extract IDEP from OIDC token
22+
* @return the IDEP from the OIDC if the app uses OIDC auth, null otherwise
23+
*/
24+
public String getIDEP() {
25+
if ("OIDC".equals(config.getAuthType())) {
26+
return Optional.ofNullable(SecurityContextHolder.getContext().getAuthentication())
27+
.map(Authentication::getName)
28+
.map(name -> name.substring(name.lastIndexOf(":") + 1))
29+
.orElse(null);
30+
}
31+
return null;
32+
}
33+
}

‎src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ public class SurveyUnitModel {
2929
private List<CollectedVariable> collectedVariables;
3030
private List<Variable> externalVariables;
3131

32+
private String modifiedBy;
33+
3234
public SurveyUnitModel(String idUE, Mode mode) {
3335
this.idUE = idUE;
3436
this.mode = mode;

0 commit comments

Comments
 (0)