New endpoints for scheduled Perret batchs + contextual variables refactor

.gitignore

@@ -50,4 +50,6 @@ src/test/resources/OUT/*
 /src/test/resources/IN/WEB/SAMPLETEST/
 /src/test/resources/IN/WEB/TEST-EDITED-PREVIOUS/
 /src/test/resources/IN/WEB/TEST-EDITED-EXTERNAL/
+/src/test/resources/IN/WEB/TEST-CONTEXTUAL-PREVIOUS/
+/src/test/resources/IN/WEB/TEST-CONTEXTUAL-EXTERNAL/
 /src/main/resources/application-mbg.properties

CHANGELOG.md

@@ -1,4 +1,11 @@
 # Changelog
+## 1.9.0 [TODO]
+### Added
+- Authorizations for scheduled Perret calls
+- Get questionnaires by review activated or not endpoint
+- Save all edited previous/external files for one questionnaire endpoint
+### Changed
+- No more 400 if the previous/external file has no corresponding part (warn in log instead)
 
 ## 1.8.6 [2025-09-18]
 ### Added

src/main/java/fr/insee/genesis/Constants.java

@@ -18,8 +18,8 @@ public class Constants {
 
     public static final String MONGODB_SCHEDULE_COLLECTION_NAME = "schedules";
     public static final String MONGODB_CONTEXT_COLLECTION_NAME = "dataProcessingContexts";
-    public static final String MONGODB_EDITED_PREVIOUS_COLLECTION_NAME = "editedPrevious";
-    public static final String MONGODB_EDITED_EXTERNAL_COLLECTION_NAME = "editedExternal";
+    public static final String MONGODB_CONTEXTUAL_PREVIOUS_COLLECTION_NAME = "editedPrevious";
+    public static final String MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME = "editedExternal";
     public static final String LOOP_NAME_PREFIX = "BOUCLE";
     public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses";
     public static final String MONGODB_RESPONSE_RAW_COLLECTION_NAME = "lunaticjsondata";

src/main/java/fr/insee/genesis/controller/rest/responses/EditedResponseController.java → src/main/java/fr/insee/genesis/controller/rest/responses/ContextualVariableController.java

@@ -2,9 +2,9 @@
 
 import fr.insee.genesis.configuration.Config;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
-import fr.insee.genesis.domain.ports.api.EditedExternalResponseApiPort;
-import fr.insee.genesis.domain.ports.api.EditedPreviousResponseApiPort;
-import fr.insee.genesis.domain.ports.api.EditedResponseApiPort;
+import fr.insee.genesis.domain.ports.api.ContextualExternalVariableApiPort;
+import fr.insee.genesis.domain.ports.api.ContextualPreviousVariableApiPort;
+import fr.insee.genesis.domain.ports.api.ContextualVariableApiPort;
 import fr.insee.genesis.exceptions.GenesisException;
 import fr.insee.genesis.infrastructure.utils.FileUtils;
 import io.swagger.v3.oas.annotations.Operation;
@@ -19,39 +19,51 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.file.Path;
 
-@RequestMapping(path = "/edited")
+@RequestMapping(path = "/contextual-variables")
 @Controller
 @Slf4j
 @AllArgsConstructor
-public class EditedResponseController {
+public class ContextualVariableController {
 
-    private final EditedPreviousResponseApiPort editedPreviousResponseApiPort;
-    private final EditedExternalResponseApiPort editedExternalResponseApiPort;
-    private final EditedResponseApiPort editedResponseApiPort;
+    private final ContextualPreviousVariableApiPort contextualPreviousVariableApiPort;
+    private final ContextualExternalVariableApiPort contextualExternalVariableApiPort;
+    private final ContextualVariableApiPort contextualVariableApiPort;
     private final Config config;
 
-    @Operation(summary = "Get edited variables (edited and previous)")
+    @Operation(summary = "Get contextual variables (external and previous)")
     @GetMapping(path = "/")
-    @PreAuthorize("hasRole('USER_PLATINE')")
-    public ResponseEntity<Object> getEditedResponses(
+    @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')")
+    public ResponseEntity<Object> getContextualVariables(
             @RequestParam("questionnaireId") String questionnaireId,
             @RequestParam("interrogationId") String interrogationId
     ){
         return ResponseEntity.ok().body(
-                editedResponseApiPort.getEditedResponse(questionnaireId, interrogationId)
+                contextualVariableApiPort.getContextualVariable(questionnaireId, interrogationId)
         );
     }
 
-    @Operation(summary = "Add edited previous json file")
+    @Operation(summary = "Save all contextual variables json files (external and previous)")
+    @PostMapping(path = "/json")
+    @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')")
+    public ResponseEntity<Object> saveContextualVariables(
+            @RequestParam("questionnaireId") String questionnaireId
+    ){
+        try {
+            FileUtils fileUtils = new FileUtils(config);
+            int fileCount = contextualVariableApiPort.saveContextualVariableFiles(questionnaireId, fileUtils);
+            return ResponseEntity.ok("%d file(s) processed for questionnaire %s !".formatted(fileCount, questionnaireId));
+        }catch (GenesisException ge){
+            return ResponseEntity.status(HttpStatusCode.valueOf(ge.getStatus())).body(ge.getMessage());
+        }
+    }
+
+    @Operation(summary = "Add contextual previous json file")
     @PostMapping(path = "previous/json")
     @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER','USER_BACK_OFFICE')")
-    public ResponseEntity<Object> readEditedPreviousJson(
+    public ResponseEntity<Object> readContextualPreviousJson(
             @RequestParam("questionnaireId") String questionnaireId,
             @RequestParam("mode") Mode mode,
             @RequestParam(value = "sourceState", required = false) String sourceState,
@@ -67,18 +79,18 @@ public ResponseEntity<Object> readEditedPreviousJson(
             if (!jsonFileName.toLowerCase().endsWith(".json")) {
                 throw new GenesisException(400, "File must be a JSON file !");
             }
-            readEditedPreviousFile(questionnaireId.toUpperCase(), sourceState, filePath);
-            moveFiles(questionnaireId, mode, fileUtils, filePath);
-            return ResponseEntity.ok("Edited previous variable file %s saved !".formatted(filePath));
+            contextualPreviousVariableApiPort.readContextualPreviousFile(questionnaireId.toUpperCase(), sourceState, filePath);
+            moveFile(questionnaireId, mode, fileUtils, filePath);
+            return ResponseEntity.ok("Contextual previous variable file %s saved !".formatted(filePath));
         }catch (GenesisException ge){
             return ResponseEntity.status(HttpStatusCode.valueOf(ge.getStatus())).body(ge.getMessage());
         }
     }
 
-    @Operation(summary = "Add edited external json file")
+    @Operation(summary = "Add contextual external json file")
     @PostMapping(path = "/external/json")
     @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER','USER_BACK_OFFICE')")
-    public ResponseEntity<Object> readEditedExternalJson(
+    public ResponseEntity<Object> readContextualExternalJson(
             @RequestParam("questionnaireId") String questionnaireId,
             @RequestParam("mode") Mode mode,
             @RequestParam(value = "jsonFileName") String jsonFileName
@@ -93,35 +105,15 @@ public ResponseEntity<Object> readEditedExternalJson(
             if (!jsonFileName.toLowerCase().endsWith(".json")) {
                 throw new GenesisException(400, "File must be a JSON file !");
             }
-            readEditedExternalFile(questionnaireId.toUpperCase(), filePath);
-            moveFiles(questionnaireId, mode, fileUtils, filePath);
-            return ResponseEntity.ok("Edited external variable file %s saved !".formatted(filePath));
+            contextualExternalVariableApiPort.readContextualExternalFile(questionnaireId, filePath);
+            moveFile(questionnaireId, mode, fileUtils, filePath);
+            return ResponseEntity.ok("Contextual external variable file %s saved !".formatted(filePath));
         }catch (GenesisException ge){
             return ResponseEntity.status(HttpStatusCode.valueOf(ge.getStatus())).body(ge.getMessage());
         }
     }
 
-    private void readEditedPreviousFile(String questionnaireId, String sourceState, String filePath) throws GenesisException {
-        try (InputStream inputStream = new FileInputStream(filePath)) {
-            editedPreviousResponseApiPort.readEditedPreviousFile(inputStream, questionnaireId, sourceState);
-        } catch (FileNotFoundException e) {
-            throw new GenesisException(404, "File %s not found".formatted(filePath));
-        } catch (IOException e) {
-            throw new GenesisException(500, e.toString());
-        }
-    }
-
-    private void readEditedExternalFile(String questionnaireId, String filePath) throws GenesisException {
-        try (InputStream inputStream = new FileInputStream(filePath)) {
-            editedExternalResponseApiPort.readEditedExternalFile(inputStream, questionnaireId);
-        } catch (FileNotFoundException e) {
-            throw new GenesisException(404, "File %s not found".formatted(filePath));
-        } catch (IOException e) {
-            throw new GenesisException(500, e.toString());
-        }
-    }
-
-    private static void moveFiles(String questionnaireId, Mode mode, FileUtils fileUtils, String filePath) throws GenesisException {
+    private static void moveFile(String questionnaireId, Mode mode, FileUtils fileUtils, String filePath) throws GenesisException {
         try {
             fileUtils.moveFiles(Path.of(filePath), fileUtils.getDoneFolder(questionnaireId, mode.getFolder()));
         } catch (IOException e) {

src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java

@@ -2,12 +2,14 @@
 
 import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign;
 import fr.insee.genesis.controller.rest.CommonApiResponse;
+import fr.insee.genesis.domain.ports.api.DataProcessingContextApiPort;
 import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
 import fr.insee.genesis.exceptions.GenesisException;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
@@ -22,21 +24,36 @@
 public class QuestionnaireController implements CommonApiResponse {
 
     private final SurveyUnitApiPort surveyUnitService;
+    private final DataProcessingContextApiPort dataProcessingContextService;
 
 
-    public QuestionnaireController(SurveyUnitApiPort surveyUnitService) {
+    public QuestionnaireController(
+            SurveyUnitApiPort surveyUnitService,
+            DataProcessingContextApiPort dataProcessingContextService
+    ) {
         this.surveyUnitService = surveyUnitService;
+        this.dataProcessingContextService = dataProcessingContextService;
     }
 
 
 
-    @Operation(summary = "List questionnaires in database")
+    @Operation(summary = "List questionnaires from responses database")
     @GetMapping(path = "/")
     public ResponseEntity<Set<String>> getQuestionnaires() {
         Set<String> questionnaires = surveyUnitService.findDistinctQuestionnaireIds();
         return ResponseEntity.ok(questionnaires);
     }
 
+    @Operation(summary = "List questionnaires in database that have a context with a specific withReview")
+    @GetMapping(path = "/with-review")
+    @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')")
+    public ResponseEntity<List<String>> getQuestionnairesWithReview(
+            @RequestParam(value = "withReview") boolean withReview
+    ) {
+        List<String> questionnaires = dataProcessingContextService.getPartitionIds(withReview);
+        return ResponseEntity.ok(questionnaires);
+    }
+
 
     @Operation(summary = "List questionnaires in database with their campaigns")
     @GetMapping(path = "/with-campaigns")

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -208,7 +208,7 @@ public ResponseEntity<List<SurveyUnitModel>> findResponsesByInterrogationAndQues
     @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database with the latest value for each available state of every variable")
     @GetMapping(path = "/by-ue-and-questionnaire/latest-states",
                 produces = MediaType.APPLICATION_JSON_VALUE)
-    @PreAuthorize("hasRole('USER_PLATINE')")
+    @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')")
     public ResponseEntity<Object> findResponsesByInterrogationAndQuestionnaireLatestStates(
             @RequestParam("interrogationId") String interrogationId,
             @RequestParam("questionnaireId") String questionnaireId) throws GenesisException {

src/main/java/fr/insee/genesis/domain/model/editedresponse/EditedExternalResponseModel.java → src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualExternalVariableModel.java

@@ -1,4 +1,4 @@
-package fr.insee.genesis.domain.model.editedresponse;
+package fr.insee.genesis.domain.model.contextualvariable;
 
 import lombok.Builder;
 import lombok.Data;
@@ -7,7 +7,7 @@
 
 @Builder
 @Data
-public class EditedExternalResponseModel {
+public class ContextualExternalVariableModel {
     String id;
     String questionnaireId;
     String interrogationId;

src/main/java/fr/insee/genesis/domain/model/editedresponse/EditedPreviousResponseModel.java → src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualPreviousVariableModel.java

@@ -1,4 +1,4 @@
-package fr.insee.genesis.domain.model.editedresponse;
+package fr.insee.genesis.domain.model.contextualvariable;
 
 import lombok.Builder;
 import lombok.Data;
@@ -7,7 +7,7 @@
 
 @Builder
 @Data
-public class EditedPreviousResponseModel {
+public class ContextualPreviousVariableModel {
     String id;
     String questionnaireId;
     String interrogationId;

src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualVariableModel.java

@@ -0,0 +1,13 @@
+package fr.insee.genesis.domain.model.contextualvariable;
+
+import fr.insee.genesis.controller.dto.VariableQualityToolDto;
+import lombok.Builder;
+
+import java.util.List;
+
+@Builder
+public record ContextualVariableModel(
+        String interrogationId,
+        List<VariableQualityToolDto> contextualPrevious,
+        List<VariableQualityToolDto> contextualExternal
+){}

src/main/java/fr/insee/genesis/domain/ports/api/ContextualExternalVariableApiPort.java

@@ -0,0 +1,10 @@
+package fr.insee.genesis.domain.ports.api;
+
+import fr.insee.genesis.domain.model.contextualvariable.ContextualExternalVariableModel;
+import fr.insee.genesis.exceptions.GenesisException;
+
+public interface ContextualExternalVariableApiPort {
+    boolean readContextualExternalFile(String questionnaireId, String filePath) throws GenesisException;
+
+    ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId);
+}

src/main/java/fr/insee/genesis/domain/ports/api/ContextualPreviousVariableApiPort.java

@@ -0,0 +1,12 @@
+package fr.insee.genesis.domain.ports.api;
+
+import fr.insee.genesis.domain.model.contextualvariable.ContextualPreviousVariableModel;
+import fr.insee.genesis.exceptions.GenesisException;
+
+import java.io.InputStream;
+
+public interface ContextualPreviousVariableApiPort {
+    boolean readContextualPreviousFile(String questionnaireId, String sourceState, String filePath) throws GenesisException;
+
+    ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId);
+}

src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java

@@ -0,0 +1,10 @@
+package fr.insee.genesis.domain.ports.api;
+
+import fr.insee.genesis.domain.model.contextualvariable.ContextualVariableModel;
+import fr.insee.genesis.exceptions.GenesisException;
+import fr.insee.genesis.infrastructure.utils.FileUtils;
+
+public interface ContextualVariableApiPort {
+    ContextualVariableModel getContextualVariable(String questionnaireId, String interrogationId);
+    int saveContextualVariableFiles(String questionnaireId, FileUtils fileUtils) throws GenesisException;
+}

src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java

@@ -32,5 +32,5 @@ void saveKraftwerkExecutionSchedule(String partitionId,
 
     DataProcessingContextModel getContext(String interrogationId) throws GenesisException;
     DataProcessingContextModel getContextByPartitionId(String partitionId) throws GenesisException;
-
+    List<String> getPartitionIds(boolean withReview);
 }

src/main/java/fr/insee/genesis/domain/ports/spi/ContextualExternalVariablePersistancePort.java

@@ -0,0 +1,14 @@
+package fr.insee.genesis.domain.ports.spi;
+
+import fr.insee.genesis.domain.model.contextualvariable.ContextualExternalVariableModel;
+
+import java.util.List;
+
+public interface ContextualExternalVariablePersistancePort {
+    void backup(String questionnaireId);
+    void deleteBackup(String questionnaireId);
+    void restoreBackup(String questionnaireId);
+    void saveAll(List<ContextualExternalVariableModel> contextualPreviousVariableModelList);
+    void delete(String questionnaireId);
+    ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId);
+}

src/main/java/fr/insee/genesis/domain/ports/spi/ContextualPreviousVariablePersistancePort.java

@@ -0,0 +1,14 @@
+package fr.insee.genesis.domain.ports.spi;
+
+import fr.insee.genesis.domain.model.contextualvariable.ContextualPreviousVariableModel;
+
+import java.util.List;
+
+public interface ContextualPreviousVariablePersistancePort {
+    void backup(String questionnaireId);
+    void deleteBackup(String questionnaireId);
+    void restoreBackup(String questionnaireId);
+    void saveAll(List<ContextualPreviousVariableModel> contextualPreviousVariableModelList);
+    void delete(String questionnaireId);
+    ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId);
+}