feat: filter questionnaire with isReview

loichenninger · loichenninger · commit 0e2404abaf3a · 2025-06-25T15:25:23.000+02:00
diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java
@@ -190,7 +190,7 @@ public ResponseEntity<String> processJsonRawData(
     }
 
     @Operation(summary = "Get processed data ids from last n hours (default 24h)")
-    @PostMapping(path = "/lunatic-json/processed/ids")
+    @GetMapping(path = "/lunatic-json/processed/ids")
     @PreAuthorize("hasRole('SCHEDULER')")
     public ResponseEntity<Map<String, List<String>>> getProcessedDataIdsSinceHours(
             @RequestParam("questionnaireId") String questionnaireId,
diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/GroupedInterrogation.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/GroupedInterrogation.java
@@ -14,5 +14,7 @@
 public class GroupedInterrogation {
 
     private String questionnaireId;
+    private String partitionOrCampaignId;
     private List<String> interrogationIds;
+
 }
diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java
@@ -32,4 +32,6 @@ void saveKraftwerkExecutionSchedule(String partitionId,
 
     DataProcessingContextModel getContext(String interrogationId) throws GenesisException;
     DataProcessingContextModel getContextByPartitionId(String partitionId) throws GenesisException;
+
+    List<DataProcessingContextModel> getPartitionIdWithReview(List<String> partitionIds);
 }
diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java
@@ -9,6 +9,8 @@
 public interface DataProcessingContextPersistancePort {
     DataProcessingContextDocument findByPartitionId(String partitionId);
 
+    List<DataProcessingContextModel> findByPartitionIds(List<String> partitionIds);
+
     void save(DataProcessingContextDocument dataProcessingContextDocument);
 
     void saveAll(List<DataProcessingContextDocument> dataProcessingContextDocuments);
diff --git a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java
@@ -172,4 +172,13 @@ public DataProcessingContextModel getContextByPartitionId(String partitionId){
                 dataProcessingContextPersistancePort.findByPartitionId(partitionId)
         );
     }
+
+    @Override
+    public List<DataProcessingContextModel> getPartitionIdWithReview (List<String> partitionIds){
+        List<DataProcessingContextModel> contexts =  dataProcessingContextPersistancePort.findByPartitionIds(partitionIds);
+        if (contexts.isEmpty()){
+            return List.of();
+        }
+        return contexts.stream().filter(DataProcessingContextModel::isWithReview).toList();
+    }
 }
diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java
@@ -5,6 +5,7 @@
 import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
 import fr.insee.genesis.controller.services.MetadataService;
 import fr.insee.genesis.controller.utils.ControllerUtils;
+import fr.insee.genesis.domain.model.context.DataProcessingContextModel;
 import fr.insee.genesis.domain.model.surveyunit.DataState;
 import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation;
 import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
@@ -15,6 +16,7 @@
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
 import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
+import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort;
 import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort;
 import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
 import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService;
@@ -49,15 +51,18 @@ public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort {
 
     @Qualifier("lunaticJsonMongoAdapterNew")
     private final LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort;
+    @Qualifier("dataProcessingContextMongoAdapter")
+    private final DataProcessingContextPersistancePort dataProcessingContextPersistancePort;
 
     @Autowired
-    public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort, ControllerUtils controllerUtils, MetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils) {
+    public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort, ControllerUtils controllerUtils, MetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils, DataProcessingContextPersistancePort dataProcessingContextPersistancePort) {
         this.controllerUtils = controllerUtils;
         this.metadataService = metadataService;
         this.surveyUnitService = surveyUnitService;
         this.surveyUnitQualityService = surveyUnitQualityService;
         this.fileUtils = fileUtils;
         this.lunaticJsonRawDataPersistencePort = lunaticJsonRawDataNewPersistencePort;
+        this.dataProcessingContextPersistancePort = dataProcessingContextPersistancePort;
     }
 
     @Override
@@ -344,7 +349,11 @@ public long countResponsesByQuestionnaireId(String campaignId) {
     @Override
     public Map<String, List<String>> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since) {
         List<GroupedInterrogation> idsByQuestionnaire = lunaticJsonRawDataPersistencePort.findProcessedIdsGroupedByQuestionnaireSince(since);
-        return idsByQuestionnaire.stream().collect(Collectors.toMap(
+        List<String> partitionIds = idsByQuestionnaire.stream().map(GroupedInterrogation::getPartitionOrCampaignId).toList();
+        List<DataProcessingContextModel> contexts = dataProcessingContextPersistancePort.findByPartitionIds(partitionIds);
+        List<String> partitionIdsWithReview = contexts.stream().filter(DataProcessingContextModel::isWithReview).map(DataProcessingContextModel::getPartitionId).toList();
+        return idsByQuestionnaire.stream().filter(groupedInterrogation -> partitionIdsWithReview.contains(groupedInterrogation.getPartitionOrCampaignId()))
+                .collect(Collectors.toMap(
                 GroupedInterrogation::getQuestionnaireId,
                 GroupedInterrogation::getInterrogationIds
         ));
diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java
@@ -5,6 +5,7 @@
 import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule;
 import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort;
 import fr.insee.genesis.infrastructure.document.context.DataProcessingContextDocument;
+import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper;
 import fr.insee.genesis.infrastructure.repository.DataProcessingContextMongoDBRepository;
 import fr.insee.genesis.infrastructure.utils.context.ContextDedupUtils;
 import lombok.extern.slf4j.Slf4j;
@@ -41,6 +42,13 @@ public DataProcessingContextDocument findByPartitionId(String partitionId) {
         return ContextDedupUtils.deduplicateContexts(partitionId, existingDocuments);
     }
 
+    @Override
+    public List<DataProcessingContextModel> findByPartitionIds(List<String> partitionIds){
+        List<DataProcessingContextDocument> existingDocuments =
+                dataProcessingContextMongoDBRepository.findByPartitionIdList(partitionIds);
+        return DataProcessingContextMapper.INSTANCE.listDocumentToListModel(ContextDedupUtils.deduplicateContexts(existingDocuments));
+    }
+
     @Override
     public void save(DataProcessingContextDocument dataProcessingContextDocument) {
         dataProcessingContextMongoDBRepository.save(dataProcessingContextDocument);
diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java
@@ -78,7 +78,7 @@ public long countResponsesByQuestionnaireId(String questionnaireId) {
 
     @Override
     public List<GroupedInterrogation> findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since){
-        return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(repository.aggregateRawGroupedByQuestionnaire(since));
+        return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(repository.aggregateRawGrouped(since));
     }
 
 
diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/GroupedInterrogationDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/GroupedInterrogationDocument.java
@@ -8,5 +8,6 @@
 public class GroupedInterrogationDocument {
 
     private String questionnaireId;
+    private String partitionOrCampaignId;
     private List<String> interrogationIds;
 }
diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java
@@ -23,9 +23,20 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository<LunaticJso
     long countByQuestionnaireId(String questionnaireId);
     @Aggregation(pipeline = {
             "{ '$match': { 'processDate': { '$gte': ?0 } } }",
-            "{ '$group': { '_id': '$questionnaireId', 'interrogationIds': { '$addToSet': '$interrogationId' } } }",
-            "{ '$project': { 'questionnaireId': '$_id', 'interrogationIds': 1, '_id': 0 } }"
+            "{ '$group': { " +
+                    "'_id': { " +
+                    "'questionnaireId': '$questionnaireId', " +
+                    "'partitionOrCampaignId': { '$ifNull': ['$partitionId', '$campaignId'] } " +
+                    "}, " +
+                    "'interrogationIds': { '$addToSet': '$interrogationId' } " +
+                    "} }",
+            "{ '$project': { " +
+                    "'questionnaireId': '$_id.questionnaireId', " +
+                    "'partitionOrCampaignId': '$_id.partitionOrCampaignId', " +
+                    "'interrogationIds': 1, " +
+                    "'_id': 0 " +
+                    "} }"
     })
-    List<GroupedInterrogationDocument> aggregateRawGroupedByQuestionnaire(LocalDateTime since);
+    List<GroupedInterrogationDocument> aggregateRawGrouped(LocalDateTime since);
 
 }
diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java
@@ -16,6 +16,7 @@
 import fr.insee.genesis.domain.utils.JsonUtils;
 import fr.insee.genesis.infrastructure.utils.FileUtils;
 import fr.insee.genesis.stubs.ConfigStub;
+import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub;
 import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub;
 import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
 import io.cucumber.java.Before;
@@ -60,12 +61,13 @@ public class RawDataDefinitions {
     LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub();
     MetadataService metadataService = new MetadataService();
     SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub();
+    DataProcessingContextPersistancePortStub contextStub = new DataProcessingContextPersistancePortStub();
     Config config = new ConfigStub();
     FileUtils fileUtils = new FileUtils(config);
     SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils);
     ControllerUtils controllerUtils = new ControllerUtils(fileUtils);
     SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService();
-    LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils);
+    LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils,contextStub);
     RawResponseController rawResponseController = new RawResponseController(
             lunaticJsonRawDataService
     );
diff --git a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java
@@ -11,12 +11,14 @@
 import fr.insee.genesis.domain.model.surveyunit.VariableModel;
 import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
 import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
+import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort;
 import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService;
 import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
 import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService;
 import fr.insee.genesis.domain.service.volumetry.VolumetryLogService;
 import fr.insee.genesis.infrastructure.utils.FileUtils;
 import fr.insee.genesis.stubs.ConfigStub;
+import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub;
 import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub;
 import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
 import org.assertj.core.api.Assertions;
@@ -43,6 +45,7 @@ class UtilsControllerTest {
     static UtilsController utilsControllerStatic;
     static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub;
     static LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistencePort;
+    static DataProcessingContextPersistancePort contextStub = new DataProcessingContextPersistancePortStub();
     static List<InterrogationId> interrogationIdList;
     static FileUtils fileUtils = new FileUtils(new ConfigStub());
     static ControllerUtils controllerUtils = new ControllerUtils(fileUtils);
@@ -58,7 +61,7 @@ static void init() {
         surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub();
         lunaticJsonRawDataPersistencePort = new LunaticJsonRawDataPersistanceStub();
         SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils);
-        LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistencePort,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils);
+        LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistencePort,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils,contextStub);
 
         utilsControllerStatic = new UtilsController(
                 surveyUnitApiPort
diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java
@@ -14,6 +14,7 @@
 import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument;
 import fr.insee.genesis.infrastructure.utils.FileUtils;
 import fr.insee.genesis.stubs.ConfigStub;
+import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub;
 import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub;
 import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
 import org.assertj.core.api.Assertions;
@@ -34,7 +35,8 @@ class RawResponseControllerTest {
             new MetadataService(),
             new SurveyUnitService(surveyUnitPersistencePortStub, new MetadataService(), fileUtils),
             new SurveyUnitQualityService(),
-            fileUtils);
+            fileUtils,
+            new DataProcessingContextPersistancePortStub());
     private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort);
 
 
diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java
@@ -14,6 +14,7 @@
 import fr.insee.genesis.infrastructure.mappers.LunaticJsonRawDataDocumentMapper;
 import fr.insee.genesis.infrastructure.utils.FileUtils;
 import fr.insee.genesis.stubs.ConfigStub;
+import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub;
 import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub;
 import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
 import org.assertj.core.api.Assertions;
@@ -37,7 +38,8 @@ class LunaticJsonRawDataServiceTest {
     SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils);
     SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService();
 
-    LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils);
+    DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub();
+    LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils,dataProcessingContextPersistancePortStub);
 
     @Test
     void saveDataTest_valid_only_collected_array() throws Exception {
diff --git a/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java b/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java
@@ -27,6 +27,11 @@ public DataProcessingContextDocument findByPartitionId(String partitionId) {
         );
     }
 
+    @Override
+    public List<DataProcessingContextModel> findByPartitionIds(List<String> partitionIds) {
+        return List.of();
+    }
+
     @Override
     public void save(DataProcessingContextDocument dataProcessingContextDocument) {
         mongoStub.removeIf(existingDoc -> existingDoc.getPartitionId().equals(
diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java
@@ -58,7 +58,7 @@ public long countByQuestionnaireId(String questionnaireId) {
     }
 
     @Override
-    public List<GroupedInterrogationDocument> aggregateRawGroupedByQuestionnaire(LocalDateTime since) {
+    public List<GroupedInterrogationDocument> aggregateRawGrouped(LocalDateTime since) {
         return List.of();
     }
 

Original file line number	Diff line number	Diff line change
`@@ -190,7 +190,7 @@ public ResponseEntity<String> processJsonRawData(`
`190`	`190`	`}`
`191`	`191`
`192`	`192`	`@Operation(summary = "Get processed data ids from last n hours (default 24h)")`
`193`		`- @PostMapping(path = "/lunatic-json/processed/ids")`
	`193`	`+ @GetMapping(path = "/lunatic-json/processed/ids")`
`194`	`194`	`@PreAuthorize("hasRole('SCHEDULER')")`
`195`	`195`	`public ResponseEntity<Map<String, List<String>>> getProcessedDataIdsSinceHours(`
`196`	`196`	`@RequestParam("questionnaireId") String questionnaireId,`
Original file line number	Diff line number	Diff line change
`@@ -14,5 +14,7 @@`
`14`	`14`	`public class GroupedInterrogation {`
`15`	`15`
`16`	`16`	`private String questionnaireId;`
	`17`	`+ private String partitionOrCampaignId;`
`17`	`18`	`private List<String> interrogationIds;`
	`19`	`+`
`18`	`20`	`}`
Original file line number	Diff line number	Diff line change
`@@ -32,4 +32,6 @@ void saveKraftwerkExecutionSchedule(String partitionId,`
`32`	`32`
`33`	`33`	`DataProcessingContextModel getContext(String interrogationId) throws GenesisException;`
`34`	`34`	`DataProcessingContextModel getContextByPartitionId(String partitionId) throws GenesisException;`
	`35`	`+`
	`36`	`+ List<DataProcessingContextModel> getPartitionIdWithReview(List<String> partitionIds);`
`35`	`37`	`}`
Original file line number	Diff line number	Diff line change
`@@ -172,4 +172,13 @@ public DataProcessingContextModel getContextByPartitionId(String partitionId){`
`172`	`172`	`dataProcessingContextPersistancePort.findByPartitionId(partitionId)`
`173`	`173`	`);`
`174`	`174`	`}`
	`175`	`+`
	`176`	`+ @Override`
	`177`	`+ public List<DataProcessingContextModel> getPartitionIdWithReview (List<String> partitionIds){`
	`178`	`+ List<DataProcessingContextModel> contexts = dataProcessingContextPersistancePort.findByPartitionIds(partitionIds);`
	`179`	`+ if (contexts.isEmpty()){`
	`180`	`+ return List.of();`
	`181`	`+ }`
	`182`	`+ return contexts.stream().filter(DataProcessingContextModel::isWithReview).toList();`
	`183`	`+ }`
`175`	`184`	`}`
Original file line number	Diff line number	Diff line change
`@@ -78,7 +78,7 @@ public long countResponsesByQuestionnaireId(String questionnaireId) {`
`78`	`78`
`79`	`79`	`@Override`
`80`	`80`	`public List<GroupedInterrogation> findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since){`
`81`		`- return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(repository.aggregateRawGroupedByQuestionnaire(since));`
	`81`	`+ return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(repository.aggregateRawGrouped(since));`
`82`	`82`	`}`
`83`	`83`
`84`	`84`
Original file line number	Diff line number	Diff line change
`@@ -8,5 +8,6 @@`
`8`	`8`	`public class GroupedInterrogationDocument {`
`9`	`9`
`10`	`10`	`private String questionnaireId;`
	`11`	`+ private String partitionOrCampaignId;`
`11`	`12`	`private List<String> interrogationIds;`
`12`	`13`	`}`