feat: new endpoint to list collection instrument ids containing unprocessed data

Author: loichenninger

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -215,6 +215,13 @@ public ResponseEntity<String> processRawResponses(
             return ResponseEntity.status(e.getStatus()).body(e.getMessage());
         }
     }
+    @Operation(summary = "Get the list of collection instruments containing unprocessed interrogations")
+    @GetMapping(path = "/raw-responses/unprocessed/collection-intrument-ids")
+    @PreAuthorize("hasRole('SCHEDULER')")
+    public ResponseEntity<List<String>> getUnprocessedCollectionInstrument(){
+        log.info("Try to get collection instruments containing unprocessed interrogations...");
+        return ResponseEntity.ok(rawResponseApiPort.getUnprocessedCollectionInstrumentIds());
+    }
 
     //GET unprocessed
     @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data")

src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java

@@ -15,5 +15,6 @@ public interface RawResponseApiPort {
     List<RawResponse> getRawResponses(String collectionInstrumentId, Mode mode, List<String> interrogationIdList);
     DataProcessResult processRawResponses(String collectionInstrumentId, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException;
     List<SurveyUnitModel> convertRawResponse(List<RawResponse> rawResponses, VariablesMap variablesMap);
+    List<String> getUnprocessedCollectionInstrumentIds();
     void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
 }

src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java

@@ -10,4 +10,5 @@ public interface RawResponsePersistencePort {
 
     List<RawResponse> findRawResponses(String collectionInstrumentId, Mode mode, List<String> interrogationIdList);
     void updateProcessDates(String collectionInstrumentId, Set<String> interrogationIds);
+    List<String> getUnprocessedCollectionIds();
 }

src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java

@@ -127,7 +127,7 @@ public DataProcessResult processRawResponses(String collectionInstrumentId, List
                 batchNumber++;
             }
         }
-        return new DataProcessResult(dataCount, formattedDataCount);
+        return new DataProcessResult(dataCount, formattedDataCount, List.of());
     }
 
     @Override
@@ -182,6 +182,11 @@ public List<SurveyUnitModel> convertRawResponse(List<RawResponse> rawResponses,
         //return List.of();
     }
 
+    @Override
+    public List<String> getUnprocessedCollectionInstrumentIds() {
+        return rawResponsePersistencePort.getUnprocessedCollectionIds();
+    }
+
     @Override
     public void updateProcessDates(List<SurveyUnitModel> surveyUnitModels) {
         Set<String> collectionInstrumentIds = new HashSet<>();

src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java

@@ -46,4 +46,9 @@ public void updateProcessDates(String collectionInstrumentId, Set<String> interr
                 , Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME
         );
     }
+
+    @Override
+    public List<String> getUnprocessedCollectionIds() {
+        return repository.findDistinctCollectionInstrumentIdByProcessDateIsNull();
+    }
 }

src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java

@@ -1,6 +1,7 @@
 package fr.insee.genesis.infrastructure.repository;
 
 import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument;
+import org.springframework.data.mongodb.repository.Aggregation;
 import org.springframework.data.mongodb.repository.MongoRepository;
 import org.springframework.data.mongodb.repository.Query;
 import org.springframework.stereotype.Repository;
@@ -12,4 +13,10 @@ public interface RawResponseRepository extends MongoRepository<RawResponseDocume
 
     @Query(value = "{ 'collectionInstrumentId' : ?0, 'mode' : ?1, 'interrogationId': {$in: ?2} }")
     List<RawResponseDocument> findByCollectionInstrumentIdAndModeAndInterrogationIdList(String questionnaireId, String mode, List<String> interrogationIdList);
+    @Aggregation(pipeline = {
+            "{ $match: { processDate: null } }",
+            "{ $group: { _id: '$collectionInstrumentId' } }",
+            "{ $project: { _id: 0, collectionInstrumentId: '$_id' } }"
+    })
+    List<String> findDistinctCollectionInstrumentIdByProcessDateIsNull();
 }

src/test/java/cucumber/functional_tests/RawDataDefinitions.java

@@ -120,6 +120,11 @@ public List<SurveyUnitModel> convertRawResponse(List<RawResponse> rawResponses,
             return List.of();
         }
 
+        @Override
+        public List<String> getUnprocessedCollectionInstrumentIds() {
+            return List.of();
+        }
+
         @Override
         public void updateProcessDates(List<SurveyUnitModel> surveyUnitModels) {
             // Do nothing - stub for test

src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java

@@ -93,6 +93,11 @@ public List<SurveyUnitModel> convertRawResponse(List<RawResponse> rawResponses,
             return List.of();
         }
 
+        @Override
+        public List<String> getUnprocessedCollectionInstrumentIds() {
+            return List.of();
+        }
+
         @Override
         public void updateProcessDates(List<SurveyUnitModel> surveyUnitModels) {
 
@@ -277,10 +282,10 @@ void processJsonRawDataV2Test(){
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotNull().isNotEmpty().hasSize(1);
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst()).isNotNull();
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCampaignId()).isEqualTo(questionnaireId);
-        Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getQuestionnaireId()).isNotNull().isEqualTo(questionnaireId);
+        Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectionInstrumentId()).isNotNull().isEqualTo(questionnaireId);
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getMode()).isNotNull().isEqualTo(Mode.WEB);
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getInterrogationId()).isEqualTo(interrogationId);
-        Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdUE()).isEqualTo(idUE);
+        Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getUsualSurveyUnitId()).isEqualTo(idUE);
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getFileDate()).isNotNull();
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getRecordDate()).isNotNull();
         Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables()).isNotNull().isNotEmpty().hasSize(1);