diff --git a/CHANGELOG.md b/CHANGELOG.md index 321444e3..97dabe2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 1.13.0 [TODO] +### Changed +- New raw data process endpoint + ## 1.12.2 [2025-11-06] ### Fixed - Scientific notation for Doubles during raw data processing diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index ed41b205..33df8669 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -161,6 +161,7 @@ public ResponseEntity getJsonRawData( @Operation(summary = "Process raw data of a campaign") @PostMapping(path = "/lunatic-json/process") @PreAuthorize("hasRole('SCHEDULER')") + @Deprecated(since = "1.13.0") public ResponseEntity processJsonRawData( @RequestParam("campaignName") String campaignName, @RequestParam("questionnaireId") String questionnaireId, @@ -180,6 +181,24 @@ public ResponseEntity processJsonRawData( } } + @Operation(summary = "Process raw data of a questionnaire") + @PostMapping(path = "/{collectionInstrumentId}/process") + @PreAuthorize("hasRole('SCHEDULER')") + public ResponseEntity processJsonRawData( + @PathVariable String collectionInstrumentId + ) { + log.info("Try to process raw JSON datas for questionnaire {}",collectionInstrumentId); + try { + DataProcessResult result = lunaticJsonRawDataApiPort.processRawData(collectionInstrumentId); + return result.formattedDataCount() == 0 ? + ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount())) + : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification" + .formatted(result.dataCount(), result.formattedDataCount())); + } catch (GenesisException e) { + return ResponseEntity.status(e.getStatus()).body(e.getMessage()); + } + } + @Operation(summary = "Get processed data ids from last n hours (default 24h)") @GetMapping(path = "/lunatic-json/processed/ids") @PreAuthorize("hasRole('ADMIN')") diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index c228535b..bf0a6dea 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -466,7 +466,7 @@ private void processCampaignWithMode(String campaignName, Mode mode, String root } //Create context if not exist - if(contextService.getContextByPartitionId(campaignName) == null){ + if(contextService.getContextByCollectionInstrumentId(campaignName) == null){ contextService.saveContext(campaignName, false); } diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java index c0739b3b..b8191dd0 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java @@ -1,4 +1,8 @@ package fr.insee.genesis.domain.model.surveyunit.rawdata; -public record DataProcessResult(int dataCount, int formattedDataCount) { +import fr.insee.genesis.exceptions.GenesisError; + +import java.util.List; + +public record DataProcessResult(int dataCount, int formattedDataCount, List errors) { } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java index f26ad129..077cdc83 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java @@ -31,7 +31,7 @@ void saveKraftwerkExecutionSchedule(String partitionId, long countSchedules(); DataProcessingContextModel getContext(String interrogationId) throws GenesisException; - DataProcessingContextModel getContextByPartitionId(String partitionId) throws GenesisException; + DataProcessingContextModel getContextByCollectionInstrumentId(String partitionId) throws GenesisException; List getPartitionIds(boolean withReview); /** diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java index f52b66b2..e8bad50c 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java @@ -28,6 +28,10 @@ public interface LunaticJsonRawDataApiPort { long countResponsesByQuestionnaireId(String campaignId); Page findRawDataByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable); + @Deprecated(since = "1.13.0") DataProcessResult processRawData(String campaignName, List interrogationIdList, List errors) throws GenesisException; + + DataProcessResult processRawData(String collectionInstrumentId) throws GenesisException; + Map> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java index 62ab1b81..399fefde 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java @@ -22,4 +22,5 @@ public interface LunaticJsonRawDataPersistencePort { long countResponsesByQuestionnaireId(String questionnaireId); List findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since); List findUnprocessedIds(); + Set findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId); } diff --git a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java index 9688a84a..a0c80a8c 100644 --- a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java +++ b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java @@ -168,7 +168,7 @@ public DataProcessingContextModel getContext(String interrogationId) throws Gene } @Override - public DataProcessingContextModel getContextByPartitionId(String partitionId){ + public DataProcessingContextModel getContextByCollectionInstrumentId(String partitionId){ return DataProcessingContextMapper.INSTANCE.documentToModel( dataProcessingContextPersistancePort.findByPartitionId(partitionId) ); diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 6489a586..2134f67a 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -100,22 +100,16 @@ public List getRawData(String campaignName, Mode mode, } @Override + @Deprecated(since = "1.13.0") public DataProcessResult processRawData(String campaignName, List interrogationIdList, List errors) throws GenesisException { int dataCount=0; int formattedDataCount=0; DataProcessingContextModel dataProcessingContext = - dataProcessingContextService.getContextByPartitionId(campaignName); + dataProcessingContextService.getContextByCollectionInstrumentId(campaignName); List modesList = controllerUtils.getModesList(campaignName, null); for (Mode mode : modesList) { //Load and save metadata into database, throw exception if none - VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(campaignName, campaignName, mode, fileUtils, - errors).getVariables(); - if (variablesMap == null) { - throw new GenesisException(400, - "Error during metadata parsing for mode %s :%n%s" - .formatted(mode, errors.getLast().getMessage()) - ); - } + VariablesMap variablesMap = getVariablesMap(campaignName, mode, errors); int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize()); int batchNumber = 1; List interrogationIdListForMode = new ArrayList<>(interrogationIdList); @@ -156,7 +150,85 @@ public DataProcessResult processRawData(String campaignName, List interr batchNumber++; } } - return new DataProcessResult(dataCount, formattedDataCount); + return new DataProcessResult(dataCount, formattedDataCount, errors); + } + + @Override + public DataProcessResult processRawData(String questionnaireId) throws GenesisException { + int dataCount=0; + int formattedDataCount=0; + DataProcessingContextModel dataProcessingContext = + dataProcessingContextService.getContextByCollectionInstrumentId(questionnaireId); + List errors = new ArrayList<>(); + + List modesList = controllerUtils.getModesList(questionnaireId, null); + for (Mode mode : modesList) { + //Load and save metadata into database, throw exception if none + VariablesMap variablesMap = getVariablesMap(questionnaireId, mode, errors); + Set interrogationIds = + lunaticJsonRawDataPersistencePort.findUnprocessedInterrogationIdsByCollectionInstrumentId(questionnaireId); + + int totalBatchs = Math.ceilDiv(interrogationIds.size() , config.getRawDataProcessingBatchSize()); + int batchNumber = 1; + List interrogationIdListForMode = new ArrayList<>(interrogationIds); + while(!interrogationIdListForMode.isEmpty()){ + log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); + + int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); + List surveyUnitModels = getConvertedSurveyUnits( + questionnaireId, + mode, + interrogationIdListForMode, + maxIndex, + variablesMap + ); + + //Save converted data + surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); + surveyUnitService.saveSurveyUnits(surveyUnitModels); + + //Update process dates + updateProcessDates(surveyUnitModels); + + //Increment data count + dataCount += surveyUnitModels.size(); + formattedDataCount += surveyUnitModels.stream() + .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) + .toList() + .size(); + + //Send processed ids grouped by questionnaire (if review activated) + if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { + sendProcessedIdsToQualityTool(surveyUnitModels); + } + + //Remove processed ids from list + interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); + batchNumber++; + } + } + return new DataProcessResult(dataCount, formattedDataCount, errors); + } + + private VariablesMap getVariablesMap(String questionnaireId, Mode mode, List errors) throws GenesisException { + VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(questionnaireId, questionnaireId, mode, fileUtils, + errors).getVariables(); + if (variablesMap == null) { + throw new GenesisException(400, + "Error during metadata parsing for mode %s :%n%s" + .formatted(mode, errors.getLast().getMessage()) + ); + } + return variablesMap; + } + + private List getConvertedSurveyUnits(String questionnaireId, Mode mode, List interrogationIdListForMode, int maxIndex, VariablesMap variablesMap) { + List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); + List rawData = getRawData(questionnaireId, mode, interrogationIdToProcess); + return convertRawData( + rawData, + variablesMap + ); } private void sendProcessedIdsToQualityTool(List surveyUnitModels) { diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java index bd05024d..9dc510bf 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java @@ -6,6 +6,7 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; +import fr.insee.genesis.infrastructure.document.surveyunit.GroupedInterrogationDocument; import fr.insee.genesis.infrastructure.mappers.GroupedInterrogationDocumentMapper; import fr.insee.genesis.infrastructure.mappers.LunaticJsonRawDataDocumentMapper; import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository; @@ -96,4 +97,14 @@ public List findUnprocessedIds() { return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(repository.aggregateRawGroupedWithNullProcessDate()); } + @Override + public Set findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { + Set interrogationIds = new HashSet<>(); + + repository.aggregateRawGroupedWithNullProcessDate(collectionInstrumentId).forEach( + groupedInterrogationDocument -> interrogationIds.addAll(groupedInterrogationDocument.getInterrogationIds()) + ); + + return interrogationIds; + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java index cdfda072..c2fe4c2b 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java @@ -5,6 +5,7 @@ import lombok.Data; import org.bson.types.ObjectId; import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; import java.time.LocalDateTime; @@ -23,6 +24,7 @@ public DataProcessingContextDocument(String partitionId, @Id private ObjectId id; + @Indexed private String partitionId; //ex Survey Name, campaignId private LocalDateTime lastExecution; private List kraftwerkExecutionScheduleList; diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java index d04fbebd..a07d3f91 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java @@ -1,6 +1,7 @@ package fr.insee.genesis.infrastructure.document.lunaticmodel; import lombok.Builder; +import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; import java.time.LocalDateTime; @@ -9,6 +10,7 @@ @Builder @Document(collection = "lunaticmodels") public record LunaticModelDocument ( + @Indexed String questionnaireId, Map lunaticModel, LocalDateTime recordDate diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java index 7c9d0528..5f99ef28 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java @@ -3,11 +3,13 @@ import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.genesis.domain.model.surveyunit.Mode; import org.springframework.data.mongodb.core.index.CompoundIndex; +import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; @CompoundIndex(name = "questionnaireId_1_mode_1", def = "{'questionnaireId': 1, 'mode': 1}") @Document(collection = "questionnaireMetadatas") public record QuestionnaireMetadataDocument( + @Indexed String questionnaireId, Mode mode, MetadataModel metadataModel diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonRawDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonRawDataDocument.java index c4221fdd..88961d0b 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonRawDataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonRawDataDocument.java @@ -17,6 +17,7 @@ public record LunaticJsonRawDataDocument( @Id ObjectId id, String campaignId, + @Indexed String questionnaireId, String interrogationId, String idUE, diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java index 61f60f2a..58486bae 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java @@ -63,4 +63,22 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository aggregateRawGroupedWithNullProcessDate(); + + @Aggregation(pipeline = { + "{ '$match': {'questionnaireId': ?0 ,'processDate': null } }", + "{ '$group': { " + + "'_id': { " + + "'questionnaireId': '$questionnaireId', " + + "'partitionOrCampaignId': { '$ifNull': ['$partitionId', '$campaignId'] } " + + "}, " + + "'interrogationIds': { '$addToSet': '$interrogationId' } " + + "} }", + "{ '$project': { " + + "'questionnaireId': '$_id.questionnaireId', " + + "'partitionOrCampaignId': '$_id.partitionOrCampaignId', " + + "'interrogationIds': 1, " + + "'_id': 0 " + + "} }" + }) + List aggregateRawGroupedWithNullProcessDate(String questionnaireId); } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 26d93a9f..7d3cc2b9 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -23,6 +23,7 @@ import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; +import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.data.web.PagedModel; @@ -35,6 +36,7 @@ import java.util.List; import java.util.Map; +@Slf4j class RawResponseControllerTest { private final FileUtils fileUtils = new FileUtils(new ConfigStub()); private final LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub(); @@ -198,6 +200,66 @@ void processJsonRawDataTest(){ .contains(interrogationId); } + @Test + void processJsonRawDataV2Test(){ + //GIVEN + lunaticJsonRawDataPersistanceStub.getMongoStub().clear(); + surveyUnitPersistencePortStub.getMongoStub().clear(); + surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().clear(); + String questionnaireId = "SAMPLETEST-PARADATA-v2"; + String interrogationId = "testinterrogationId1"; + String idUE = "testIdUE1"; + String varName = "AVIS_MAIL"; + String varValue = "TEST"; + addJsonRawDataDocumentToStub(questionnaireId, questionnaireId, interrogationId, idUE, null, LocalDateTime.now(),varName + , varValue); + + dataProcessingContextPersistancePortStub.getMongoStub().add( + DataProcessingContextMapper.INSTANCE.modelToDocument( + DataProcessingContextModel.builder() + .partitionId(questionnaireId) + .kraftwerkExecutionScheduleList(new ArrayList<>()) + .withReview(true) + .build() + ) + ); + + //WHEN + ResponseEntity response = rawResponseController.processJsonRawData(questionnaireId); + + + //THEN + if(!response.getStatusCode().is2xxSuccessful()){ + log.error(response.getBody().toString()); + } + Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); + + //Genesis model survey unit created successfully + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotNull().isNotEmpty().hasSize(1); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst()).isNotNull(); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCampaignId()).isEqualTo(questionnaireId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getQuestionnaireId()).isNotNull().isEqualTo(questionnaireId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getMode()).isNotNull().isEqualTo(Mode.WEB); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getInterrogationId()).isEqualTo(interrogationId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdUE()).isEqualTo(idUE); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getFileDate()).isNotNull(); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getRecordDate()).isNotNull(); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables()).isNotNull().isNotEmpty().hasSize(1); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables().getFirst()).isNotNull(); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables().getFirst().varId()).isNotNull().isEqualTo(varName); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables().getFirst().value()).isNotNull().isEqualTo(varValue); + + //Process date check + Assertions.assertThat(lunaticJsonRawDataPersistanceStub.getMongoStub().getFirst().processDate()).isNotNull(); + + //Perret call check + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps()) + .hasSize(1); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst()).containsKey(questionnaireId); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst().get(questionnaireId)) + .contains(interrogationId); + } + @Test void getRawResponsesFromJsonBody() { diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java index f57c0a64..7a0017a1 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java @@ -14,6 +14,7 @@ import java.time.LocalDateTime; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -136,6 +137,17 @@ public List aggregateRawGroupedWithNullProcessDate return result; } + @Override + public List aggregateRawGroupedWithNullProcessDate(String questionnaireId) { + GroupedInterrogationDocument groupedInterrogationDocument = new GroupedInterrogationDocument(); + groupedInterrogationDocument.setQuestionnaireId(questionnaireId); + groupedInterrogationDocument.setPartitionOrCampaignId(questionnaireId); + groupedInterrogationDocument.setInterrogationIds(new ArrayList<>()); + documents.stream().filter(doc -> doc.questionnaireId().equals(questionnaireId) && doc.processDate() == null).toList() + .forEach(doc -> groupedInterrogationDocument.getInterrogationIds().add(doc.interrogationId())); + return Collections.singletonList(groupedInterrogationDocument); + } + // Implémentations vides requises par MongoRepository @Override public S save(S entity) { return null; } @Override public Optional findById(String s) { return Optional.empty(); } diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java index 5fb6ec68..c7244925 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java @@ -176,4 +176,15 @@ public List findUnprocessedIds() { return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(result); } + @Override + public Set findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { + List unprocessedDocuments = + mongoStub.stream().filter( + lunaticJsonDataDocument -> lunaticJsonDataDocument.processDate() == null + && lunaticJsonDataDocument.questionnaireId().equals(collectionInstrumentId) + ).toList(); + Set interrogationIds = new HashSet<>(); + unprocessedDocuments.forEach(doc -> interrogationIds.add(doc.interrogationId())); + return interrogationIds; + } }