refactor: collectionInstrumentId + process raw data

Author: alexisszmundy

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -182,14 +182,14 @@ public ResponseEntity<String> processJsonRawData(
     }
 
     @Operation(summary = "Process raw data of a questionnaire")
-    @PostMapping(path = "/lunatic-json/{questionnaireId}/process")
+    @PostMapping(path = "/{collectionInstrumentId}/process")
     @PreAuthorize("hasRole('SCHEDULER')")
     public ResponseEntity<String> processJsonRawData(
-            @PathVariable String questionnaireId
+            @PathVariable String collectionInstrumentId
     ) {
-        log.info("Try to process raw JSON datas for questionnaire {}",questionnaireId);
+        log.info("Try to process raw JSON datas for questionnaire {}",collectionInstrumentId);
         try {
-            DataProcessResult result = lunaticJsonRawDataApiPort.processRawData(questionnaireId);
+            DataProcessResult result = lunaticJsonRawDataApiPort.processRawData(collectionInstrumentId);
             return result.formattedDataCount() == 0 ?
                     ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount()))
                     : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification"

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -466,7 +466,7 @@ private void processCampaignWithMode(String campaignName, Mode mode, String root
         }
 
         //Create context if not exist
-        if(contextService.getContextByPartitionId(campaignName) == null){
+        if(contextService.getContextByCollectionInstrumentId(campaignName) == null){
             contextService.saveContext(campaignName, false);
         }
 

src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java

@@ -31,7 +31,7 @@ void saveKraftwerkExecutionSchedule(String partitionId,
     long countSchedules();
 
     DataProcessingContextModel getContext(String interrogationId) throws GenesisException;
-    DataProcessingContextModel getContextByPartitionId(String partitionId) throws GenesisException;
+    DataProcessingContextModel getContextByCollectionInstrumentId(String partitionId) throws GenesisException;
     List<String> getPartitionIds(boolean withReview);
 
     /**

src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java

@@ -31,7 +31,7 @@ public interface LunaticJsonRawDataApiPort {
     @Deprecated(since = "1.13.0")
     DataProcessResult processRawData(String campaignName, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException;
 
-    DataProcessResult processRawData(String questionnaireId) throws GenesisException;
+    DataProcessResult processRawData(String collectionInstrumentId) throws GenesisException;
 
     Map<String, List<String>> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since);
 }

src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java

@@ -22,5 +22,5 @@ public interface LunaticJsonRawDataPersistencePort {
     long countResponsesByQuestionnaireId(String questionnaireId);
     List<GroupedInterrogation> findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since);
     List<GroupedInterrogation> findUnprocessedIds();
-    Set<String> findUnprocessedInterrogationIdsByQuestionnaire(String questionnaireId);
+    Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId);
 }

src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java

@@ -168,7 +168,7 @@ public DataProcessingContextModel getContext(String interrogationId) throws Gene
     }
 
     @Override
-    public DataProcessingContextModel getContextByPartitionId(String partitionId){
+    public DataProcessingContextModel getContextByCollectionInstrumentId(String partitionId){
         return DataProcessingContextMapper.INSTANCE.documentToModel(
                 dataProcessingContextPersistancePort.findByPartitionId(partitionId)
         );

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java

@@ -105,18 +105,11 @@ public DataProcessResult processRawData(String campaignName, List<String> interr
         int dataCount=0;
         int formattedDataCount=0;
         DataProcessingContextModel dataProcessingContext =
-                dataProcessingContextService.getContextByPartitionId(campaignName);
+                dataProcessingContextService.getContextByCollectionInstrumentId(campaignName);
         List<Mode> modesList = controllerUtils.getModesList(campaignName, null);
         for (Mode mode : modesList) {
             //Load and save metadata into database, throw exception if none
-            VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(campaignName, campaignName, mode, fileUtils,
-                    errors).getVariables();
-            if (variablesMap == null) {
-                throw new GenesisException(400,
-                        "Error during metadata parsing for mode %s :%n%s"
-                                .formatted(mode, errors.getLast().getMessage())
-                );
-            }
+            VariablesMap variablesMap = getVariablesMap(campaignName, mode, errors);
             int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize());
             int batchNumber = 1;
             List<String> interrogationIdListForMode = new ArrayList<>(interrogationIdList);
@@ -165,36 +158,28 @@ public DataProcessResult processRawData(String questionnaireId) throws GenesisEx
         int dataCount=0;
         int formattedDataCount=0;
         DataProcessingContextModel dataProcessingContext =
-                dataProcessingContextService.getContextByPartitionId(questionnaireId);
+                dataProcessingContextService.getContextByCollectionInstrumentId(questionnaireId);
         List<GenesisError> errors = new ArrayList<>();
 
         List<Mode> modesList = controllerUtils.getModesList(questionnaireId, null);
         for (Mode mode : modesList) {
             //Load and save metadata into database, throw exception if none
-            VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(questionnaireId, questionnaireId, mode, fileUtils,
-                    errors).getVariables();
-            if (variablesMap == null) {
-                throw new GenesisException(400,
-                        "Error during metadata parsing for mode %s :%n%s"
-                                .formatted(mode, errors.getLast().getMessage())
-                );
-            }
+            VariablesMap variablesMap = getVariablesMap(questionnaireId, mode, errors);
             Set<String> interrogationIds =
-                    lunaticJsonRawDataPersistencePort.findUnprocessedInterrogationIdsByQuestionnaire(questionnaireId);
-
+                    lunaticJsonRawDataPersistencePort.findUnprocessedInterrogationIdsByCollectionInstrumentId(questionnaireId);
 
             int totalBatchs = Math.ceilDiv(interrogationIds.size() , config.getRawDataProcessingBatchSize());
             int batchNumber = 1;
             List<String> interrogationIdListForMode = new ArrayList<>(interrogationIds);
             while(!interrogationIdListForMode.isEmpty()){
                 log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs);
-                int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize());
-                List<String> interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex);
 
-                List<LunaticJsonRawDataModel> rawData = getRawData(questionnaireId, mode, interrogationIdToProcess);
-
-                List<SurveyUnitModel> surveyUnitModels = convertRawData(
-                        rawData,
+                int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize());
+                List<SurveyUnitModel> surveyUnitModels = getConvertedSurveyUnits(
+                        questionnaireId,
+                        mode,
+                        interrogationIdListForMode,
+                        maxIndex,
                         variablesMap
                 );
 
@@ -219,13 +204,33 @@ public DataProcessResult processRawData(String questionnaireId) throws GenesisEx
 
                 //Remove processed ids from list
                 interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size());
-
                 batchNumber++;
             }
         }
         return new DataProcessResult(dataCount, formattedDataCount, errors);
     }
 
+    private VariablesMap getVariablesMap(String questionnaireId, Mode mode, List<GenesisError> errors) throws GenesisException {
+        VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(questionnaireId, questionnaireId, mode, fileUtils,
+                errors).getVariables();
+        if (variablesMap == null) {
+            throw new GenesisException(400,
+                    "Error during metadata parsing for mode %s :%n%s"
+                            .formatted(mode, errors.getLast().getMessage())
+            );
+        }
+        return variablesMap;
+    }
+
+    private List<SurveyUnitModel> getConvertedSurveyUnits(String questionnaireId, Mode mode, List<String> interrogationIdListForMode, int maxIndex, VariablesMap variablesMap) {
+        List<String> interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex);
+        List<LunaticJsonRawDataModel> rawData = getRawData(questionnaireId, mode, interrogationIdToProcess);
+        return convertRawData(
+                rawData,
+                variablesMap
+        );
+    }
+
     private void sendProcessedIdsToQualityTool(List<SurveyUnitModel> surveyUnitModels) {
         try {
             ResponseEntity<Object> response =

src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java

@@ -98,10 +98,10 @@ public List<GroupedInterrogation> findUnprocessedIds() {
     }
 
     @Override
-    public Set<String> findUnprocessedInterrogationIdsByQuestionnaire(String questionnaireId) {
+    public Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) {
         Set<String> interrogationIds = new HashSet<>();
 
-        repository.aggregateRawGroupedWithNullProcessDate(questionnaireId).forEach(
+        repository.aggregateRawGroupedWithNullProcessDate(collectionInstrumentId).forEach(
                 groupedInterrogationDocument -> interrogationIds.addAll(groupedInterrogationDocument.getInterrogationIds())
         );
 

src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java

@@ -177,11 +177,11 @@ public List<GroupedInterrogation> findUnprocessedIds() {
     }
 
     @Override
-    public Set<String> findUnprocessedInterrogationIdsByQuestionnaire(String questionnaireId) {
+    public Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) {
        List<LunaticJsonRawDataDocument> unprocessedDocuments =
                mongoStub.stream().filter(
                        lunaticJsonDataDocument -> lunaticJsonDataDocument.processDate() == null
-                       && lunaticJsonDataDocument.questionnaireId().equals(questionnaireId)
+                       && lunaticJsonDataDocument.questionnaireId().equals(collectionInstrumentId)
                ).toList();
        Set<String> interrogationIds = new HashSet<>();
        unprocessedDocuments.forEach(doc -> interrogationIds.add(doc.interrogationId()));