Merge pull request #297 from InseeFr/devJsonExtraction

loichenninger · web-flow · commit 049050c10351 · 2025-10-17T14:27:26.000+02:00
New endpoints for Json extraction
diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java
@@ -14,6 +14,7 @@ public class Constants {
     public static final String FILTER_RESULT_PREFIX = "FILTER_RESULT_";
     public static final String MISSING_SUFFIX = "_MISSING";
     public static final String MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME = "lunaticjsondata";
+    public static final String MONGODB_EXTRACTION_JSON_COLLECTION_NAME = "lastjsonextraction";
     private static final String[] ENO_VARIABLES = {"COMMENT_QE","COMMENT_UE","HEURE_REMPL","MIN_REMPL"};
 
     public static final String MONGODB_SCHEDULE_COLLECTION_NAME = "schedules";
diff --git a/src/main/java/fr/insee/genesis/controller/dto/LastExtractionResponseDto.java b/src/main/java/fr/insee/genesis/controller/dto/LastExtractionResponseDto.java
@@ -0,0 +1,15 @@
+package fr.insee.genesis.controller.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.time.LocalDateTime;
+
+@Getter
+@Setter
+public class LastExtractionResponseDto {
+    private final String lastExtractionDate;
+    public LastExtractionResponseDto(LocalDateTime lastExtractionDate) {
+        this.lastExtractionDate = lastExtractionDate != null ? lastExtractionDate.toString() : null;
+    }
+}
diff --git a/src/main/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDto.java b/src/main/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDto.java
@@ -0,0 +1,26 @@
+package fr.insee.genesis.controller.mappers;
+
+import fr.insee.genesis.controller.dto.ScheduleDto;
+import fr.insee.genesis.domain.model.context.DataProcessingContextModel;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DataProcessingContextMapperDto {
+
+    public ScheduleDto dataProcessingContextToScheduleDto(DataProcessingContextModel dataProcessingContext){
+        return ScheduleDto.builder()
+                .surveyName(dataProcessingContext.getPartitionId())
+                .lastExecution(dataProcessingContext.getLastExecution())
+                .kraftwerkExecutionScheduleList(dataProcessingContext.getKraftwerkExecutionScheduleList())
+                .build();
+    }
+
+    public List<ScheduleDto> dataProcessingContextListToScheduleDtoList(List<DataProcessingContextModel> contexts){
+        List<ScheduleDto> dtos = new ArrayList<>();
+        for(DataProcessingContextModel context : contexts){
+            dtos.add(dataProcessingContextToScheduleDto(context));
+        }
+        return dtos;
+    }
+}
diff --git a/src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java b/src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java
@@ -0,0 +1,60 @@
+package fr.insee.genesis.controller.rest;
+
+import fr.insee.genesis.controller.dto.LastExtractionResponseDto;
+import fr.insee.genesis.domain.model.extraction.json.LastJsonExtractionModel;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.ports.api.LastJsonExtractionApiPort;
+import fr.insee.genesis.exceptions.GenesisException;
+import io.swagger.v3.oas.annotations.Operation;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import java.time.LocalDateTime;
+
+@Slf4j
+@Controller
+@AllArgsConstructor
+@RequestMapping(path = "/extractions")
+public class JsonExtractionController {
+
+    LastJsonExtractionApiPort lastJsonExtractionApiPort;
+
+    @Operation(summary = "Record the date of the latest JSON data extraction in Kraftwerk")
+    @PutMapping(path = "/json")
+    @PreAuthorize("hasAnyRole('USER_KRAFTWERK','SCHEDULER')")
+    public ResponseEntity<String> saveLastJsonExtractionDate(
+            @RequestParam("questionnaireId") String questionnaireId,
+            @RequestParam(value = "mode", required = false) Mode mode){
+        LocalDateTime extractDate = LocalDateTime.now();
+        LastJsonExtractionModel extract = LastJsonExtractionModel.builder()
+                .questionnaireModelId(questionnaireId)
+                .mode(mode)
+                .lastExtractionDate(extractDate)
+                .build();
+        lastJsonExtractionApiPort.recordDate(extract);
+        return ResponseEntity.ok().build();
+    }
+
+    @Operation(summary = "Get the date of the latest JSON data extraction in Kraftwerk")
+    @GetMapping(path = "/json")
+    @PreAuthorize("hasAnyRole('USER_KRAFTWERK','SCHEDULER')")
+    public ResponseEntity<LastExtractionResponseDto> getLastJsonExtractionDate(
+            @RequestParam("questionnaireId") String questionnaireId,
+            @RequestParam(value = "mode", required = false) Mode mode){
+        try{
+            LastJsonExtractionModel lastJsonExtraction = lastJsonExtractionApiPort.getLastExtractionDate(questionnaireId,mode);
+            return ResponseEntity.ok(new LastExtractionResponseDto(lastJsonExtraction.getLastExtractionDate()));
+        } catch (GenesisException e){
+            return ResponseEntity.notFound().build();
+        }
+    }
+
+}
diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java
@@ -6,13 +6,15 @@
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.format.annotation.DateTimeFormat;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
+import java.time.LocalDateTime;
 import java.util.List;
 
 @RequestMapping(path = "/interrogations" )
@@ -35,6 +37,15 @@ public ResponseEntity<List<InterrogationId>> getAllInterrogationIdsByQuestionnai
         return ResponseEntity.ok(responses);
     }
 
+    @Operation(summary = "Retrieve interrogations recorded since a specified date for a given questionnaire")
+    @GetMapping(path = "/by-questionnaire-and-since-datetime")
+    public ResponseEntity<List<InterrogationId>> getAllInterrogationIdsByQuestionnaire(
+            @RequestParam("questionnaireId") String questionnaireId,
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) LocalDateTime since) {
+        List<InterrogationId> responses = surveyUnitService.findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(questionnaireId, since);
+        return ResponseEntity.ok(responses);
+    }
+
 
     //========= OPTIMISATIONS PERFS (START) ==========
     /**
diff --git a/src/main/java/fr/insee/genesis/domain/model/extraction/json/LastJsonExtractionModel.java b/src/main/java/fr/insee/genesis/domain/model/extraction/json/LastJsonExtractionModel.java
@@ -0,0 +1,22 @@
+package fr.insee.genesis.domain.model.extraction.json;
+
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.springframework.data.annotation.Id;
+
+import java.time.LocalDateTime;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class LastJsonExtractionModel {
+        @Id
+        private String id; //Used to remove warning
+        String questionnaireModelId;
+        Mode mode;
+        LocalDateTime lastExtractionDate;
+}
diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LastJsonExtractionApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LastJsonExtractionApiPort.java
@@ -0,0 +1,11 @@
+package fr.insee.genesis.domain.ports.api;
+
+import fr.insee.genesis.domain.model.extraction.json.LastJsonExtractionModel;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.exceptions.GenesisException;
+
+public interface LastJsonExtractionApiPort {
+    void recordDate(LastJsonExtractionModel extraction);
+
+    LastJsonExtractionModel getLastExtractionDate(String questionnaireModelId, Mode mode) throws GenesisException;
+}
diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java
@@ -10,6 +10,7 @@
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
 import fr.insee.genesis.exceptions.GenesisException;
 
+import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Stream;
@@ -37,6 +38,8 @@ public interface SurveyUnitApiPort {
 
     List<InterrogationId> findDistinctInterrogationIdsByQuestionnaireId(String questionnaireId);
 
+    List<InterrogationId> findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since);
+
     //========= OPTIMISATIONS PERFS (START) ==========
     long countInterrogationIdsByQuestionnaireId(String questionnaireId);
 
diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LastJsonExtractionPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LastJsonExtractionPersistencePort.java
@@ -0,0 +1,10 @@
+package fr.insee.genesis.domain.ports.spi;
+
+import fr.insee.genesis.domain.model.extraction.json.LastJsonExtractionModel;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.exceptions.GenesisException;
+
+public interface LastJsonExtractionPersistencePort {
+    void save(LastJsonExtractionModel extraction);
+    LastJsonExtractionModel getLastExecutionDate(String questionnaireModelId, Mode mode) throws GenesisException;
+}
diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java
@@ -2,6 +2,7 @@
 
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
 
+import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Stream;
@@ -27,6 +28,8 @@ public interface SurveyUnitPersistencePort {
 
     List<SurveyUnitModel> findInterrogationIdsByQuestionnaireId(String questionnaireId);
 
+    List<SurveyUnitModel> findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since);
+
     //======== OPTIMISATIONS PERFS (START) ========
     long countInterrogationIdsByQuestionnaireId(String questionnaireId);
 
diff --git a/src/main/java/fr/insee/genesis/domain/service/extraction/LastJsonExtractionService.java b/src/main/java/fr/insee/genesis/domain/service/extraction/LastJsonExtractionService.java
@@ -0,0 +1,40 @@
+package fr.insee.genesis.domain.service.extraction;
+
+import fr.insee.genesis.domain.model.extraction.json.LastJsonExtractionModel;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.ports.api.LastJsonExtractionApiPort;
+import fr.insee.genesis.domain.ports.spi.LastJsonExtractionPersistencePort;
+import fr.insee.genesis.exceptions.GenesisException;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Service;
+
+@Service
+@Slf4j
+public class LastJsonExtractionService implements LastJsonExtractionApiPort {
+
+    @Qualifier("lastJsonExtractionMongoAdapter")
+    LastJsonExtractionPersistencePort extractionPersistencePort;
+
+    @Autowired
+    public LastJsonExtractionService(LastJsonExtractionPersistencePort extractionPersistencePort) {
+        this.extractionPersistencePort = extractionPersistencePort;
+    }
+
+    @Override
+    public void recordDate(LastJsonExtractionModel extraction) {
+        // Create a unique ID based on the questionnaire and the mode.
+        extraction.setId(String.format("%s_%s",extraction.getQuestionnaireModelId(),extraction.getMode()));
+
+        // save() does an insert if the id doesn't exist, otherwise an update
+        extractionPersistencePort.save(extraction);
+    }
+
+    @Override
+    public LastJsonExtractionModel getLastExtractionDate(String questionnaireModelId, Mode mode) throws GenesisException {
+        return extractionPersistencePort.getLastExecutionDate(questionnaireModelId,mode);
+    }
+
+
+}
diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java
@@ -275,6 +275,16 @@ public List<InterrogationId> findDistinctInterrogationIdsByQuestionnaireId(Strin
         return suIds.stream().distinct().toList();
     }
 
+    @Override
+    public List<InterrogationId> findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since) {
+        return  surveyUnitPersistencePort
+                .findInterrogationIdsByQuestionnaireIdAndDateAfter(questionnaireId, since)
+                .stream()
+                .map(su -> new InterrogationId(su.getInterrogationId()))
+                .distinct()
+                .toList();
+    }
+
     //============ OPTIMISATIONS PERFS (START) ============
 
     /**
diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LastJsonExtractionMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LastJsonExtractionMongoAdapter.java
@@ -0,0 +1,46 @@
+package fr.insee.genesis.infrastructure.adapter;
+
+import fr.insee.genesis.domain.model.extraction.json.LastJsonExtractionModel;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.ports.spi.LastJsonExtractionPersistencePort;
+import fr.insee.genesis.exceptions.GenesisException;
+import fr.insee.genesis.infrastructure.document.extraction.json.LastJsonExtractionDocument;
+import fr.insee.genesis.infrastructure.mappers.LastJsonExtractionDocumentMapper;
+import fr.insee.genesis.infrastructure.repository.LastJsonExtractionMongoDBRepository;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Service;
+
+import java.util.Optional;
+
+@Service
+@Qualifier("lastJsonExtractionMongoAdapter")
+@Slf4j
+public class LastJsonExtractionMongoAdapter implements LastJsonExtractionPersistencePort {
+
+    private final LastJsonExtractionMongoDBRepository extractionRepository;
+
+    @Autowired
+    public LastJsonExtractionMongoAdapter(LastJsonExtractionMongoDBRepository extractionRepository) {
+        this.extractionRepository = extractionRepository;
+    }
+
+    @Override
+    public void save(LastJsonExtractionModel extraction) {
+        extractionRepository.save(LastJsonExtractionDocumentMapper.INSTANCE.modelToDocument(extraction));
+    }
+
+    @Override
+    public LastJsonExtractionModel getLastExecutionDate(String questionnaireModelId, Mode mode) throws GenesisException {
+        String id = String.format("%s_%s",questionnaireModelId, mode);
+        Optional<LastJsonExtractionDocument> extraction = extractionRepository.findById(id);
+        if (extraction.isPresent()) {
+            return LastJsonExtractionDocumentMapper.INSTANCE.documentToModel(extraction.get());
+        } else {
+            String message = String.format("No extraction date found for questionnaire %s and mode %s",questionnaireModelId,mode==null?null:mode.getModeName());
+            throw new GenesisException(404,message);
+        }
+    }
+
+}
diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java
@@ -15,6 +15,7 @@
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.stereotype.Service;
 
+import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -154,6 +155,12 @@ public List<SurveyUnitModel> findInterrogationIdsByQuestionnaireId(String questi
 		return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits);
 	}
 
+	@Override
+	public List<SurveyUnitModel> findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since) {
+		List<SurveyUnitDocument> surveyUnits = mongoRepository.findInterrogationIdsByQuestionnaireIdAndDateAfter(questionnaireId, since);
+		return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits);
+	}
+
 	//========== OPTIMISATIONS PERFS (START) ===========
 	@Override
 	public long countInterrogationIdsByQuestionnaireId(String questionnaireId) {
diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/extraction/json/LastJsonExtractionDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/extraction/json/LastJsonExtractionDocument.java
@@ -0,0 +1,21 @@
+package fr.insee.genesis.infrastructure.document.extraction.json;
+
+import fr.insee.genesis.Constants;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import lombok.Data;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+import java.time.LocalDateTime;
+
+@Data
+@Document(collection = Constants.MONGODB_EXTRACTION_JSON_COLLECTION_NAME)
+public class LastJsonExtractionDocument {
+
+    @Id
+    private String id;
+    private String questionnaireModelId;
+    private Mode mode;
+    private LocalDateTime lastExtractionDate;
+
+}
diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/LastJsonExtractionDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/LastJsonExtractionDocumentMapper.java
@@ -0,0 +1,22 @@
+package fr.insee.genesis.infrastructure.mappers;
+
+import fr.insee.genesis.domain.model.extraction.json.LastJsonExtractionModel;
+import fr.insee.genesis.infrastructure.document.extraction.json.LastJsonExtractionDocument;
+import org.mapstruct.Mapper;
+import org.mapstruct.factory.Mappers;
+
+import java.util.List;
+
+@Mapper
+public interface LastJsonExtractionDocumentMapper {
+
+    LastJsonExtractionDocumentMapper INSTANCE = Mappers.getMapper(LastJsonExtractionDocumentMapper.class);
+
+    LastJsonExtractionModel documentToModel(LastJsonExtractionDocument lastJsonExtractionDoc);
+
+    LastJsonExtractionDocument modelToDocument(LastJsonExtractionModel lastJsonExtractionModel);
+
+    List<LastJsonExtractionModel> listDocumentToListModel(List<LastJsonExtractionDocument> lastJsonExtractionDocumentList);
+
+    List<LastJsonExtractionDocument> listModelToListDocument(List<LastJsonExtractionModel> lastJsonExtractionModelList);
+}
diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LastJsonExtractionMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LastJsonExtractionMongoDBRepository.java
diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java
diff --git a/src/test/java/cucumber/functional_tests/config/CucumberSpringConfiguration.java b/src/test/java/cucumber/functional_tests/config/CucumberSpringConfiguration.java
diff --git a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java
diff --git a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java
diff --git a/src/test/java/fr/insee/genesis/stubs/LastJsonExtractionPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/LastJsonExtractionPersistencePortStub.java
diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java
