Merge pull request #205 from InseeFr/mergeRawProcessAndLunaticModel

Raw process

Author: loichenninger

src/main/java/fr/insee/genesis/configuration/LogRequestFilter.java

@@ -13,7 +13,6 @@
 import org.springframework.web.util.ContentCachingResponseWrapper;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 
 
 @Component
@@ -24,8 +23,6 @@ public class LogRequestFilter extends OncePerRequestFilter {
 
 	private static final String REQUEST_MESSAGE_FORMAT = 
 			 "CALL {} {} - "
-		//	+ "Content-Type :  {} \n "
-		//	+ "Headers : {} \n "
 			+ "Params : {}";
 
 	private static final String RESPONSE_MESSAGE_FORMAT = 
@@ -59,9 +56,4 @@ protected void doFilterInternal(HttpServletRequest request, HttpServletResponse
         resp.copyBodyToResponse();
     }
 
-	private String getResponseBody(ContentCachingRequestWrapper req, ContentCachingResponseWrapper resp) {
-		if (req.getRequestURI().contains("swagger-ui") ||req.getRequestURI().contains("api-docs")) return "Hidden Swagger response";
-		return new String(resp.getContentAsByteArray(), StandardCharsets.UTF_8);
-	}
-    
 }

src/main/java/fr/insee/genesis/configuration/auth/security/OIDCSecurityConfig.java

@@ -20,7 +20,6 @@
 import org.springframework.security.oauth2.jwt.Jwt;
 import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationConverter;
 import org.springframework.security.web.SecurityFilterChain;
-import org.springframework.security.web.authentication.www.BasicAuthenticationFilter;
 import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
 
 import java.util.Collection;

src/main/java/fr/insee/genesis/controller/dto/InterrogationId.java

@@ -1,30 +1,34 @@
 package fr.insee.genesis.controller.dto;
 
 import lombok.AllArgsConstructor;
-import lombok.Getter;
 import lombok.NoArgsConstructor;
-import lombok.Setter;
 
 import java.util.Objects;
 
-@Setter
-@Getter
 @NoArgsConstructor
 @AllArgsConstructor
 public class InterrogationId {
 
-	private String interrogationId;
+	private String identifier;
 
 	@Override
 	public boolean equals(Object o) {
 		if (this == o) return true;
 		if (o == null || getClass() != o.getClass()) return false;
 		InterrogationId that = (InterrogationId) o;
-		return Objects.equals(interrogationId, that.interrogationId);
+		return Objects.equals(identifier, that.identifier);
 	}
 
 	@Override
 	public int hashCode() {
-		return Objects.hash(interrogationId);
+		return Objects.hash(identifier);
+	}
+
+	public String getInterrogationId() {
+		return identifier;
+	}
+
+	public void setInterrogationId(String identifier) {
+		this.identifier = identifier;
 	}
 }

src/main/java/fr/insee/genesis/controller/dto/rawdata/LunaticJsonRawDataUnprocessedDto.java

@@ -3,4 +3,4 @@
 import lombok.Builder;
 
 @Builder
-public record LunaticJsonRawDataUnprocessedDto(String campaignId, String interrogationId){}
+public record LunaticJsonRawDataUnprocessedDto(String campaignId, String questionnaireId, String interrogationId){}

src/main/java/fr/insee/genesis/controller/rest/UtilsController.java

@@ -14,6 +14,7 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
+import javax.xml.stream.XMLStreamException;
 import java.io.IOException;
 
 @RequestMapping(path = "/")
@@ -43,7 +44,7 @@ public ResponseEntity<Object> saveResponsesFromXmlFile(@RequestParam("inputFolde
 														   @RequestParam("outputFolder") String outputFolder,
 														   @RequestParam("filename") String filename,
 														   @RequestParam("nbResponsesByFile") int nbSU)
-			throws Exception {
+			throws XMLStreamException, IOException {
 		XMLSplitter.split(inputFolder, filename, outputFolder, "SurveyUnit", nbSU);
 		return ResponseEntity.ok("File split");
 	}

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -1,22 +1,36 @@
 package fr.insee.genesis.controller.rest.responses;
 
+import fr.insee.bpm.metadata.model.VariablesMap;
 import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
+import fr.insee.genesis.controller.services.MetadataService;
+import fr.insee.genesis.controller.utils.ControllerUtils;
+import fr.insee.genesis.domain.model.surveyunit.DataState;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
+import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
 import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService;
+import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
+import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService;
+import fr.insee.genesis.exceptions.GenesisError;
+import fr.insee.genesis.exceptions.GenesisException;
+import fr.insee.genesis.infrastructure.utils.FileUtils;
 import io.swagger.v3.oas.annotations.Hidden;
 import io.swagger.v3.oas.annotations.Operation;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
 import java.time.LocalDateTime;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
@@ -26,23 +40,9 @@
 public class RawResponseController {
 
     private static final String SUCCESS_MESSAGE = "Interrogation %s saved";
-    private final LunaticJsonRawDataService lunaticJsonRawDataApiPort;
-//    private final ControllerUtils controllerUtils;
-//    private final MetadataService metadataService;
-//    private final SurveyUnitService surveyUnitService;
-//    private final SurveyUnitQualityService surveyUnitQualityService;
-//    private final FileUtils fileUtils;
+    private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
 
-   /* public RawResponseController(LunaticJsonRawDataService lunaticJsonRawDataApiPort, ControllerUtils controllerUtils, MetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils) {
-        this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
-        this.controllerUtils = controllerUtils;
-        this.metadataService = metadataService;
-        this.surveyUnitService = surveyUnitService;
-        this.surveyUnitQualityService = surveyUnitQualityService;
-        this.fileUtils = fileUtils;
-    }*/
-
-    public RawResponseController(LunaticJsonRawDataService lunaticJsonRawDataApiPort) {
+    public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort) {
         this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
     }
 
@@ -100,63 +100,26 @@ public ResponseEntity<LunaticJsonRawDataModel> getJsonRawData(
     }
 
     //PROCESS
-  /*  @Operation(summary = "Process raw data of a campaign")
+    @Operation(summary = "Process raw data of a campaign")
     @PostMapping(path = "/lunatic-json/process")
-    @PreAuthorize("hasRole('ADMIN')")
+    @PreAuthorize("hasRole('SCHEDULER')")
     public ResponseEntity<Object> processJsonRawData(
             @RequestParam("campaignName") String campaignName,
             @RequestParam("questionnaireId") String questionnaireId,
             @RequestBody List<String> interrogationIdList
     ){
         log.info("Try to process raw JSON datas for campaign {} and {} interrogationIds", campaignName, interrogationIdList.size());
-
-        int dataCount = 0;
-        int forcedDataCount = 0;
         List<GenesisError> errors = new ArrayList<>();
 
         try {
-            List<Mode> modesList = controllerUtils.getModesList(campaignName, null);
-            for (Mode mode : modesList) {
-                //Load and save metadatas into database, throw exception if none
-                VariablesMap variablesMap = metadataService.readMetadatas(campaignName, mode.getModeName(), fileUtils,
-                        errors);
-                if (variablesMap == null) {
-                    throw new GenesisException(400,
-                            "Error during metadata parsing for mode %s :%n%s"
-                                    .formatted(mode, errors.getLast().getMessage())
-                    );
-                }
-
-                List<LunaticJsonRawDataModel> rawData = lunaticJsonRawDataApiPort.getRawData(campaignName,mode,interrogationIdList);
-                //Save converted data
-                List<SurveyUnitModel> surveyUnitModels = lunaticJsonRawDataApiPort.convertRawData(
-                        rawData,
-                        variablesMap
-                );
-
-                surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap);
-                surveyUnitService.saveSurveyUnits(surveyUnitModels);
-
-                //Update process dates
-                lunaticJsonRawDataApiPort.updateProcessDates(surveyUnitModels);
-
-                //Save metadatas
-                //TODO Enable when mapping problem solved for get metadatas step
-                //variableTypeApiPort.saveMetadatas(campaignName, questionnaireId, mode, variablesMap);
-
-                //Increment data count
-                dataCount += surveyUnitModels.size();
-                forcedDataCount += surveyUnitModels.stream().filter(
-                        surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORCED)
-                ).toList().size();
-            }
-            return forcedDataCount == 0 ?
-                    ResponseEntity.ok("%d document(s) processed".formatted(dataCount))
-                    : ResponseEntity.ok("%d document(s) processed, including %d FORCED after data verification"
-                    .formatted(dataCount, forcedDataCount));
-        }catch (GenesisException e){ //TODO replace with spring exception handler
+            DataProcessResult result = lunaticJsonRawDataApiPort.processRawData(campaignName, interrogationIdList, errors);
+            return result.formattedDataCount() == 0 ?
+                    ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount()))
+                    : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification"
+                    .formatted(result.dataCount(), result.formattedDataCount()));
+        }catch (GenesisException e){
             return ResponseEntity.status(e.getStatus()).body(e.getMessage());
         }
     }
-*/
+
 }

src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java

@@ -0,0 +1,4 @@
+package fr.insee.genesis.domain.model.surveyunit.rawdata;
+
+public record DataProcessResult(int dataCount, int formattedDataCount) {
+}

src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java

@@ -4,7 +4,10 @@
 import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
+import fr.insee.genesis.exceptions.GenesisError;
+import fr.insee.genesis.exceptions.GenesisException;
 
 import java.util.List;
 import java.util.Set;
@@ -15,13 +18,9 @@ public interface LunaticJsonRawDataApiPort {
     List<LunaticJsonRawDataModel> getRawData(String campaignName, Mode mode, List<String> interrogationIdList);
     List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawData, VariablesMap variablesMap);
     List<LunaticJsonRawDataUnprocessedDto> getUnprocessedDataIds();
-    List<SurveyUnitModel> parseRawData(
-            String campaignName,
-            Mode mode,
-            List<String> interrogationIdList,
-            VariablesMap variablesMap
-    );
     void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
     Set<String> findDistinctQuestionnaireIds();
     long countResponsesByQuestionnaireId(String campaignId);
+
+    DataProcessResult processRawData(String campaignName, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException;
 }