Merge pull request #148 from InseeFr/devSaveFolderHotfix

Fix save folder saving only 1 file at a time

Author: alexisszmundy

.gitignore

@@ -47,3 +47,4 @@ hs_err_pid*
 src/test/resources/OUT/*
 /src/test/resources/DONE/*
 /src/test/resources/genesis_deleted_schedules/*
+/src/test/resources/IN/WEB/SAMPLETEST/

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -422,7 +422,7 @@ private void processCampaignWithMode(String campaignName, Mode mode, List<Genesi
             if(isDataFileInDoneFolder(filepath, campaignName, mode.getFolder())){
                 log.warn("File {} already exists in DONE folder ! Deleting...", fileName);
                 Files.deleteIfExists(filepath);
-                return;
+                continue; //Go to next file
             }
             //Read file
             log.info("Try to read Xml file : {}", fileName);
@@ -433,11 +433,13 @@ private void processCampaignWithMode(String campaignName, Mode mode, List<Genesi
                 response = processXmlFileSequentially(filepath, mode, variablesMap);
             }
             log.debug("File {} saved", fileName);
-            if(response.getStatusCode() == HttpStatus.OK){
-                fileUtils.moveDataFile(campaignName, mode.getFolder(), filepath);
-                return;
+            if (response.getStatusCode() == HttpStatus.OK) {
+                fileUtils.moveDataFile(campaignName, mode.getFolder(),
+                        filepath);
+                //Sonar is annoying when there is more than 1 continue
+            } else {
+                log.error("Error {} on file {} : {}", response.getStatusCode(), fileName,  response.getBody());
             }
-            log.error("Error on file {}", fileName);
         }
     }
 

src/test/java/cucumber/functional_tests/MainDefinitions.java

@@ -4,17 +4,30 @@
 import fr.insee.bpm.exceptions.MetadataParserException;
 import fr.insee.bpm.metadata.model.VariablesMap;
 import fr.insee.bpm.metadata.reader.ddi.DDIReader;
+import fr.insee.genesis.configuration.Config;
 import fr.insee.genesis.controller.adapter.LunaticXmlAdapter;
-import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
-import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
+import fr.insee.genesis.controller.rest.responses.ResponseController;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlCampaign;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlDataParser;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit;
+import fr.insee.genesis.controller.utils.ControllerUtils;
 import fr.insee.genesis.domain.model.surveyunit.CollectedVariable;
 import fr.insee.genesis.domain.model.surveyunit.DataState;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
 import fr.insee.genesis.domain.model.surveyunit.Variable;
+import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService;
+import fr.insee.genesis.domain.service.rawdata.LunaticXmlRawDataService;
+import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
+import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService;
 import fr.insee.genesis.exceptions.GenesisException;
+import fr.insee.genesis.infrastructure.utils.FileUtils;
+import fr.insee.genesis.stubs.ConfigStub;
+import fr.insee.genesis.stubs.LunaticJsonPersistanceStub;
+import fr.insee.genesis.stubs.LunaticXmlPersistanceStub;
+import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
+import io.cucumber.java.After;
+import io.cucumber.java.Before;
 import io.cucumber.java.en.Given;
 import io.cucumber.java.en.Then;
 import io.cucumber.java.en.When;
@@ -24,8 +37,10 @@
 import javax.xml.parsers.ParserConfigurationException;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -36,19 +51,37 @@ public class MainDefinitions {
     final Path ddiDirectory = Paths.get(TestConstants.FUNCTIONAL_TESTS_DDI_DIRECTORY);
 
     SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService();
+    SurveyUnitPersistencePortStub surveyUnitPersistence = new SurveyUnitPersistencePortStub();
+    Config config = new ConfigStub();
+
+    ResponseController responseController = new ResponseController(
+            new SurveyUnitService(surveyUnitPersistence),
+            surveyUnitQualityService,
+            new LunaticXmlRawDataService(new LunaticXmlPersistanceStub()),
+            new LunaticJsonRawDataService(new LunaticJsonPersistanceStub()),
+            new FileUtils(config),
+            new ControllerUtils(new FileUtils(config))
+        );
 
     List<SurveyUnitModel> surveyUnitModels;
 
+    @Before
+    public void init() {
+        this.surveyUnitPersistence.getMongoStub().clear();
+    }
+
     @Given("We have data in directory {string}")
-    public void init(String directory){
+    public void init_folder(String directory) throws IOException {
         this.directory = directory;
         inDirectory = inDirectory.resolve(directory);
+        Files.createDirectories(inDirectory);
     }
 
     @When("We create DTOs from file {string} with DDI {string}")
-    public void get_dtos(String fileName, String DDIName) throws IOException, ParserConfigurationException, SAXException, GenesisException, MetadataParserException {
+    public void get_dtos(String fileName, String ddiName) throws IOException, ParserConfigurationException,
+            SAXException, GenesisException, MetadataParserException {
         Path filePath = inDirectory.resolve(fileName);
-        Path ddiFilePath = ddiDirectory.resolve(directory).resolve(DDIName);
+        Path ddiFilePath = ddiDirectory.resolve(directory).resolve(ddiName);
 
         if(fileName.endsWith(".xml")){
             LunaticXmlDataParser parser = new LunaticXmlDataParser();
@@ -67,6 +100,32 @@ public void get_dtos(String fileName, String DDIName) throws IOException, Parser
         }
     }
 
+    @When("We copy data file {string} to that directory")
+    public void copy_data_file(String dataFile) throws IOException {
+        Path dataFilePath = Paths.get(TestConstants.TEST_RESOURCES_DIRECTORY).resolve(dataFile);
+        Files.copy(dataFilePath, this.inDirectory.resolve(dataFilePath.getFileName().toString()),
+                StandardCopyOption.REPLACE_EXISTING);
+    }
+
+    @When("We save data from that directory")
+    public void get_dtos_from_folder() throws Exception {
+        responseController.saveResponsesFromXmlCampaignFolder(this.inDirectory.getFileName().toString(), null, true);
+    }
+
+    @When("We delete that directory")
+    public void delete_directory() throws IOException {
+        org.springframework.util.FileSystemUtils.deleteRecursively(inDirectory);
+    }
+
+    @Then("There should be {int} {string} SurveyUnit in database")
+    public void check_surveyunits_by_state(int expectedCount, String expectedStatus) {
+        Assertions.assertThat(this.surveyUnitPersistence.getMongoStub().stream().filter(
+                surveyUnitModel -> surveyUnitModel.getState().name().equals(expectedStatus)
+        ).toList()).hasSize(expectedCount);
+    }
+
+
+
     @Then("For SurveyUnit {string} there should be at least one {string} SurveyUnit DTO")
     public void check_expected_datastate_dto(String surveyUnitId, String expectedDataState) {
         Assertions.assertThat(this.surveyUnitModels).filteredOn(surveyUnitDto ->
@@ -76,9 +135,9 @@ public void check_expected_datastate_dto(String surveyUnitId, String expectedDat
     }
 
     @Then("For SurveyUnit {string} there shouldn't be a {string} SurveyUnit DTO")
-    public void check_unexpected_datastate_dto(String surveyUnitId, String UnexpectedDataState) {
+    public void check_unexpected_datastate_dto(String surveyUnitId, String unexpectedDataState) {
         Assertions.assertThat(this.surveyUnitModels).filteredOn(surveyUnitDto ->
-                surveyUnitDto.getState().toString().equals(UnexpectedDataState)
+                surveyUnitDto.getState().toString().equals(unexpectedDataState)
                 && surveyUnitDto.getIdUE().equals(surveyUnitId)
         ).isEmpty();
     }
@@ -185,4 +244,11 @@ public void external_variable_content_check(String externalVariableName, String
         Assertions.assertThat(concernedVariable.getValues()).hasSizeGreaterThan(expectedValueIndex);
         Assertions.assertThat(concernedVariable.getValues().get(expectedValueIndex)).isEqualTo(expectedValue);
     }
+
+    @After
+    public void clean() throws IOException {
+        //Clean DONE test folder
+        org.springframework.util.FileSystemUtils.deleteRecursively(Path.of(TestConstants.TEST_RESOURCES_DIRECTORY,
+                "DONE"));
+    }
 }

src/test/resources/data_backup/data.complete.partial.STPDv1.20231122164209.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Campaign>
+  <Id>SAMPLETEST-PARADATA-v1</Id>
+  <Label />
+  <SurveyUnits />
+</Campaign>