Fix missing columns during csv export

src/main/java/org/gridsuite/securityanalysis/server/dto/LimitViolationDTO.java

@@ -60,13 +60,17 @@ private static String convertDoubleToLocale(Double value, String language) {
 
     public List<String> toCsvRow(Map<String, String> translations, String language) {
         List<String> csvRow = new ArrayList<>();
-        csvRow.add(this.getLocationId());
         csvRow.add(this.getLimitType() != null ? CsvExportUtils.translate(this.getLimitType().name(), translations) : "");
+        csvRow.add(this.getLocationId());  // busId
         csvRow.add(CsvExportUtils.replaceNullWithEmptyString(CsvExportUtils.translate(this.getLimitName(), translations)));
-        csvRow.add(convertDoubleToLocale(this.getLimit(), language));
-        csvRow.add(convertDoubleToLocale(this.getValue(), language));
         csvRow.add(this.getLoading() == null ? "" : convertDoubleToLocale(this.getLoading(), language));
+        csvRow.add(this.getPatlLoading() == null ? "" : convertDoubleToLocale(this.getPatlLoading(), language));
         csvRow.add(this.getAcceptableDuration() == Integer.MAX_VALUE ? null : Integer.toString(this.getAcceptableDuration()));
+        csvRow.add(this.getUpcomingAcceptableDuration() == null || this.getUpcomingAcceptableDuration() == Integer.MAX_VALUE ? null : Integer.toString(this.getUpcomingAcceptableDuration()));
+        csvRow.add(CsvExportUtils.replaceNullWithEmptyString(CsvExportUtils.translate(this.getNextLimitName(), translations)));
+        csvRow.add(convertDoubleToLocale(this.getLimit(), language));
+        csvRow.add(this.getPatlLimit() == null ? "" : convertDoubleToLocale(this.getPatlLimit(), language));
+        csvRow.add(convertDoubleToLocale(this.getValue(), language));
         csvRow.add(this.getSide() != null ? CsvExportUtils.translate(this.getSide().name(), translations) : "");
         return csvRow;
     }

src/main/java/org/gridsuite/securityanalysis/server/dto/PreContingencyLimitViolationResultDTO.java

@@ -44,6 +44,7 @@ public List<String> toCsvRow(Map<String, String> translations, String language)
         List<String> csvRow = List.of();
 
         if (this.getLimitViolation() != null) {
+            csvRow = List.of(this.getSubjectId());
             return Stream.concat(csvRow.stream(), this.getLimitViolation().toCsvRow(translations, language).stream()).toList();
         }
 

src/test/java/org/gridsuite/securityanalysis/server/SecurityAnalysisControllerTest.java

@@ -44,6 +44,8 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.MockitoAnnotations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
 import org.springframework.boot.test.context.SpringBootTest;
@@ -73,8 +75,10 @@
 import static org.gridsuite.computation.service.NotificationService.*;
 import static org.gridsuite.securityanalysis.server.SecurityAnalysisProviderMock.*;
 import static org.gridsuite.securityanalysis.server.service.SecurityAnalysisService.COMPUTATION_TYPE;
+import static org.gridsuite.securityanalysis.server.util.CsvExportUtils.csvRowsToZippedCsv;
 import static org.gridsuite.securityanalysis.server.util.DatabaseQueryUtils.assertRequestsCount;
 import static org.gridsuite.securityanalysis.server.util.TestUtils.assertLogMessage;
+import static org.gridsuite.securityanalysis.server.util.TestUtils.readLinesFromFilePath;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.*;
 import static org.mockito.ArgumentMatchers.any;
@@ -93,6 +97,7 @@
 @SpringBootTest
 @ContextConfigurationWithTestChannel
 class SecurityAnalysisControllerTest {
+    private static final Logger LOGGER = LoggerFactory.getLogger(SecurityAnalysisControllerTest.class);
 
     private static final UUID NETWORK_UUID = UUID.fromString("7928181c-7977-4592-ba19-88027e4254e4");
     private static final UUID NETWORK_STOP_UUID = UUID.fromString("7928181c-7977-4592-ba19-88027e4254e6");
@@ -512,7 +517,7 @@ private void checkNResultEnumFilters(UUID resultUuid) throws Exception {
         List<LimitViolationType> limitTypes = mapper.readValue(mvcResult.getResponse().getContentAsString(), new TypeReference<>() { });
         Assertions.assertThat(limitTypes).hasSameElementsAs(expectedLimitTypes);
 
-        List<ThreeSides> expectedSides = nResults.stream().map(result -> result.getLimitViolation().getSide()).distinct().filter(side -> side != null).toList();
+        List<ThreeSides> expectedSides = nResults.stream().map(result -> result.getLimitViolation().getSide()).distinct().filter(Objects::nonNull).toList();
         mvcResult = mockMvc.perform(get("/" + VERSION + "/results/{resultUuid}/n-branch-sides", resultUuid))
             .andExpectAll(
                 status().isOk(),
@@ -835,7 +840,7 @@ void getDefaultProviderTest() throws Exception {
 
     @Test
     void getZippedCsvResults() throws Exception {
-        // running computation to create result
+        // running computation to create some results
         MvcResult mvcResult = mockMvc.perform(post("/" + VERSION + "/networks/" + NETWORK_UUID + "/run-and-save?reportType=SecurityAnalysis&contingencyListName=" + CONTINGENCY_LIST_NAME
                 + "&receiver=me&variantId=" + VARIANT_2_ID + "&provider=OpenLoadFlow" + "&loadFlowParametersUuid=" + UUID.randomUUID())
                 .header(HEADER_USER_ID, "testUserId")
@@ -856,14 +861,50 @@ void getZippedCsvResults() throws Exception {
         checkAllZippedCsvResults();
     }
 
+    @Test
+    void getZippedCsvResultsFromCustomData() throws Exception {
+        final String expectedCsvFile = "/results/n-result-fr-custom.csv";
+        final String lang = "fr";
+        final List<String> header = getCsvHeaderFromResource(expectedCsvFile, lang);
+
+        // Build binary/zipped data from custom DTOs rather than from AS results.
+        PreContingencyLimitViolationResultDTO dto = PreContingencyLimitViolationResultDTO.builder()
+                .subjectId("Ouvrage")
+                .limitViolation(LimitViolationDTO.builder()
+                    .locationId("BUS")
+                    .limitName("lim_name")
+                    .acceptableDuration(100)
+                    .patlLoading(111.5)
+                    .patlLimit(80.66)
+                    .upcomingAcceptableDuration(Integer.MAX_VALUE)
+                    .build())
+                .build();
+        PreContingencyLimitViolationResultDTO dto2 = PreContingencyLimitViolationResultDTO.builder()
+                .subjectId("Ouvrage2")
+                .limitViolation(LimitViolationDTO.builder()
+                        .locationId("BUS2")
+                        .limitName("lim_name2")
+                        .acceptableDuration(100)
+                        .patlLoading(111.5)
+                        .patlLimit(80.66)
+                        .upcomingAcceptableDuration(1000)
+                        .build())
+                .build();
+        List<List<String>> csvRows = List.of(dto.toCsvRow(ENUM_TRANSLATIONS_FR, lang), dto2.toCsvRow(ENUM_TRANSLATIONS_FR, lang));
+        byte[] resultAsByteArray = csvRowsToZippedCsv(header, lang, csvRows);
+        // and compare with the expected file
+        checkCsvResultFromBytes(expectedCsvFile, resultAsByteArray);
+    }
+
+    private List<String> getCsvHeaderFromResource(String resourcePath, String lang) {
+        List<String> lines = readLinesFromFilePath(resourcePath, 1);
+        String header = lines.isEmpty() ? "" : lines.getFirst();
+        return Arrays.asList(header.strip().split("en".equalsIgnoreCase(lang) ? "," : ";"));
+    }
+
     private void checkAllZippedCsvResults() throws Exception {
         SQLStatementCountValidator.reset();
-        checkZippedCsvResult("n-result", "/results/n-result-en.csv",
-            CsvTranslationDTO.builder()
-                .headers(List.of("Equipment", "Violation type", "Limit name", "Limit value (A or kV)", "Calculated value (A or kV)", "Load (%)", "Overload", "Side"))
-                .enumValueTranslations(ENUM_TRANSLATIONS_EN)
-                .language("en")
-                .build());
+        checkZippedCsvResult("n-result", "/results/n-result-en.csv", "en");
         /*
          * SELECT
          * assert result exists
@@ -872,20 +913,11 @@ private void checkAllZippedCsvResults() throws Exception {
         assertRequestsCount(2, 0, 0, 0);
 
         SQLStatementCountValidator.reset();
-        checkZippedCsvResult("n-result", "/results/n-result-fr.csv",
-            CsvTranslationDTO.builder()
-                .headers(List.of("Ouvrage", "Type de contrainte", "Nom du seuil", "Valeur du seuil (A ou kV)", "Valeur calculée (A ou kV)", "Charge (%)", "Surcharge", "Côté"))
-                .enumValueTranslations(ENUM_TRANSLATIONS_FR)
-                .language("fr")
-                .build());
+        checkZippedCsvResult("n-result", "/results/n-result-fr.csv", "fr");
 
         SQLStatementCountValidator.reset();
-        checkZippedCsvResult("nmk-contingencies-result", "/results/nmk-contingencies-result-en.csv",
-            CsvTranslationDTO.builder()
-                .headers(List.of("Contingency ID", "Status", "Constraint", "Bus", "Violation type", "Limit name", "Limit value (A or kV)", "Calculated value (A or kV)", "Load (%)", "Overload", "Side"))
-                .enumValueTranslations(ENUM_TRANSLATIONS_EN)
-                .language("en")
-                .build());
+        checkZippedCsvResult("nmk-contingencies-result", "/results/nmk-contingencies-result-en.csv", "en");
+
         /*
          * SELECT
          * assert result exists
@@ -896,21 +928,11 @@ private void checkAllZippedCsvResults() throws Exception {
         assertRequestsCount(4, 0, 0, 0);
 
         SQLStatementCountValidator.reset();
-        checkZippedCsvResult("nmk-contingencies-result", "/results/nmk-contingencies-result-fr.csv",
-            CsvTranslationDTO.builder()
-                .headers(List.of("Id aléa", "Statut", "Contrainte", "Noeud électrique", "Type de contrainte", "Nom du seuil", "Valeur du seuil (A ou kV)", "Charge (%)", "Surcharge", "Côté"))
-                .enumValueTranslations(ENUM_TRANSLATIONS_FR)
-                .language("fr")
-                .build());
+        checkZippedCsvResult("nmk-contingencies-result", "/results/nmk-contingencies-result-fr.csv", "fr");
         assertRequestsCount(4, 0, 0, 0);
 
         SQLStatementCountValidator.reset();
-        checkZippedCsvResult("nmk-constraints-result", "/results/nmk-constraints-result-en.csv",
-            CsvTranslationDTO.builder()
-                .headers(List.of("Constraint", "Contingency ID", "Status", "Bus", "Violation type", "Limit name", "Limit value (A or kV)", "Calculated value (A or kV)", "Load (%)", "Overload", "Side"))
-                .enumValueTranslations(ENUM_TRANSLATIONS_EN)
-                .language("en")
-                .build());
+        checkZippedCsvResult("nmk-constraints-result", "/results/nmk-constraints-result-en.csv", "en");
         /*
          * SELECT
          * assert result exists
@@ -921,47 +943,55 @@ private void checkAllZippedCsvResults() throws Exception {
         assertRequestsCount(4, 0, 0, 0);
 
         SQLStatementCountValidator.reset();
-        checkZippedCsvResult("nmk-constraints-result", "/results/nmk-constraints-result-fr.csv",
-            CsvTranslationDTO.builder()
-                .headers(List.of("Contrainte", "ID aléa", "Statut", "Noeud électrique", "Type de contrainte", "Nom du seuil", "Valeur du seuil (A ou kV)", "Valeur calculée (A ou kV)", "Charge (%)", "Surcharge", "Côté"))
-                .enumValueTranslations(ENUM_TRANSLATIONS_FR)
-                .language("fr")
-                .build());
+        checkZippedCsvResult("nmk-constraints-result", "/results/nmk-constraints-result-fr.csv", "fr");
         assertRequestsCount(4, 0, 0, 0);
     }
 
-    private void checkZippedCsvResult(String resultType, String resourcePath, CsvTranslationDTO csvTranslationDTO) throws Exception {
-        // get csv file
-        byte[] resultAsByteArray = mockMvc.perform(post("/" + VERSION + "/results/" + RESULT_UUID + "/" + resultType + "/csv")
-                .contentType(MediaType.APPLICATION_JSON)
-                .content(mapper.writeValueAsString(csvTranslationDTO)))
-            .andExpectAll(
-                status().isOk(),
-                content().contentType(APPLICATION_OCTET_STREAM_VALUE)
-            ).andReturn().getResponse().getContentAsByteArray();
-
+    private void checkCsvResultFromBytes(String expectedCsvResource, byte[] resultAsByteArray) throws Exception {
         // get zip file stream
         try (ZipInputStream zin = new ZipInputStream(new ByteArrayInputStream(resultAsByteArray));
-             ByteArrayOutputStream contentOutputStream = new ByteArrayOutputStream();
-             ByteArrayOutputStream expectedContentOutputStream = new ByteArrayOutputStream()) {
+            ByteArrayOutputStream contentOutputStream = new ByteArrayOutputStream();
+            ByteArrayOutputStream expectedContentOutputStream = new ByteArrayOutputStream()) {
             // get first entry
             ZipEntry zipEntry = zin.getNextEntry();
             // check zip entry name
             assertEquals(CsvExportUtils.CSV_RESULT_FILE_NAME, zipEntry.getName());
-
             // get entry content as outputStream
             StreamUtils.copy(zin, contentOutputStream);
 
             // get expected content as outputStream
-            InputStream csvStream = getClass().getResourceAsStream(resourcePath);
+            InputStream csvStream = getClass().getResourceAsStream(expectedCsvResource);
             StreamUtils.copy(csvStream, expectedContentOutputStream);
 
+            // For debug
+            LOGGER.info("CSV result  :\n {}", contentOutputStream);
+            LOGGER.info("CSV expected:\n {}", expectedContentOutputStream);
+
             // using bytearray comparison to check BOM presence in CSV files
-            Assertions.assertThat(expectedContentOutputStream.toByteArray()).isEqualTo(contentOutputStream.toByteArray());
+            Assertions.assertThat(contentOutputStream.toByteArray()).isEqualTo(expectedContentOutputStream.toByteArray());
             zin.closeEntry();
         }
     }
 
+    private void checkZippedCsvResult(String resultType, String expectedCsvResource, String lang) throws Exception {
+        CsvTranslationDTO csvTranslationDTO = CsvTranslationDTO.builder()
+                .headers(getCsvHeaderFromResource(expectedCsvResource, lang))
+                .enumValueTranslations("en".equalsIgnoreCase(lang) ? ENUM_TRANSLATIONS_EN : ENUM_TRANSLATIONS_FR)
+                .language(lang)
+                .build();
+
+        // get csv file as binary (zip)
+        byte[] resultAsByteArray = mockMvc.perform(post("/" + VERSION + "/results/" + RESULT_UUID + "/" + resultType + "/csv")
+                .contentType(MediaType.APPLICATION_JSON)
+                .content(mapper.writeValueAsString(csvTranslationDTO)))
+            .andExpectAll(
+                status().isOk(),
+                content().contentType(APPLICATION_OCTET_STREAM_VALUE)
+            ).andReturn().getResponse().getContentAsByteArray();
+
+        checkCsvResultFromBytes(expectedCsvResource, resultAsByteArray);
+    }
+
     private void assertResultNotFound(UUID resultUuid) throws Exception {
         mockMvc.perform(get("/" + VERSION + "/results/" + resultUuid + "/n-result"))
                 .andExpect(status().isNotFound());

src/test/java/org/gridsuite/securityanalysis/server/util/TestUtils.java

@@ -5,7 +5,16 @@
 import org.apache.commons.text.StringSubstitutor;
 import org.mockito.ArgumentCaptor;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
 
@@ -59,4 +68,25 @@ private static Optional<String> getMessageFromReporter(String reportKey, ReportN
     private static String formatReportMessage(ReportNode report, ReportNode reporterModel) {
         return new StringSubstitutor(reporterModel.getValues()).replace(new StringSubstitutor(report.getValues()).replace(report.getMessageTemplate()));
     }
+
+    public static List<String> readLinesFromFilePath(String resourceFileName, int nbLines) {
+        final String utf8Bom = "\uFEFF";
+        List<String> lines = new ArrayList<>();
+        try {
+            Path resourceFilePath = Paths.get("src", "test", "resources", resourceFileName);
+            InputStream inputStream = Files.newInputStream(resourceFilePath);
+            try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) {
+                String line;
+                while ((line = br.readLine()) != null && (nbLines == -1 || lines.size() < nbLines)) {
+                    if (lines.isEmpty() && line.startsWith(utf8Bom)) {
+                        line = line.substring(utf8Bom.length()); // skip BOM
+                    }
+                    lines.add(line);
+                }
+            }
+        } catch (IOException ex) {
+            return List.of();
+        }
+        return lines;
+    }
 }

src/test/resources/results/n-result-en.csv

@@ -1,4 +1,4 @@
-Equipment,Violation type,Limit name,Limit value (A or kV),Calculated value (A or kV),Load (%),Overload,Side
-,Current,l3_name,10,11,110,,Side 1
-,Current,l6_name,10,11,110,1200,Side 1
-"vl1 (VLGEN_0, VLLOAD_0)",High voltage,IST,400,410,,0,
+Equipment,Violation type,Bus,Limit name,Load (% limit),Load (% PATL),Actual overload,Upcoming overload,Next limit name,Limit (A or kV),PATL (A),Calculated value (A or kV),Side
+l3,Current,,l3_name,110,,,,,10,,11,Side 1
+l6,Current,,l6_name,110,,1200,,,10,,11,Side 1
+vl1,High voltage,"vl1 (VLGEN_0, VLLOAD_0)",IST,,,0,,,400,,410,

src/test/resources/results/n-result-fr-custom.csv

@@ -0,0 +1,3 @@
+Ouvrage;Type de contrainte;Noeud électrique;Nom de la limite;Charge (% limite);Charge (% IST);Tempo effective;Tempo imminente;Nom de la limite suivante;Limite (A ou kV);IST (A);Valeur calculée (A ou kV);Côté
+Ouvrage;;BUS;lim_name;;111,5;100;;;0;80,66;0;
+Ouvrage2;;BUS2;lim_name2;;111,5;100;1000;;0;80,66;0;

src/test/resources/results/n-result-fr.csv

@@ -1,4 +1,4 @@
-Ouvrage;Type de contrainte;Nom du seuil;Valeur du seuil (A ou kV);Valeur calculée (A ou kV);Charge (%);Surcharge;Côté
-;Intensité;l3_name;10;11;110;;Côté 1
-;Intensité;l6_name;10;11;110;1200;Côté 1
-vl1 (VLGEN_0, VLLOAD_0);Tension haute;IST;400;410;;0;
+Ouvrage;Type de contrainte;Noeud électrique;Nom de la limite;Charge (% limite);Charge (% IST);Tempo effective;Tempo imminente;Nom de la limite suivante;Limite (A ou kV);IST (A);Valeur calculée (A ou kV);Côté
+l3;Intensité;;l3_name;110;;;;;10;;11;Côté 1
+l6;Intensité;;l6_name;110;;1200;;;10;;11;Côté 1
+vl1;Tension haute;vl1 (VLGEN_0, VLLOAD_0);IST;;;0;;;400;;410;