Skip to content

Commit 33dd337

Browse files
dbraquartdbraquart
authored
Fix missing columns during csv export (#207)
Signed-off-by: David BRAQUART <[email protected]>
1 parent 5254056 commit 33dd337

File tree

11 files changed

+240
-172
lines changed

11 files changed

+240
-172
lines changed

‎src/main/java/org/gridsuite/securityanalysis/server/dto/LimitViolationDTO.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,17 @@ private static String convertDoubleToLocale(Double value, String language) {
6060

6161
public List<String> toCsvRow(Map<String, String> translations, String language) {
6262
List<String> csvRow = new ArrayList<>();
63-
csvRow.add(this.getLocationId());
6463
csvRow.add(this.getLimitType() != null ? CsvExportUtils.translate(this.getLimitType().name(), translations) : "");
64+
csvRow.add(this.getLocationId()); // busId
6565
csvRow.add(CsvExportUtils.replaceNullWithEmptyString(CsvExportUtils.translate(this.getLimitName(), translations)));
66-
csvRow.add(convertDoubleToLocale(this.getLimit(), language));
67-
csvRow.add(convertDoubleToLocale(this.getValue(), language));
6866
csvRow.add(this.getLoading() == null ? "" : convertDoubleToLocale(this.getLoading(), language));
67+
csvRow.add(this.getPatlLoading() == null ? "" : convertDoubleToLocale(this.getPatlLoading(), language));
6968
csvRow.add(this.getAcceptableDuration() == Integer.MAX_VALUE ? null : Integer.toString(this.getAcceptableDuration()));
69+
csvRow.add(this.getUpcomingAcceptableDuration() == null || this.getUpcomingAcceptableDuration() == Integer.MAX_VALUE ? null : Integer.toString(this.getUpcomingAcceptableDuration()));
70+
csvRow.add(CsvExportUtils.replaceNullWithEmptyString(CsvExportUtils.translate(this.getNextLimitName(), translations)));
71+
csvRow.add(convertDoubleToLocale(this.getLimit(), language));
72+
csvRow.add(this.getPatlLimit() == null ? "" : convertDoubleToLocale(this.getPatlLimit(), language));
73+
csvRow.add(convertDoubleToLocale(this.getValue(), language));
7074
csvRow.add(this.getSide() != null ? CsvExportUtils.translate(this.getSide().name(), translations) : "");
7175
return csvRow;
7276
}

‎src/main/java/org/gridsuite/securityanalysis/server/dto/PreContingencyLimitViolationResultDTO.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ public List<String> toCsvRow(Map<String, String> translations, String language)
4444
List<String> csvRow = List.of();
4545

4646
if (this.getLimitViolation() != null) {
47+
csvRow = List.of(this.getSubjectId());
4748
return Stream.concat(csvRow.stream(), this.getLimitViolation().toCsvRow(translations, language).stream()).toList();
4849
}
4950

‎src/test/java/org/gridsuite/securityanalysis/server/SecurityAnalysisControllerTest.java‎

Lines changed: 83 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@
4444
import org.junit.jupiter.api.BeforeEach;
4545
import org.junit.jupiter.api.Test;
4646
import org.mockito.MockitoAnnotations;
47+
import org.slf4j.Logger;
48+
import org.slf4j.LoggerFactory;
4749
import org.springframework.beans.factory.annotation.Autowired;
4850
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
4951
import org.springframework.boot.test.context.SpringBootTest;
@@ -73,8 +75,10 @@
7375
import static org.gridsuite.computation.service.NotificationService.*;
7476
import static org.gridsuite.securityanalysis.server.SecurityAnalysisProviderMock.*;
7577
import static org.gridsuite.securityanalysis.server.service.SecurityAnalysisService.COMPUTATION_TYPE;
78+
import static org.gridsuite.securityanalysis.server.util.CsvExportUtils.csvRowsToZippedCsv;
7679
import static org.gridsuite.securityanalysis.server.util.DatabaseQueryUtils.assertRequestsCount;
7780
import static org.gridsuite.securityanalysis.server.util.TestUtils.assertLogMessage;
81+
import static org.gridsuite.securityanalysis.server.util.TestUtils.readLinesFromFilePath;
7882
import static org.hamcrest.MatcherAssert.assertThat;
7983
import static org.junit.jupiter.api.Assertions.*;
8084
import static org.mockito.ArgumentMatchers.any;
@@ -93,6 +97,7 @@
9397
@SpringBootTest
9498
@ContextConfigurationWithTestChannel
9599
class SecurityAnalysisControllerTest {
100+
private static final Logger LOGGER = LoggerFactory.getLogger(SecurityAnalysisControllerTest.class);
96101

97102
private static final UUID NETWORK_UUID = UUID.fromString("7928181c-7977-4592-ba19-88027e4254e4");
98103
private static final UUID NETWORK_STOP_UUID = UUID.fromString("7928181c-7977-4592-ba19-88027e4254e6");
@@ -512,7 +517,7 @@ private void checkNResultEnumFilters(UUID resultUuid) throws Exception {
512517
List<LimitViolationType> limitTypes = mapper.readValue(mvcResult.getResponse().getContentAsString(), new TypeReference<>() { });
513518
Assertions.assertThat(limitTypes).hasSameElementsAs(expectedLimitTypes);
514519

515-
List<ThreeSides> expectedSides = nResults.stream().map(result -> result.getLimitViolation().getSide()).distinct().filter(side -> side != null).toList();
520+
List<ThreeSides> expectedSides = nResults.stream().map(result -> result.getLimitViolation().getSide()).distinct().filter(Objects::nonNull).toList();
516521
mvcResult = mockMvc.perform(get("/" + VERSION + "/results/{resultUuid}/n-branch-sides", resultUuid))
517522
.andExpectAll(
518523
status().isOk(),
@@ -835,7 +840,7 @@ void getDefaultProviderTest() throws Exception {
835840

836841
@Test
837842
void getZippedCsvResults() throws Exception {
838-
// running computation to create result
843+
// running computation to create some results
839844
MvcResult mvcResult = mockMvc.perform(post("/" + VERSION + "/networks/" + NETWORK_UUID + "/run-and-save?reportType=SecurityAnalysis&contingencyListName=" + CONTINGENCY_LIST_NAME
840845
+ "&receiver=me&variantId=" + VARIANT_2_ID + "&provider=OpenLoadFlow" + "&loadFlowParametersUuid=" + UUID.randomUUID())
841846
.header(HEADER_USER_ID, "testUserId")
@@ -856,14 +861,50 @@ void getZippedCsvResults() throws Exception {
856861
checkAllZippedCsvResults();
857862
}
858863

864+
@Test
865+
void getZippedCsvResultsFromCustomData() throws Exception {
866+
final String expectedCsvFile = "/results/n-result-fr-custom.csv";
867+
final String lang = "fr";
868+
final List<String> header = getCsvHeaderFromResource(expectedCsvFile, lang);
869+
870+
// Build binary/zipped data from custom DTOs rather than from AS results.
871+
PreContingencyLimitViolationResultDTO dto = PreContingencyLimitViolationResultDTO.builder()
872+
.subjectId("Ouvrage")
873+
.limitViolation(LimitViolationDTO.builder()
874+
.locationId("BUS")
875+
.limitName("lim_name")
876+
.acceptableDuration(100)
877+
.patlLoading(111.5)
878+
.patlLimit(80.66)
879+
.upcomingAcceptableDuration(Integer.MAX_VALUE)
880+
.build())
881+
.build();
882+
PreContingencyLimitViolationResultDTO dto2 = PreContingencyLimitViolationResultDTO.builder()
883+
.subjectId("Ouvrage2")
884+
.limitViolation(LimitViolationDTO.builder()
885+
.locationId("BUS2")
886+
.limitName("lim_name2")
887+
.acceptableDuration(100)
888+
.patlLoading(111.5)
889+
.patlLimit(80.66)
890+
.upcomingAcceptableDuration(1000)
891+
.build())
892+
.build();
893+
List<List<String>> csvRows = List.of(dto.toCsvRow(ENUM_TRANSLATIONS_FR, lang), dto2.toCsvRow(ENUM_TRANSLATIONS_FR, lang));
894+
byte[] resultAsByteArray = csvRowsToZippedCsv(header, lang, csvRows);
895+
// and compare with the expected file
896+
checkCsvResultFromBytes(expectedCsvFile, resultAsByteArray);
897+
}
898+
899+
private List<String> getCsvHeaderFromResource(String resourcePath, String lang) {
900+
List<String> lines = readLinesFromFilePath(resourcePath, 1);
901+
String header = lines.isEmpty() ? "" : lines.getFirst();
902+
return Arrays.asList(header.strip().split("en".equalsIgnoreCase(lang) ? "," : ";"));
903+
}
904+
859905
private void checkAllZippedCsvResults() throws Exception {
860906
SQLStatementCountValidator.reset();
861-
checkZippedCsvResult("n-result", "/results/n-result-en.csv",
862-
CsvTranslationDTO.builder()
863-
.headers(List.of("Equipment", "Violation type", "Limit name", "Limit value (A or kV)", "Calculated value (A or kV)", "Load (%)", "Overload", "Side"))
864-
.enumValueTranslations(ENUM_TRANSLATIONS_EN)
865-
.language("en")
866-
.build());
907+
checkZippedCsvResult("n-result", "/results/n-result-en.csv", "en");
867908
/*
868909
* SELECT
869910
* assert result exists
@@ -872,20 +913,11 @@ private void checkAllZippedCsvResults() throws Exception {
872913
assertRequestsCount(2, 0, 0, 0);
873914

874915
SQLStatementCountValidator.reset();
875-
checkZippedCsvResult("n-result", "/results/n-result-fr.csv",
876-
CsvTranslationDTO.builder()
877-
.headers(List.of("Ouvrage", "Type de contrainte", "Nom du seuil", "Valeur du seuil (A ou kV)", "Valeur calculée (A ou kV)", "Charge (%)", "Surcharge", "Côté"))
878-
.enumValueTranslations(ENUM_TRANSLATIONS_FR)
879-
.language("fr")
880-
.build());
916+
checkZippedCsvResult("n-result", "/results/n-result-fr.csv", "fr");
881917

882918
SQLStatementCountValidator.reset();
883-
checkZippedCsvResult("nmk-contingencies-result", "/results/nmk-contingencies-result-en.csv",
884-
CsvTranslationDTO.builder()
885-
.headers(List.of("Contingency ID", "Status", "Constraint", "Bus", "Violation type", "Limit name", "Limit value (A or kV)", "Calculated value (A or kV)", "Load (%)", "Overload", "Side"))
886-
.enumValueTranslations(ENUM_TRANSLATIONS_EN)
887-
.language("en")
888-
.build());
919+
checkZippedCsvResult("nmk-contingencies-result", "/results/nmk-contingencies-result-en.csv", "en");
920+
889921
/*
890922
* SELECT
891923
* assert result exists
@@ -896,21 +928,11 @@ private void checkAllZippedCsvResults() throws Exception {
896928
assertRequestsCount(4, 0, 0, 0);
897929

898930
SQLStatementCountValidator.reset();
899-
checkZippedCsvResult("nmk-contingencies-result", "/results/nmk-contingencies-result-fr.csv",
900-
CsvTranslationDTO.builder()
901-
.headers(List.of("Id aléa", "Statut", "Contrainte", "Noeud électrique", "Type de contrainte", "Nom du seuil", "Valeur du seuil (A ou kV)", "Charge (%)", "Surcharge", "Côté"))
902-
.enumValueTranslations(ENUM_TRANSLATIONS_FR)
903-
.language("fr")
904-
.build());
931+
checkZippedCsvResult("nmk-contingencies-result", "/results/nmk-contingencies-result-fr.csv", "fr");
905932
assertRequestsCount(4, 0, 0, 0);
906933

907934
SQLStatementCountValidator.reset();
908-
checkZippedCsvResult("nmk-constraints-result", "/results/nmk-constraints-result-en.csv",
909-
CsvTranslationDTO.builder()
910-
.headers(List.of("Constraint", "Contingency ID", "Status", "Bus", "Violation type", "Limit name", "Limit value (A or kV)", "Calculated value (A or kV)", "Load (%)", "Overload", "Side"))
911-
.enumValueTranslations(ENUM_TRANSLATIONS_EN)
912-
.language("en")
913-
.build());
935+
checkZippedCsvResult("nmk-constraints-result", "/results/nmk-constraints-result-en.csv", "en");
914936
/*
915937
* SELECT
916938
* assert result exists
@@ -921,47 +943,55 @@ private void checkAllZippedCsvResults() throws Exception {
921943
assertRequestsCount(4, 0, 0, 0);
922944

923945
SQLStatementCountValidator.reset();
924-
checkZippedCsvResult("nmk-constraints-result", "/results/nmk-constraints-result-fr.csv",
925-
CsvTranslationDTO.builder()
926-
.headers(List.of("Contrainte", "ID aléa", "Statut", "Noeud électrique", "Type de contrainte", "Nom du seuil", "Valeur du seuil (A ou kV)", "Valeur calculée (A ou kV)", "Charge (%)", "Surcharge", "Côté"))
927-
.enumValueTranslations(ENUM_TRANSLATIONS_FR)
928-
.language("fr")
929-
.build());
946+
checkZippedCsvResult("nmk-constraints-result", "/results/nmk-constraints-result-fr.csv", "fr");
930947
assertRequestsCount(4, 0, 0, 0);
931948
}
932949

933-
private void checkZippedCsvResult(String resultType, String resourcePath, CsvTranslationDTO csvTranslationDTO) throws Exception {
934-
// get csv file
935-
byte[] resultAsByteArray = mockMvc.perform(post("/" + VERSION + "/results/" + RESULT_UUID + "/" + resultType + "/csv")
936-
.contentType(MediaType.APPLICATION_JSON)
937-
.content(mapper.writeValueAsString(csvTranslationDTO)))
938-
.andExpectAll(
939-
status().isOk(),
940-
content().contentType(APPLICATION_OCTET_STREAM_VALUE)
941-
).andReturn().getResponse().getContentAsByteArray();
942-
950+
private void checkCsvResultFromBytes(String expectedCsvResource, byte[] resultAsByteArray) throws Exception {
943951
// get zip file stream
944952
try (ZipInputStream zin = new ZipInputStream(new ByteArrayInputStream(resultAsByteArray));
945-
ByteArrayOutputStream contentOutputStream = new ByteArrayOutputStream();
946-
ByteArrayOutputStream expectedContentOutputStream = new ByteArrayOutputStream()) {
953+
ByteArrayOutputStream contentOutputStream = new ByteArrayOutputStream();
954+
ByteArrayOutputStream expectedContentOutputStream = new ByteArrayOutputStream()) {
947955
// get first entry
948956
ZipEntry zipEntry = zin.getNextEntry();
949957
// check zip entry name
950958
assertEquals(CsvExportUtils.CSV_RESULT_FILE_NAME, zipEntry.getName());
951-
952959
// get entry content as outputStream
953960
StreamUtils.copy(zin, contentOutputStream);
954961

955962
// get expected content as outputStream
956-
InputStream csvStream = getClass().getResourceAsStream(resourcePath);
963+
InputStream csvStream = getClass().getResourceAsStream(expectedCsvResource);
957964
StreamUtils.copy(csvStream, expectedContentOutputStream);
958965

966+
// For debug
967+
LOGGER.info("CSV result :\n {}", contentOutputStream);
968+
LOGGER.info("CSV expected:\n {}", expectedContentOutputStream);
969+
959970
// using bytearray comparison to check BOM presence in CSV files
960-
Assertions.assertThat(expectedContentOutputStream.toByteArray()).isEqualTo(contentOutputStream.toByteArray());
971+
Assertions.assertThat(contentOutputStream.toByteArray()).isEqualTo(expectedContentOutputStream.toByteArray());
961972
zin.closeEntry();
962973
}
963974
}
964975

976+
private void checkZippedCsvResult(String resultType, String expectedCsvResource, String lang) throws Exception {
977+
CsvTranslationDTO csvTranslationDTO = CsvTranslationDTO.builder()
978+
.headers(getCsvHeaderFromResource(expectedCsvResource, lang))
979+
.enumValueTranslations("en".equalsIgnoreCase(lang) ? ENUM_TRANSLATIONS_EN : ENUM_TRANSLATIONS_FR)
980+
.language(lang)
981+
.build();
982+
983+
// get csv file as binary (zip)
984+
byte[] resultAsByteArray = mockMvc.perform(post("/" + VERSION + "/results/" + RESULT_UUID + "/" + resultType + "/csv")
985+
.contentType(MediaType.APPLICATION_JSON)
986+
.content(mapper.writeValueAsString(csvTranslationDTO)))
987+
.andExpectAll(
988+
status().isOk(),
989+
content().contentType(APPLICATION_OCTET_STREAM_VALUE)
990+
).andReturn().getResponse().getContentAsByteArray();
991+
992+
checkCsvResultFromBytes(expectedCsvResource, resultAsByteArray);
993+
}
994+
965995
private void assertResultNotFound(UUID resultUuid) throws Exception {
966996
mockMvc.perform(get("/" + VERSION + "/results/" + resultUuid + "/n-result"))
967997
.andExpect(status().isNotFound());

‎src/test/java/org/gridsuite/securityanalysis/server/util/TestUtils.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,16 @@
55
import org.apache.commons.text.StringSubstitutor;
66
import org.mockito.ArgumentCaptor;
77

8+
import java.io.BufferedReader;
9+
import java.io.IOException;
10+
import java.io.InputStream;
11+
import java.io.InputStreamReader;
12+
import java.nio.file.Files;
13+
import java.nio.file.Path;
14+
import java.nio.file.Paths;
15+
import java.util.ArrayList;
816
import java.util.Iterator;
17+
import java.util.List;
918
import java.util.Optional;
1019
import java.util.UUID;
1120

@@ -59,4 +68,25 @@ private static Optional<String> getMessageFromReporter(String reportKey, ReportN
5968
private static String formatReportMessage(ReportNode report, ReportNode reporterModel) {
6069
return new StringSubstitutor(reporterModel.getValues()).replace(new StringSubstitutor(report.getValues()).replace(report.getMessageTemplate()));
6170
}
71+
72+
public static List<String> readLinesFromFilePath(String resourceFileName, int nbLines) {
73+
final String utf8Bom = "\uFEFF";
74+
List<String> lines = new ArrayList<>();
75+
try {
76+
Path resourceFilePath = Paths.get("src", "test", "resources", resourceFileName);
77+
InputStream inputStream = Files.newInputStream(resourceFilePath);
78+
try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) {
79+
String line;
80+
while ((line = br.readLine()) != null && (nbLines == -1 || lines.size() < nbLines)) {
81+
if (lines.isEmpty() && line.startsWith(utf8Bom)) {
82+
line = line.substring(utf8Bom.length()); // skip BOM
83+
}
84+
lines.add(line);
85+
}
86+
}
87+
} catch (IOException ex) {
88+
return List.of();
89+
}
90+
return lines;
91+
}
6292
}

‎src/test/resources/results/n-result-en.csv‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Equipment,Violation type,Limit name,Limit value (A or kV),Calculated value (A or kV),Load (%),Overload,Side
2-
,Current,l3_name,10,11,110,,Side 1
3-
,Current,l6_name,10,11,110,1200,Side 1
4-
"vl1 (VLGEN_0, VLLOAD_0)",High voltage,IST,400,410,,0,
1+
Equipment,Violation type,Bus,Limit name,Load (% limit),Load (% PATL),Actual overload,Upcoming overload,Next limit name,Limit (A or kV),PATL (A),Calculated value (A or kV),Side
2+
l3,Current,,l3_name,110,,,,,10,,11,Side 1
3+
l6,Current,,l6_name,110,,1200,,,10,,11,Side 1
4+
vl1,High voltage,"vl1 (VLGEN_0, VLLOAD_0)",IST,,,0,,,400,,410,

‎src/test/resources/results/n-result-fr-custom.csv‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Ouvrage;Type de contrainte;Noeud électrique;Nom de la limite;Charge (% limite);Charge (% IST);Tempo effective;Tempo imminente;Nom de la limite suivante;Limite (A ou kV);IST (A);Valeur calculée (A ou kV);Côté
2+
Ouvrage;;BUS;lim_name;;111,5;100;;;0;80,66;0;
3+
Ouvrage2;;BUS2;lim_name2;;111,5;100;1000;;0;80,66;0;

‎src/test/resources/results/n-result-fr.csv‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Ouvrage;Type de contrainte;Nom du seuil;Valeur du seuil (A ou kV);Valeur calculée (A ou kV);Charge (%);Surcharge;Côté
2-
;Intensité;l3_name;10;11;110;;Côté 1
3-
;Intensité;l6_name;10;11;110;1200;Côté 1
4-
vl1 (VLGEN_0, VLLOAD_0);Tension haute;IST;400;410;;0;
1+
Ouvrage;Type de contrainte;Noeud électrique;Nom de la limite;Charge (% limite);Charge (% IST);Tempo effective;Tempo imminente;Nom de la limite suivante;Limite (A ou kV);IST (A);Valeur calculée (A ou kV);Côté
2+
l3;Intensité;;l3_name;110;;;;;10;;11;Côté 1
3+
l6;Intensité;;l6_name;110;;1200;;;10;;11;Côté 1
4+
vl1;Tension haute;vl1 (VLGEN_0, VLLOAD_0);IST;;;0;;;400;;410;

0 commit comments

Comments
 (0)