From c218a33938d3d28e0d3e2f0d4b4b00a204c59378 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Thu, 31 Jul 2025 14:42:01 +0530 Subject: [PATCH 1/2] CSV header optional bug fixed --- .../core/dataexport/CsvExportManager.java | 6 +- .../core/dataexport/CsvExportManagerTest.java | 69 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java index 64f059852e..0e95b2586d 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java @@ -39,8 +39,10 @@ public CsvExportManager( void processHeader(ExportOptions exportOptions, TableMetadata tableMetadata, Writer writer) throws IOException { String header = createCsvHeaderRow(exportOptions, tableMetadata); - writer.append(header); - writer.flush(); + if (!exportOptions.isExcludeHeaderRow()) { + writer.append(header); + writer.flush(); + } } /** diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java index ca65c10010..2944500f7b 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java @@ -14,6 +14,7 @@ import com.scalar.db.io.Column; import com.scalar.db.io.IntColumn; import com.scalar.db.io.Key; +import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.IOException; @@ -129,4 +130,72 @@ void startExport_givenPartitionKey_shouldGenerateOutputFile() Assertions.assertTrue(file.exists()); Assertions.assertTrue(file.delete()); } + + @Test + void startExport_givenNoHeaderRequired_shouldGenerateOutputFileWithoutHeader() throws Exception { + String expectedFirstLine = + "9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,2000-01-01,01:01:01,2000-01-01T01:01,1970-01-21T03:20:41.740Z"; + + runExportAndAssertFirstLine(true, expectedFirstLine); + } + + @Test + void startExport_givenHeaderRequired_shouldGenerateOutputFileWithHeader() throws Exception { + String expectedFirstLine = "col1,col2,col3,col4,col5,col6,col7,col8,col9,col10,col11"; + + runExportAndAssertFirstLine(false, expectedFirstLine); + } + + private void runExportAndAssertFirstLine(boolean includeHeader, String expectedFirstLine) + throws Exception { + // Arrange + producerTaskFactory = new ProducerTaskFactory(",", false, false); + exportManager = new CsvExportManager(storage, dao, producerTaskFactory); + Scanner scanner = Mockito.mock(Scanner.class); + String filePath = Paths.get("").toAbsolutePath() + "/output.csv"; + Map> values = UnitTestUtils.createTestValues(); + Result result = new ResultImpl(values, mockData); + List results = Collections.singletonList(result); + + ExportOptions exportOptions = + ExportOptions.builder( + "namespace", + "table", + Key.newBuilder().add(IntColumn.of("col1", 1)).build(), + FileFormat.CSV) + .sortOrders(Collections.emptyList()) + .scanRange(new ScanRange(null, null, false, false)) + .delimiter(",") + .excludeHeaderRow(includeHeader) + .build(); + + Mockito.when( + dao.createScanner( + exportOptions.getNamespace(), + exportOptions.getTableName(), + exportOptions.getScanPartitionKey(), + exportOptions.getScanRange(), + exportOptions.getSortOrders(), + exportOptions.getProjectionColumns(), + exportOptions.getLimit(), + storage)) + .thenReturn(scanner); + Mockito.when(scanner.iterator()).thenReturn(results.iterator()); + try (BufferedWriter writer = + new BufferedWriter( + Files.newBufferedWriter( + Paths.get(filePath), + Charset.defaultCharset(), + StandardOpenOption.CREATE, + StandardOpenOption.APPEND))) { + exportManager.startExport(exportOptions, mockData, writer); + } + File file = new File(filePath); + Assertions.assertTrue(file.exists()); + try (BufferedReader br = Files.newBufferedReader(file.toPath(), Charset.defaultCharset())) { + String firstLine = br.readLine(); + Assertions.assertEquals(expectedFirstLine, firstLine); + } + Assertions.assertTrue(file.delete()); + } } From 6af47f2bd4e0adaf4b56380ddf28ce700e2bf43a Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Thu, 31 Jul 2025 16:35:43 +0530 Subject: [PATCH 2/2] Suggestions applied --- .../core/dataexport/CsvExportManager.java | 2 +- .../core/dataexport/CsvExportManagerTest.java | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java index 0e95b2586d..bccf691adc 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java @@ -38,8 +38,8 @@ public CsvExportManager( @Override void processHeader(ExportOptions exportOptions, TableMetadata tableMetadata, Writer writer) throws IOException { - String header = createCsvHeaderRow(exportOptions, tableMetadata); if (!exportOptions.isExcludeHeaderRow()) { + String header = createCsvHeaderRow(exportOptions, tableMetadata); writer.append(header); writer.flush(); } diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java index 2944500f7b..c61aa6f989 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java @@ -146,7 +146,7 @@ void startExport_givenHeaderRequired_shouldGenerateOutputFileWithHeader() throws runExportAndAssertFirstLine(false, expectedFirstLine); } - private void runExportAndAssertFirstLine(boolean includeHeader, String expectedFirstLine) + private void runExportAndAssertFirstLine(boolean excludeHeader, String expectedFirstLine) throws Exception { // Arrange producerTaskFactory = new ProducerTaskFactory(",", false, false); @@ -166,7 +166,7 @@ private void runExportAndAssertFirstLine(boolean includeHeader, String expectedF .sortOrders(Collections.emptyList()) .scanRange(new ScanRange(null, null, false, false)) .delimiter(",") - .excludeHeaderRow(includeHeader) + .excludeHeaderRow(excludeHeader) .build(); Mockito.when( @@ -182,12 +182,11 @@ private void runExportAndAssertFirstLine(boolean includeHeader, String expectedF .thenReturn(scanner); Mockito.when(scanner.iterator()).thenReturn(results.iterator()); try (BufferedWriter writer = - new BufferedWriter( - Files.newBufferedWriter( - Paths.get(filePath), - Charset.defaultCharset(), - StandardOpenOption.CREATE, - StandardOpenOption.APPEND))) { + Files.newBufferedWriter( + Paths.get(filePath), + Charset.defaultCharset(), + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING)) { exportManager.startExport(exportOptions, mockData, writer); } File file = new File(filePath); @@ -195,7 +194,8 @@ private void runExportAndAssertFirstLine(boolean includeHeader, String expectedF try (BufferedReader br = Files.newBufferedReader(file.toPath(), Charset.defaultCharset())) { String firstLine = br.readLine(); Assertions.assertEquals(expectedFirstLine, firstLine); + } finally { + Assertions.assertTrue(file.delete()); } - Assertions.assertTrue(file.delete()); } }