Skip to content

Commit 3566732

Browse files
JohannesJanderJohannesJander
committed
API extension: asCSV() and asJSON() methods on Table
1 parent ec9caea commit 3566732

File tree

5 files changed

+135
-29
lines changed

5 files changed

+135
-29
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>io.frictionlessdata</groupId>
55
<artifactId>tableschema-java</artifactId>
6-
<version>0.8.0-SNAPSHOT</version>
6+
<version>0.8.1-SNAPSHOT</version>
77
<packaging>jar</packaging>
88
<issueManagement>
99
<url>https://github.com/frictionlessdata/tableschema-java/issues</url>

‎src/main/java/io/frictionlessdata/tableschema/Table.java‎

Lines changed: 90 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package io.frictionlessdata.tableschema;
22

3+
import com.fasterxml.jackson.core.JsonProcessingException;
4+
import com.fasterxml.jackson.databind.ObjectMapper;
35
import io.frictionlessdata.tableschema.exception.*;
46
import io.frictionlessdata.tableschema.field.Field;
57
import io.frictionlessdata.tableschema.iterator.BeanIterator;
@@ -411,7 +413,7 @@ public List<Object[]> read(){
411413
/**
412414
* Read all data from the Table and return it as JSON. If no Schema is set on the table, one will be inferred.
413415
* This can be used for smaller data tables but for huge or unknown sizes, there will be performance considerations,
414-
* as this method loads all data into RAM *and* does a costly schema inferral.
416+
* as this method loads all data into RAM *and* does a costly schema inferal.
415417
*
416418
* It ignores relations to other data sources.
417419
*
@@ -434,33 +436,57 @@ public String asJson() {
434436
rows.add(obj);
435437
});
436438

437-
return JsonUtil.getInstance().serialize(rows);
439+
String retVal = null;
440+
ObjectMapper mapper = JsonUtil.getInstance().getMapper();
441+
try {
442+
retVal = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(rows);
443+
} catch (JsonProcessingException ex) {
444+
throw new JsonSerializingException(ex);
445+
}
446+
return retVal;
438447
}
439448

440449
/**
441-
* Write as CSV file, the `format` parameter decides on the CSV options. If it is
442-
* null, then the file will be written as RFC 4180 compliant CSV
443-
* @param out the Writer to write to
450+
* Read all data from the Table and return it as a RFC 4180 compliant CSV string.
451+
* Column order will be deducted from the table data source.
452+
*
453+
* @return A CSV representation of the data as a String.
454+
*/
455+
public String asCsv() {
456+
return asCsv(null, null);
457+
}
458+
459+
/**
460+
* Return the data as a CSV string,
461+
*
462+
* - the `format` parameter decides on the CSV options. If it is null, then the file will
463+
* be written as RFC 4180 compliant CSV
464+
* - the `headerNames` parameter decides on the order of the headers in the CSV file. If it is null,
465+
* the order of the columns will be the same as in the data source.
466+
*
467+
* It ignores relations to other data sources.
468+
*
444469
* @param format the CSV format to use
445-
* @param sortedHeaders the header row names in the order in which data should be
446-
* exported
470+
* @param headerNames the header row names in the order in which data should be exported
471+
*
472+
* @return A CSV representation of the data as a String.
447473
*/
448-
private void writeCsv(Writer out, CSVFormat format, String[] sortedHeaders) {
474+
public String asCsv(CSVFormat format, String[] headerNames) {
475+
StringBuilder out = new StringBuilder();
449476
try {
450-
if (null == sortedHeaders) {
451-
writeCsv(out, format, getHeaders());
452-
return;
477+
if (null == headerNames) {
478+
return asCsv(format, getHeaders());
453479
}
454480
CSVFormat locFormat = (null != format)
455481
? format
456482
: TableDataSource.getDefaultCsvFormat();
457483

458-
locFormat = locFormat.builder().setHeader(sortedHeaders).get();
484+
locFormat = locFormat.builder().setHeader(headerNames).get();
459485
CSVPrinter csvPrinter = new CSVPrinter(out, locFormat);
460486

461487
String[] headers = getHeaders();
462488
Map<Integer, Integer> mapping
463-
= TableSchemaUtil.createSchemaHeaderMapping(headers, sortedHeaders, dataSource.hasReliableHeaders());
489+
= TableSchemaUtil.createSchemaHeaderMapping(headers, headerNames, dataSource.hasReliableHeaders());
464490
if ((null != schema)) {
465491
writeCSVData(mapping, schema, csvPrinter);
466492
} else {
@@ -470,6 +496,14 @@ private void writeCsv(Writer out, CSVFormat format, String[] sortedHeaders) {
470496
} catch (IOException ex) {
471497
throw new TableIOException(ex);
472498
}
499+
String result = out.toString();
500+
if (result.endsWith("\n")) {
501+
result = result.substring(0, result.length() - 1);
502+
}
503+
if (result.endsWith("\r")) {
504+
result = result.substring(0, result.length() - 1);
505+
}
506+
return result;
473507
}
474508

475509
/**
@@ -715,6 +749,49 @@ public int hashCode() {
715749
}
716750

717751

752+
753+
/**
754+
* Write as CSV file, the `format` parameter decides on the CSV options. If it is
755+
* null, then the file will be written as RFC 4180 compliant CSV
756+
* @param out the Writer to write to
757+
* @param format the CSV format to use
758+
* @param sortedHeaders the header row names in the order in which data should be
759+
* exported
760+
*/
761+
private void writeCsv(Writer out, CSVFormat format, String[] sortedHeaders) {
762+
try {
763+
if (null == sortedHeaders) {
764+
writeCsv(out, format, getHeaders());
765+
return;
766+
}
767+
CSVFormat locFormat = (null != format)
768+
? format
769+
: TableDataSource.getDefaultCsvFormat();
770+
771+
locFormat = locFormat.builder().setHeader(sortedHeaders).get();
772+
CSVPrinter csvPrinter = new CSVPrinter(out, locFormat);
773+
774+
String[] headers = getHeaders();
775+
Map<Integer, Integer> mapping
776+
= TableSchemaUtil.createSchemaHeaderMapping(headers, sortedHeaders, dataSource.hasReliableHeaders());
777+
if ((null != schema)) {
778+
writeCSVData(mapping, schema, csvPrinter);
779+
} else {
780+
writeCSVData(mapping, csvPrinter);
781+
}
782+
csvPrinter.close();
783+
} catch (IOException ex) {
784+
throw new TableIOException(ex);
785+
}
786+
}
787+
788+
789+
/**
790+
* Append the data to a {@link org.apache.commons.csv.CSVPrinter}. Column sorting is according to the mapping
791+
* @param mapping the mapping of the column numbers in the CSV file to the column numbers in the data source
792+
* @param schema the Schema to use for formatting the data
793+
* @param csvPrinter the CSVPrinter to write to
794+
*/
718795
private void writeCSVData(Map<Integer, Integer> mapping, Schema schema, CSVPrinter csvPrinter) {
719796
Iterator<Object> iter = this.iterator(false, false, true, false);
720797
iter.forEachRemaining((rec) -> {

‎src/main/java/io/frictionlessdata/tableschema/iterator/TableIterator.java‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,7 @@ public T next() {
109109
}
110110
Map<String, Object> keyedRow = new LinkedHashMap<>();
111111
Object[] extendedRow;
112-
Object[] castRow = new Object[rowLength];
113-
Object[] plainRow = new Object[rowLength];
112+
Object[] resultRow = new Object[rowLength];
114113

115114
// If there's a schema, attempt to cast the row.
116115
if(this.schema != null){
@@ -136,27 +135,24 @@ public T next() {
136135
val = field.castValue(rawVal);
137136
}
138137
}
139-
if (!extended && keyed) {
140-
keyedRow.put(this.headers[i], val);
141-
} else if (cast) {
142-
castRow[i] = val;
143-
} else if (extended){
144-
castRow[i] = val;
138+
139+
Object endVal = cast ? val : field.formatValueAsString(val);
140+
141+
if (keyed) {
142+
keyedRow.put(this.headers[i], endVal);
145143
} else {
146-
plainRow[i] = field.formatValueAsString(val);
144+
resultRow[i] = endVal;
147145
}
148146
}
149147

150148
if (extended){
151-
extendedRow = new Object[]{index, this.headers, castRow};
149+
extendedRow = new Object[]{index, this.headers, resultRow};
152150
index++;
153151
return (T)extendedRow;
154152
} else if(keyed){
155153
return (T)keyedRow;
156-
} else if(cast){
157-
return (T)castRow;
158154
} else{
159-
return (T)plainRow;
155+
return (T)resultRow;
160156
}
161157
}else{
162158
// Enter here if no Schema has been defined.

‎src/test/java/io/frictionlessdata/tableschema/table_tests/TableIterationTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public void testReadExtendedDataWithSchema() throws Exception{
8888
File file = new File("data/employee_data.csv");
8989
Table employeeTable = Table.fromSource(file, testDataDir, employeeTableSchema, null);
9090

91-
Iterator iter = employeeTable.iterator(false, true, false, false);
91+
Iterator iter = employeeTable.iterator(false, true, true, false);
9292

9393
String referenceContent =
9494
String.join("", Files.readAllLines(new File(testDataDir, "data/employee_data_string.json").toPath()));

‎src/test/java/io/frictionlessdata/tableschema/table_tests/TableOtherTest.java‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@
33
import com.fasterxml.jackson.databind.JsonNode;
44
import com.fasterxml.jackson.databind.ObjectMapper;
55
import io.frictionlessdata.tableschema.Table;
6+
import io.frictionlessdata.tableschema.TestHelper;
67
import io.frictionlessdata.tableschema.exception.TableSchemaException;
78
import io.frictionlessdata.tableschema.exception.TableValidationException;
89
import io.frictionlessdata.tableschema.field.*;
910
import io.frictionlessdata.tableschema.schema.Schema;
1011
import io.frictionlessdata.tableschema.tabledatasource.TableDataSource;
12+
import io.frictionlessdata.tableschema.util.JsonUtil;
1113
import org.apache.commons.csv.CSVFormat;
1214
import org.junit.jupiter.api.Assertions;
1315
import org.junit.jupiter.api.DisplayName;
@@ -79,6 +81,7 @@ public class TableOtherTest {
7981

8082

8183
@Test
84+
@DisplayName("Read JSON object array data from string with Schema and compare to Table read from CSV")
8285
public void testReadFromValidJSONArrayWithSchema() throws Exception{
8386
File schemaFile = new File(getTestDataDirectory(), "schema/population_schema.json");
8487
Schema schema = Schema.fromJson (schemaFile, true);
@@ -91,6 +94,36 @@ public void testReadFromValidJSONArrayWithSchema() throws Exception{
9194
Assertions.assertEquals(expectedTable, table);
9295
}
9396

97+
@Test
98+
@DisplayName("Read JSON object array data from string with Schema, return data as CSV and compare to CSV")
99+
public void testReadFromValidJSONArrayWithSchemaReturnCSV() throws Exception{
100+
File schemaFile = new File(getTestDataDirectory(), "schema/population_schema.json");
101+
Schema schema = Schema.fromJson (schemaFile, true);
102+
Table table = Table.fromSource(populationTestJson, schema, TableDataSource.getDefaultCsvFormat());
103+
104+
String csv = table.asCsv();
105+
String refString = TestHelper.getResourceFileContent("/fixtures/data/population.csv");
106+
Assertions.assertEquals(
107+
refString.replaceAll("[\r\n]+", "\n"),
108+
csv.replaceAll("[\r\n]+", "\n"));
109+
}
110+
111+
@Test
112+
@DisplayName("Read CSV data from string with Schema, return data as JSON object array and compare to file")
113+
public void testReadFromCSVWithSchemaReturnValidJSONArray() throws Exception{
114+
File schemaFile = new File(getTestDataDirectory(), "schema/population_schema.json");
115+
Schema schema = Schema.fromJson (schemaFile, true);
116+
File inFile = new File("data/population.csv");
117+
File baseDir = getTestDataDirectory();
118+
Table table = Table.fromSource(inFile, baseDir, schema, TableDataSource.getDefaultCsvFormat());
119+
120+
String json = table.asJson();
121+
JsonNode testNode = JsonUtil.getInstance().readValue(json);
122+
String refString = TestHelper.getResourceFileContent("/fixtures/data/population.json");
123+
JsonNode refNode = JsonUtil.getInstance().readValue(refString);
124+
Assertions.assertEquals(refNode, testNode);
125+
}
126+
94127

95128
@Test
96129
public void testCsvDataSourceFormatToJson() throws Exception{
@@ -229,7 +262,7 @@ public void testIterateCastKeyedData() throws Exception{
229262
File file = new File("data/employee_data.csv");
230263
Table employeeTable = Table.fromSource(file, testDataDir, employeeTableSchema, TableDataSource.getDefaultCsvFormat());
231264

232-
Iterator<Map<String, Object>> iter = employeeTable.mappingIterator(false, false, false);
265+
Iterator<Map<String, Object>> iter = employeeTable.mappingIterator(false, true, false);
233266

234267
while(iter.hasNext()){
235268
Map row = iter.next();

0 commit comments

Comments
 (0)