Skip to content

Commit e65e8a2

Browse files
JohannesJanderJohannesJander
committed
Changes schemaless tables
1 parent 3c823cc commit e65e8a2

File tree

7 files changed

+83
-27
lines changed

7 files changed

+83
-27
lines changed

‎src/main/java/io/frictionlessdata/tableschema/Table.java‎

Lines changed: 34 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -328,10 +328,10 @@ public Iterator<Map<String, Object>> mappingIterator(boolean extended, boolean c
328328
public Map<Integer, Integer> getSchemaHeaderMapping() {
329329
if (null == schema) {
330330
return TableSchemaUtil
331-
.createSchemaHeaderMapping(dataSource.getHeaders(), dataSource.getHeaders());
331+
.createSchemaHeaderMapping(dataSource.getHeaders(), dataSource.getHeaders(), true);
332332
} else {
333333
return TableSchemaUtil
334-
.createSchemaHeaderMapping(dataSource.getHeaders(), schema.getHeaders());
334+
.createSchemaHeaderMapping(dataSource.getHeaders(), schema.getHeaders(), dataSource.hasReliableHeaders());
335335
}
336336
}
337337

@@ -460,8 +460,12 @@ private void writeCsv(Writer out, CSVFormat format, String[] sortedHeaders) {
460460

461461
String[] headers = getHeaders();
462462
Map<Integer, Integer> mapping
463-
= TableSchemaUtil.createSchemaHeaderMapping(headers, sortedHeaders);
464-
writeCSVData( mapping, csvPrinter);
463+
= TableSchemaUtil.createSchemaHeaderMapping(headers, sortedHeaders, dataSource.hasReliableHeaders());
464+
if (null != schema) {
465+
writeCSVData(mapping, schema, csvPrinter);
466+
} else {
467+
writeCSVData(mapping, csvPrinter);
468+
}
465469
csvPrinter.close();
466470
} catch (IOException ex) {
467471
throw new TableIOException(ex);
@@ -616,8 +620,7 @@ public Schema inferSchema(int rowLimit) throws TypeInferringException{
616620
public Schema inferSchema(String[] headers, int rowLimit) throws TypeInferringException{
617621
try{
618622
List<Object[]> data = read();
619-
schema = Schema.infer(data, headers, rowLimit);
620-
return schema;
623+
return Schema.infer(data, headers, rowLimit);
621624

622625
} catch(Exception e){
623626
throw new TypeInferringException(e);
@@ -712,10 +715,7 @@ public int hashCode() {
712715
}
713716

714717

715-
private void writeCSVData(Map<Integer, Integer> mapping, CSVPrinter csvPrinter) {
716-
List<Map<String, Object>> rows = new ArrayList<>();
717-
Schema schema = (null != this.schema) ? this.schema : this.inferSchema();
718-
718+
private void writeCSVData(Map<Integer, Integer> mapping, Schema schema, CSVPrinter csvPrinter) {
719719
Iterator<Object> iter = this.iterator(false, false, true, false);
720720
iter.forEachRemaining((rec) -> {
721721
Object[] row = (Object[])rec;
@@ -730,6 +730,30 @@ private void writeCSVData(Map<Integer, Integer> mapping, CSVPrinter csvPrinter)
730730
obj.add(field.formatValueAsString(s));
731731
i++;
732732
}
733+
734+
try {
735+
csvPrinter.printRecord(obj);
736+
} catch (Exception ex) {
737+
throw new TableIOException(ex);
738+
}
739+
});
740+
}
741+
742+
private void writeCSVData(Map<Integer, Integer> mapping, CSVPrinter csvPrinter) {
743+
Iterator<Object> iter = this.iterator(false, false, false, false);
744+
iter.forEachRemaining((rec) -> {
745+
Object[] row = (Object[])rec;
746+
Object[] sortedRec = new Object[row.length];
747+
for (int i = 0; i < row.length; i++) {
748+
sortedRec[mapping.get(i)] = row[i];
749+
}
750+
List<String> obj = new ArrayList<>();
751+
752+
for (int j = 0; j < sortedRec.length; j++) {
753+
Object s = sortedRec[j];
754+
obj.add((null != s) ? s.toString() : "");
755+
}
756+
733757
try {
734758
csvPrinter.printRecord(obj);
735759
} catch (Exception ex) {

‎src/main/java/io/frictionlessdata/tableschema/fk/ForeignKey.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,17 @@ public List<String> getFieldNames(){
5252
throw new IllegalArgumentException("Invalid fields type in reference: "+this.fields.getClass().getName());
5353
}
5454
}
55-
55+
56+
@JsonIgnore
57+
public Map<String, String> getFieldMapping() {
58+
Map<String, String> fieldMapping = new HashMap<>();
59+
List<String> fieldNames1 = getFieldNames();
60+
for (int i = 0; i < fieldNames1.size(); i++) {
61+
fieldMapping.put(fieldNames1.get(i), reference.getFieldNames().get(i));
62+
}
63+
return fieldMapping;
64+
}
65+
5666
public void setReference(Reference reference){
5767
this.reference = reference;
5868
}

‎src/main/java/io/frictionlessdata/tableschema/iterator/TableIterator.java‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,15 @@
11
package io.frictionlessdata.tableschema.iterator;
22

33
import io.frictionlessdata.tableschema.Table;
4+
import io.frictionlessdata.tableschema.exception.TableValidationException;
45
import io.frictionlessdata.tableschema.field.Field;
56
import io.frictionlessdata.tableschema.schema.Schema;
67

7-
import java.util.HashMap;
8-
import java.util.Iterator;
9-
import java.util.LinkedHashMap;
10-
import java.util.Map;
8+
import java.util.*;
119

1210
/**
13-
*
11+
* Iterator that can read data from a Table in the various permutations
12+
* of the `keyed`, `extended`, `cast`, `relations` flags.
1413
*
1514
*/
1615
public class TableIterator<T> implements Iterator<T> {
@@ -23,6 +22,10 @@ public class TableIterator<T> implements Iterator<T> {
2322
* The table's Schema
2423
*/
2524
Schema schema = null;
25+
26+
/**
27+
* the underlying iterator
28+
*/
2629
Iterator<String[]> wrappedIterator = null;
2730

2831
/**
@@ -46,7 +49,7 @@ public class TableIterator<T> implements Iterator<T> {
4649
boolean relations = false;
4750

4851
/**
49-
* Mapping of column indices between Schema and CSV file
52+
* Mapping of column indices between Schema and CSV file headers
5053
*/
5154
Map<Integer, Integer> mapping = null;
5255

‎src/main/java/io/frictionlessdata/tableschema/schema/Schema.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -495,10 +495,12 @@ public void validate(Table t) throws ValidationException{
495495
errors.add(ve);
496496
}
497497
}
498-
if (strictValidation && !errors.isEmpty()) {
499-
throw new ValidationException(errors);
500-
} else {
501-
log.warn("Schema validation failed: {}", errors);
498+
if (!errors.isEmpty()) {
499+
if (strictValidation) {
500+
throw new ValidationException(errors);
501+
} else {
502+
log.warn("Schema validation failed: {}", errors);
503+
}
502504
}
503505
}
504506

‎src/main/java/io/frictionlessdata/tableschema/tabledatasource/JsonArrayTableDataSource.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,10 @@ public boolean hasReliableHeaders() {
4949

5050
@Override
5151
public Iterator<String[]> iterator() {
52-
String[] headers = getHeaders();
52+
boolean deleteHeaderRow = (null == this.headers);
53+
this.headers = getHeaders();
5354
JsonNode firstRow = dataSource.get(0);
54-
if (firstRow instanceof ArrayNode) {
55+
if ((deleteHeaderRow) && (firstRow instanceof ArrayNode)) {
5556
dataSource.remove(0);
5657
}
5758

@@ -87,6 +88,9 @@ public Iterator<String[]> iterator() {
8788
*/
8889
@Override
8990
public String[] getHeaders() {
91+
if (null != this.headers) {
92+
return this.headers;
93+
}
9094
Set<String> headers = new LinkedHashSet<>();
9195
Iterator<JsonNode> iterator = dataSource.elements();
9296
if (iterator.hasNext()) {

‎src/main/java/io/frictionlessdata/tableschema/util/JsonUtil.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ public static JsonUtil getInstance() {
4343
return instance;
4444
}
4545

46+
public ObjectMapper getMapper(){return mapper;}
47+
48+
4649
public ObjectNode createNode() {
4750
return mapper.createObjectNode();
4851
}

‎src/main/java/io/frictionlessdata/tableschema/util/TableSchemaUtil.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,16 @@
11
package io.frictionlessdata.tableschema.util;
22

3+
import io.frictionlessdata.tableschema.exception.TableValidationException;
4+
35
import java.util.HashMap;
46
import java.util.Map;
57

68
public class TableSchemaUtil {
79

8-
public static Map<Integer, Integer> createSchemaHeaderMapping(String[] headers, String[] sortedHeaders) {
10+
public static Map<Integer, Integer> createSchemaHeaderMapping(
11+
String[] headers,
12+
String[] sortedHeaders,
13+
boolean reliableHeaders) {
914
if ((null == headers) || (null == sortedHeaders))
1015
return null;
1116
Map<Integer, Integer> mapping = new HashMap<>();
@@ -16,10 +21,15 @@ public static Map<Integer, Integer> createSchemaHeaderMapping(String[] headers,
1621
mapping.put(i, j);
1722
}
1823
}
19-
// declared header not found in actual data - can happen with JSON Arrays
20-
// of JSON objects as they will not have keys for null values
24+
// declared header not found in actual data - can happen
25+
// - if no header row in CSV data (throw)
26+
// - with JSON Arrays of JSON objects as they will not have keys for null values (accept, but don't map)
2127
if (!mapping.containsKey(i)) {
22-
mapping.put(i, null);
28+
if (reliableHeaders) {
29+
throw new TableValidationException("Field '" + sortedHeaders[i] + "' not found in table headers or table has no headers.");
30+
} else {
31+
mapping.put(i, null);
32+
}
2333
}
2434
}
2535
return mapping;

0 commit comments

Comments
 (0)