Skip to content

Commit 3c823cc

Browse files
JohannesJanderJohannesJander
committed
Couple of fixes for JSON tables. Improved Schema validation, now including primary keys. Schema loading from JSON is improved.
1 parent ea4e260 commit 3c823cc

File tree

12 files changed

+283
-188
lines changed

12 files changed

+283
-188
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>io.frictionlessdata</groupId>
55
<artifactId>tableschema-java</artifactId>
6-
<version>0.7.6-SNAPSHOT</version>
6+
<version>0.8.0-SNAPSHOT</version>
77
<packaging>jar</packaging>
88
<issueManagement>
99
<url>https://github.com/frictionlessdata/tableschema-java/issues</url>

‎src/main/java/io/frictionlessdata/tableschema/Table.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -571,7 +571,7 @@ public void validate() throws TableValidationException, TableSchemaException {
571571
}
572572
}
573573
if (null != schema)
574-
schema.validate();
574+
schema.validate(this);
575575
}
576576

577577
/**

‎src/main/java/io/frictionlessdata/tableschema/exception/ValidationException.java‎

Lines changed: 50 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import java.util.stream.Collectors;
99

1010
public class ValidationException extends TableSchemaException {
11-
List<ValidationMessage> validationMessages = new ArrayList<>();
11+
List<String> validationMessages = new ArrayList<>();
1212
List<Exception> wrappedExceptions = new ArrayList<>();
1313

1414
public ValidationException(String msg) {
@@ -19,36 +19,68 @@ public ValidationException(Exception ex) {
1919
wrappedExceptions.add(ex);
2020
}
2121

22-
public ValidationException(FormalSchemaValidator schema, Collection<ValidationMessage> messages) {
23-
this(String.format("%s: %s", schema.getName(), "validation failed"));
24-
this.validationMessages.addAll(messages);
22+
public ValidationException(String schemaName, Collection<ValidationMessage> messages) {
23+
this("Formal validation failed for Schema "+ schemaName);
24+
addValidationMessages(messages);
2525
}
2626

27-
public ValidationException(String message, String schemaName, Collection<ValidationMessage> messages) {
28-
this(String.format("%s: %s", "validation failed: "+message, schemaName));
29-
this.validationMessages.addAll(messages);
30-
}
3127

32-
public ValidationException(String schemaName, Collection<ValidationException> exceptions) {
33-
this(String.format("%s: %s", schemaName, "validation failed: "));
28+
public ValidationException(ValidationException exception) {
29+
this("Validation failed");
30+
wrappedExceptions.add(exception);
31+
}
32+
public ValidationException(Collection<ValidationException> exceptions) {
33+
this("Validation failed");
3434
wrappedExceptions.addAll(exceptions);
35-
final Set<ValidationMessage> messages = new LinkedHashSet<>();
36-
exceptions.forEach((m) -> {
37-
if (m instanceof ValidationException) {
38-
messages.addAll(((ValidationException)m).validationMessages);
39-
}
40-
});
41-
this.validationMessages.addAll(messages);
35+
}
36+
37+
38+
void addValidationMessage(ValidationMessage message) {
39+
validationMessages.add(message.getMessage());
40+
}
41+
42+
void addValidationMessages(Collection<ValidationMessage> messages) {
43+
messages.forEach(this::addValidationMessage);
4244
}
4345

4446
public List<Exception> getWrappedExceptions() {
4547
return wrappedExceptions;
4648
}
4749

48-
public List<ValidationMessage> getValidationMessages() {
50+
public List<String> getValidationMessages() {
4951
return validationMessages;
5052
}
5153

54+
@Override
55+
public String getMessage(){
56+
String message = super.getMessage() == null ? "Exception: " : super.getMessage();
57+
StringBuilder sb = new StringBuilder(message);
58+
if (!wrappedExceptions.isEmpty()) {
59+
sb.append(" with ");
60+
if (wrappedExceptions.size() > 1) {
61+
sb.append(wrappedExceptions.size()).append(" exceptions: [");
62+
for (Exception ex : wrappedExceptions) {
63+
sb.append((ex.getClass().getSimpleName() + ": " + ex.getMessage())).append("\n");
64+
sb.append("]");
65+
}
66+
} else {
67+
Exception ex = wrappedExceptions.get(0);
68+
sb.append((ex.getClass().getSimpleName()+": "+ex.getMessage()));
69+
}
70+
if (!validationMessages.isEmpty()) {
71+
sb.append("Additionally, formal validation failed with:\n");
72+
for (String s : validationMessages) {
73+
sb.append(s).append("\n");
74+
}
75+
}
76+
} else {
77+
for (String s : validationMessages) {
78+
sb.append(s).append("\n");
79+
}
80+
}
81+
return sb.toString();
82+
}
83+
5284
public List<Object> getMessages() {
5385
List<Object> retVal = new ArrayList<>();
5486
retVal.addAll(validationMessages);

‎src/main/java/io/frictionlessdata/tableschema/field/GeojsonField.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ private void validateGeoJsonSchema(String json) throws ValidationException {
8585
try {
8686
Set<ValidationMessage> errors = geoFormalSchemaValidator.validate(json);
8787
if (!errors.isEmpty()) {
88-
throw new ValidationException("Geojson field validation failed", geoFormalSchemaValidator.getName(), errors);
88+
throw new ValidationException("Geojson field validation failed", errors);
8989
}
9090
} catch (JsonParsingException ex) {
9191
throw new ValidationException(ex);
@@ -104,7 +104,7 @@ private void validateTopoJsonSchema(String json) throws ValidationException {
104104
try {
105105
Set<ValidationMessage> errors = topoFormalSchemaValidator.validate(json);
106106
if (!errors.isEmpty()) {
107-
throw new ValidationException("Geojson field validation failed", geoFormalSchemaValidator.getName(), errors);
107+
throw new ValidationException("Topojson field validation failed", errors);
108108
}
109109
} catch (JsonParsingException ex) {
110110
throw new ValidationException(ex);

‎src/main/java/io/frictionlessdata/tableschema/fk/ForeignKey.java‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ public Reference getReference(){
6363

6464
public final void validate() throws ForeignKeyException{
6565
ForeignKeyException fke = null;
66+
this.errors.clear();
6667

6768
if(this.fields == null || this.reference == null){
6869
fke = new ForeignKeyException("A foreign key must have the fields and reference properties.");
@@ -115,16 +116,12 @@ public final void validate(Table table) throws ForeignKeyException{
115116
if (reference.getResource().equals("")) {
116117
List<String> fieldNames = new ArrayList<>();
117118
List<String> foreignFieldNames = new ArrayList<>();
118-
if (fields instanceof String) {
119-
fieldNames.add((String) fields);
120-
foreignFieldNames.add(reference.getFields());
121-
} else if (fields instanceof Collection) {
122-
List<String> lFields = getFieldNames();
123-
for (int i = 0; i < lFields.size(); i++) {
124-
fieldNames.add(lFields.get(i));
125-
foreignFieldNames.add(reference.getFieldNames().get(i));
126-
}
119+
List<String> lFields = getFieldNames();
120+
for (int i = 0; i < lFields.size(); i++) {
121+
fieldNames.add(lFields.get(i));
122+
foreignFieldNames.add(reference.getFieldNames().get(i));
127123
}
124+
128125
Iterator<Object> iterator = table.iterator(true, false, false, false);
129126
while (iterator.hasNext()) {
130127
Map<String, Object> next = (Map<String, Object>)iterator.next();

‎src/main/java/io/frictionlessdata/tableschema/fk/Reference.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ public List<String> getFieldNames(){
9191
}
9292

9393
public final void validate() throws ForeignKeyException{
94+
errors.clear();
9495
ForeignKeyException fke = null;
9596
if(this.resourceName == null || this.fields == null){
9697
fke = new ForeignKeyException("A foreign key's reference must have the fields and resource properties.");

‎src/main/java/io/frictionlessdata/tableschema/iterator/TableIterator.java‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,45 @@
1414
*
1515
*/
1616
public class TableIterator<T> implements Iterator<T> {
17+
/**
18+
* The table's headers
19+
*/
1720
String[] headers = null;
21+
22+
/**
23+
* The table's Schema
24+
*/
1825
Schema schema = null;
1926
Iterator<String[]> wrappedIterator = null;
27+
28+
/**
29+
* If true, return rows of Map objects, otherwise rows of Arrays
30+
*/
2031
boolean keyed = false;
32+
33+
/**
34+
* If true, return rows of entries in the extended format
35+
*/
2136
boolean extended = false;
37+
38+
/**
39+
* If true, case values to Java objects
40+
*/
2241
boolean cast = true;
42+
43+
/**
44+
* follow relations to other tables
45+
*/
2346
boolean relations = false;
47+
48+
/**
49+
* Mapping of column indices between Schema and CSV file
50+
*/
2451
Map<Integer, Integer> mapping = null;
52+
53+
/**
54+
* The index of the row when reading in `extended` mode
55+
*/
2556
int index = 0;
2657

2758

@@ -49,7 +80,6 @@ void init(Table table) {
4980
this.mapping = table.getSchemaHeaderMapping();
5081
this.headers = table.getHeaders();
5182
this.schema = table.getSchema();
52-
table.validate();
5383
this.wrappedIterator = table.getTableDataSource().iterator();
5484
}
5585

@@ -67,6 +97,7 @@ public void remove() {
6797
@Override
6898
public T next() {
6999
String[] row = this.wrappedIterator.next();
100+
String rawVal = null;
70101
int rowLength = row.length;
71102
if (null != this.schema) {
72103
rowLength = Math.max(row.length, this.schema.getFields().size());
@@ -96,13 +127,15 @@ public T next() {
96127
if (null != mappedKey) {
97128
// if the last column(s) contain nulls, prevent an ArrayIndexOutOfBoundsException
98129
if (mappedKey < row.length) {
99-
String rawVal = row[mappedKey];
130+
rawVal = row[mappedKey];
100131
val = field.castValue(rawVal);
101132
}
102133
}
103134
if (!extended && keyed) {
104135
keyedRow.put(this.headers[i], val);
105-
} else if (cast || extended){
136+
} else if (cast) {
137+
castRow[i] = val;
138+
} else if (extended){
106139
castRow[i] = val;
107140
} else {
108141
plainRow[i] = field.formatValueAsString(val);

0 commit comments

Comments
 (0)