Skip to content

Commit 7e2c483

Browse files
JohannesJanderJohannesJander
committed
Suppress unnecessary networknt validation messages for the frictionless table-schema.json
1 parent caa1d60 commit 7e2c483

File tree

11 files changed

+157
-68
lines changed

11 files changed

+157
-68
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
[![Support](https://img.shields.io/badge/support-discord-brightgreen)](https://discordapp.com/invite/Sewv6av)
99

1010
A Java library for working with Table data.
11-
**tableschema-java** is a library aimed at parsing CSV and JSON-Array documents into a live Java objects according
11+
**tableschema-java** is a library aimed at parsing CSV and JSON-Array documents into live Java objects according
1212
to [Table Schema](https://frictionlessdata.io/specs/table-schema/), a format definition based on
1313
[JSON Schema](https://json-schema.org/understanding-json-schema/).
1414

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>io.frictionlessdata</groupId>
55
<artifactId>tableschema-java</artifactId>
6-
<version>0.7.2-SNAPSHOT</version>
6+
<version>0.7.3-SNAPSHOT</version>
77
<packaging>jar</packaging>
88
<issueManagement>
99
<url>https://github.com/frictionlessdata/tableschema-java/issues</url>

‎src/main/java/io/frictionlessdata/tableschema/exception/ValidationException.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ public ValidationException(FormalSchemaValidator schema, Collection<ValidationMe
2525
this.validationMessages.addAll(messages);
2626
}
2727

28+
public ValidationException(String message, String schemaName, Collection<ValidationMessage> messages) {
29+
this(String.format("%s: %s", "validation failed: "+message, schemaName));
30+
this.validationMessages.addAll(messages);
31+
}
32+
2833
public ValidationException(String schemaName, Collection<ValidationException> exceptions) {
2934
this(String.format("%s: %s", schemaName, "validation failed: "));
3035
otherMessages.addAll(exceptions

‎src/main/java/io/frictionlessdata/tableschema/field/GeojsonField.java‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package io.frictionlessdata.tableschema.field;
22

33
import com.fasterxml.jackson.databind.JsonNode;
4+
import com.networknt.schema.ValidationMessage;
45
import io.frictionlessdata.tableschema.exception.*;
56
import io.frictionlessdata.tableschema.schema.FormalSchemaValidator;
67
import io.frictionlessdata.tableschema.schema.TypeInferrer;
@@ -9,10 +10,21 @@
910
import java.io.InputStream;
1011
import java.net.URI;
1112
import java.util.Map;
13+
import java.util.Set;
1214

1315
public class GeojsonField extends Field<JsonNode> {
14-
private FormalSchemaValidator geoFormalSchemaValidator = null;
15-
private FormalSchemaValidator topoFormalSchemaValidator = null;
16+
private static FormalSchemaValidator geoFormalSchemaValidator;
17+
private static FormalSchemaValidator topoFormalSchemaValidator;
18+
19+
static {
20+
// FIXME: Maybe this inferring against geojson and topojson scheme is too much.
21+
// Grabbed geojson schema from here: https://github.com/fge/sample-json-schemas/tree/master/geojson
22+
InputStream geoJsonSchemaInputStream = TypeInferrer.class.getResourceAsStream("/schemas/geojson-schema/geojson.json");
23+
geoFormalSchemaValidator = FormalSchemaValidator.fromJson(geoJsonSchemaInputStream);
24+
// Grabbed topojson schema from here: https://github.com/nhuebel/TopoJSON_schema
25+
InputStream topoJsonSchemaInputStream = TypeInferrer.class.getResourceAsStream("/schemas/topojson-schema/topojson.json");
26+
topoFormalSchemaValidator = FormalSchemaValidator.fromJson(topoJsonSchemaInputStream);
27+
}
1628

1729
GeojsonField(){
1830
super();
@@ -71,13 +83,10 @@ String formatObjectValueAsString(Object value, String format, Map<String, Object
7183
*/
7284
private void validateGeoJsonSchema(String json) throws ValidationException {
7385
try {
74-
if(this.geoFormalSchemaValidator == null){
75-
// FIXME: Maybe this inferring against geojson scheme is too much.
76-
// Grabbed geojson schema from here: https://github.com/fge/sample-json-schemas/tree/master/geojson
77-
InputStream geoJsonSchemaInputStream = TypeInferrer.class.getResourceAsStream("/schemas/geojson-schema/geojson.json");
78-
geoFormalSchemaValidator = FormalSchemaValidator.fromJson(geoJsonSchemaInputStream, true);
86+
Set<ValidationMessage> errors = geoFormalSchemaValidator.validate(json);
87+
if (!errors.isEmpty()) {
88+
throw new ValidationException("Geojson field validation failed", geoFormalSchemaValidator.getName(), errors);
7989
}
80-
geoFormalSchemaValidator.validate(json);
8190
} catch (JsonParsingException ex) {
8291
throw new ValidationException(ex);
8392
}
@@ -93,13 +102,10 @@ private void validateGeoJsonSchema(String json) throws ValidationException {
93102
*/
94103
private void validateTopoJsonSchema(String json) throws ValidationException {
95104
try {
96-
if (topoFormalSchemaValidator == null) {
97-
// FIXME: Maybe this infering against topojson scheme is too much.
98-
// Grabbed topojson schema from here: https://github.com/nhuebel/TopoJSON_schema
99-
InputStream topoJsonSchemaInputStream = TypeInferrer.class.getResourceAsStream("/schemas/topojson-schema/topojson.json");
100-
topoFormalSchemaValidator = FormalSchemaValidator.fromJson(topoJsonSchemaInputStream, true);
105+
Set<ValidationMessage> errors = topoFormalSchemaValidator.validate(json);
106+
if (!errors.isEmpty()) {
107+
throw new ValidationException("Geojson field validation failed", geoFormalSchemaValidator.getName(), errors);
101108
}
102-
topoFormalSchemaValidator.validate(json);
103109
} catch (JsonParsingException ex) {
104110
throw new ValidationException(ex);
105111
}

‎src/main/java/io/frictionlessdata/tableschema/schema/FormalSchemaValidator.java‎

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
package io.frictionlessdata.tableschema.schema;
22

33
import com.fasterxml.jackson.databind.JsonNode;
4-
import com.networknt.schema.JsonSchema;
5-
import com.networknt.schema.JsonSchemaFactory;
4+
import com.networknt.schema.*;
65
import com.networknt.schema.SpecVersion.VersionFlag;
7-
import com.networknt.schema.ValidationMessage;
6+
import com.networknt.schema.serialization.JsonNodeReader;
87
import io.frictionlessdata.tableschema.exception.ValidationException;
98
import io.frictionlessdata.tableschema.util.JsonUtil;
109
import org.slf4j.Logger;
@@ -13,50 +12,53 @@
1312
import java.io.InputStream;
1413
import java.util.LinkedHashSet;
1514
import java.util.Set;
15+
import java.util.function.Consumer;
1616

17+
/**
18+
* A class to validate a JSON document against the a JSON schema.
19+
* Mostly used to validate the frictionless data table-schema.json at
20+
* https://specs.frictionlessdata.io/schemas/table-schema.json, but also for topo and geojson schemas used
21+
* in the respective fields.
22+
*
23+
* This class uses the networknt JSON schema validator to validate the JSON document.
24+
*/
1725
public class FormalSchemaValidator {
18-
19-
private static final Logger log = LoggerFactory.getLogger(FormalSchemaValidator.class);
20-
21-
private final boolean strictValidation;
26+
2227
private final JsonSchema jsonSchema;
23-
24-
private FormalSchemaValidator(JsonNode schemaNode, boolean strictValidation) {
25-
JsonSchemaFactory factory = JsonSchemaFactory.getInstance(VersionFlag.V4);
28+
29+
/**
30+
* Instantiate a new FormalSchemaValidator with the given schemaNode and strictValidation flag.
31+
* All occuring validation errors will be returned by `validate()`.
32+
* @param schemaNode the schema to validate against as a JsonNode
33+
*/
34+
private FormalSchemaValidator(JsonNode schemaNode) {
35+
Consumer<JsonSchemaFactory.Builder> customizer = builder -> builder.metaSchema(new TableSchemaVersion().getInstance());
36+
JsonSchemaFactory factory = JsonSchemaFactory.getInstance(VersionFlag.V4, customizer);
2637
this.jsonSchema = factory.getSchema(schemaNode);
27-
this.strictValidation = strictValidation;
2838
}
2939

3040
public static FormalSchemaValidator fromJson(String jsonSchema) {
31-
return fromJson(jsonSchema, true);
32-
}
33-
34-
public static FormalSchemaValidator fromJson(String jsonSchema, boolean strictValidation) {
35-
return new FormalSchemaValidator(JsonUtil.getInstance().readValue(jsonSchema), strictValidation);
41+
return new FormalSchemaValidator(JsonUtil.getInstance().readValue(jsonSchema));
3642
}
3743

38-
public static FormalSchemaValidator fromJson(InputStream jsonSchema, boolean strictValidation) {
39-
return new FormalSchemaValidator(JsonUtil.getInstance().readValue(jsonSchema), strictValidation);
44+
public static FormalSchemaValidator fromJson(InputStream jsonSchema) {
45+
return new FormalSchemaValidator(JsonUtil.getInstance().readValue(jsonSchema));
4046
}
4147

4248
public Set<ValidationMessage> validate(String json) {
4349
return validate(JsonUtil.getInstance().readValue(json));
4450
}
45-
51+
52+
/**
53+
* Validate the given JSON document against the schema and return a set of {@link ValidationMessage} objects.
54+
* If the document is valid, an empty set is returned.
55+
* If the document is invalid, a set of ValidationMessages is returned.
56+
* @param json the JSON document to validate
57+
* @return a set of ValidationMessages if the document is invalid, an empty set otherwise
58+
*/
4659
public Set<ValidationMessage> validate(JsonNode json) {
47-
Set<ValidationMessage> errors = jsonSchema.validate(json);
48-
if (errors.isEmpty()) {
49-
return new LinkedHashSet<>();
50-
} else {
51-
String msg = String.format("validation failed: %s", errors);
52-
if (this.strictValidation) {
53-
log.warn(msg);
54-
throw new ValidationException(this, errors);
55-
} else {
56-
log.warn(msg);
57-
return errors;
58-
}
59-
}
60+
return jsonSchema.validate(json);
61+
6062
}
6163

6264
public String getName() {

‎src/main/java/io/frictionlessdata/tableschema/schema/Schema.java‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
import io.frictionlessdata.tableschema.io.LocalFileReference;
1616
import io.frictionlessdata.tableschema.io.URLFileReference;
1717
import io.frictionlessdata.tableschema.util.JsonUtil;
18+
import org.slf4j.Logger;
19+
import org.slf4j.LoggerFactory;
1820

1921
import java.io.*;
2022
import java.net.URL;
@@ -43,17 +45,34 @@
4345
@JsonIgnoreProperties(ignoreUnknown = true)
4446
@JsonInclude(value = Include.NON_EMPTY)
4547
public class Schema {
48+
private static final Logger log = LoggerFactory.getLogger(Schema.class);
49+
4650
public static final String JSON_KEY_FIELDS = "fields";
4751
public static final String JSON_KEY_PRIMARY_KEY = "primaryKey";
4852
public static final String JSON_KEY_FOREIGN_KEYS = "foreignKeys";
4953

54+
// the schema validator
5055
private FormalSchemaValidator tableFormalSchemaValidator = null;
56+
57+
/**
58+
* List of {@link Field}s of this schema
59+
*/
60+
5161
List<Field<?>> fields = new ArrayList<>();
5262

53-
private FormalSchemaValidator tableJsonSchema = null;
63+
/**
64+
* The primary key of this schema, if any
65+
*/
5466
private Object primaryKey = null;
67+
68+
/**
69+
* List of {@link ForeignKey}s of this schema
70+
*/
5571
private final List<ForeignKey> foreignKeys = new ArrayList<>();
5672

73+
/**
74+
* Whether validation errors should be thrown as exceptions or only reported
75+
*/
5776
boolean strictValidation = true;
5877

5978
@JsonIgnore
@@ -415,8 +434,9 @@ public int hashCode() {
415434
* Validate the loaded Schema. First do a formal validation via JSON schema,
416435
* then check foreign keys match to existing fields.
417436
* <p>
418-
* Validation is strict or lenient depending on how the package was
419-
* instantiated with the strict flag.
437+
* Validation is strict or lenient depending on how the Schema was
438+
* instantiated with the `strictValidation` flag. With strict validation, all validation
439+
* errors will lead to a ValidationException thrown.
420440
*
421441
* @throws ValidationException If validation fails and validation is strict
422442
*/
@@ -447,8 +467,12 @@ public void validate() throws ValidationException{
447467
}
448468
}
449469
}
450-
if (strictValidation && !errors.isEmpty()) {
451-
throw new ValidationException(tableFormalSchemaValidator.getName(), errors);
470+
if (!errors.isEmpty()) {
471+
if (strictValidation) {
472+
throw new ValidationException(tableFormalSchemaValidator.getName(), errors);
473+
} else {
474+
log.warn("Schema validation failed: {}", errors);
475+
}
452476
}
453477
}
454478

@@ -507,7 +531,7 @@ private void initFromSchemaJson(String json) throws PrimaryKeyException, Foreign
507531
private void initValidator() {
508532
// Init for validation
509533
InputStream tableSchemaInputStream = TypeInferrer.class.getResourceAsStream("/schemas/table-schema.json");
510-
this.tableFormalSchemaValidator = FormalSchemaValidator.fromJson(tableSchemaInputStream, strictValidation);
534+
this.tableFormalSchemaValidator = FormalSchemaValidator.fromJson(tableSchemaInputStream);
511535
}
512536

513537

‎src/main/java/io/frictionlessdata/tableschema/schema/TableSchemaVersion.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package io.frictionlessdata.tableschema.schema;
2+
3+
import com.networknt.schema.*;
4+
5+
import java.util.Arrays;
6+
7+
/**
8+
* The frictionless table-schema.json does not really adher to the V4 JSON schema specification.
9+
* This class tells the networknt validator to ignore a couple of keywords that are not part of the V4 spec.
10+
*/
11+
public class TableSchemaVersion implements JsonSchemaVersion {
12+
private static final String IRI = SchemaId.V4;
13+
private static final String ID = "$id";
14+
15+
private static class Holder {
16+
private static final JsonMetaSchema INSTANCE;
17+
static {
18+
JsonMetaSchema.Builder builder = JsonMetaSchema.builder(IRI);
19+
builder.specification(SpecVersion.VersionFlag.V4);
20+
builder.idKeyword(ID);
21+
builder.formats(Formats.DEFAULT);
22+
builder.keywords(ValidatorTypeCode.getKeywords(SpecVersion.VersionFlag.V4));
23+
builder.keywords(Arrays.asList(
24+
new NonValidationKeyword("$schema"),
25+
new NonValidationKeyword("id"),
26+
new AnnotationKeyword("title"),
27+
new AnnotationKeyword("description"),
28+
new AnnotationKeyword("default"),
29+
new NonValidationKeyword("definitions"),
30+
new NonValidationKeyword("$comment"),
31+
new AnnotationKeyword("examples"),
32+
new NonValidationKeyword("then"),
33+
new NonValidationKeyword("else"),
34+
new NonValidationKeyword("FIXME"),
35+
new NonValidationKeyword("TODO"),
36+
new NonValidationKeyword("context"),
37+
new NonValidationKeyword("notes"),
38+
new NonValidationKeyword("options"),
39+
new NonValidationKeyword("propertyOrder"),
40+
new NonValidationKeyword("additionalItems")));// keywords that may validly exist, but have no validation aspect to them
41+
INSTANCE = builder
42+
.build();
43+
}
44+
}
45+
46+
@Override
47+
public JsonMetaSchema getInstance() {
48+
return Holder.INSTANCE;
49+
}
50+
}

‎src/main/resources/schemas/geojson-schema/geojson.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@
66
"type": "object",
77
"required": [ "type" ],
88
"properties": {
9-
"crs": { "$ref": "crs.json" },
10-
"bbox": { "$ref": "bbox.json" }
9+
"crs": { "$ref": "resource:/schemas/geojson-schema/crs.json" },
10+
"bbox": { "$ref": "resource:/schemas/geojson-schema/bbox.json" }
1111
},
1212
"oneOf": [
13-
{ "$ref": "geometry.json" },
13+
{ "$ref": "resource:/schemas/geojson-schema/geometry.json" },
1414
{ "$ref": "#/definitions/geometryCollection" },
1515
{ "$ref": "#/definitions/feature" },
1616
{ "$ref": "#/definitions/featureCollection" }
@@ -24,7 +24,7 @@
2424
"type": { "enum": [ "GeometryCollection" ] },
2525
"geometries": {
2626
"type": "array",
27-
"items": { "$ref": "geometry.json" }
27+
"items": { "$ref": "resource:/schemas/geojson-schema/geometry.json" }
2828
}
2929
}
3030
},
@@ -37,7 +37,7 @@
3737
"geometry": {
3838
"oneOf": [
3939
{ "type": "null" },
40-
{ "$ref": "geometry.json" }
40+
{ "$ref": "resource:/schemas/geojson-schema/geometry.json" }
4141
]
4242
},
4343
"properties": { "type": [ "object", "null" ] },

‎src/main/resources/schemas/topojson-schema/topojson.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@
66
"type": "object",
77
"required": [ "type" ],
88
"properties": {
9-
"bbox": { "$ref": "bbox.json" }
9+
"bbox": { "$ref": "resource:/schemas/topojson-schema/bbox.json" }
1010
},
1111
"oneOf": [
12-
{ "$ref": "topology.json" },
13-
{ "$ref": "geometry.json" }
12+
{ "$ref": "resource:/schemas/topojson-schema/topology.json" },
13+
{ "$ref": "resource:/schemas/topojson-schema/geometry.json" }
1414
]
1515
}

‎src/test/java/io/frictionlessdata/tableschema/iterator/TableBeanIteratorTest.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,6 @@ void testBeanDeserialization2() throws Exception {
9393
Assertions.assertEquals(2, interests.size());
9494
Assertions.assertEquals("sports", interests.get(0));
9595
Assertions.assertEquals("reading", interests.get(1));
96-
97-
Object info = frank.getInfo();
98-
/*Assertions.assertEquals(45, info.get("pin"));
99-
Assertions.assertEquals(83.23, info.get("rate"));
100-
Assertions.assertEquals(90, info.get("ssn"));*/
101-
System.out.println(info);
10296
}
10397

10498
@Test

0 commit comments

Comments
 (0)