apply default in objects and arrays (#477)

SiemelNaran · web-flow · commit fad2a584fb5f · 2021-12-01T20:23:39.000-05:00
* apply default in objects and arrays

* introduce a class ApplyDefaultsStrategy

* ensure that validate function does not apply defaults

* add documentation

* support default value in referenced property

* fix NullPointerException when no applyDefaultStragey is supplied

* fix the 4 test errors mentioned: call CollectorContext.getInstance().reset() after each test
diff --git a/doc/walkers.md b/doc/walkers.md
@@ -1,6 +1,6 @@
 ### JSON Schema Walkers
 
-There can be use-cases where we need the capability to walk through the given JsonNode allowing functionality beyond validation like collecting information,handling cross cutting concerns like logging or instrumentation. JSON walkers were introduced to complement the validation functionality this library already provides.
+There can be use-cases where we need the capability to walk through the given JsonNode allowing functionality beyond validation like collecting information,handling cross cutting concerns like logging or instrumentation, or applying default values. JSON walkers were introduced to complement the validation functionality this library already provides.
 
 Currently, walking is defined at the validator instance level for all the built-in keywords.
 
@@ -237,4 +237,56 @@ Few important points to note about the flow.
 5. Since we have a property listener defined, When we are walking through a property that has a "$ref" keyword which might have some more properties defined, 
    Our property listener would be invoked for each of the property defined in the "$ref" schema. 
 6. As mentioned earlier anywhere during the "Walk Flow", we can return a  WalkFlow.SKIP from onWalkStart method to stop the walk method of a particular "property schema" from being called. 
-   Since the walk method will not be called any property or keyword listeners in the "property schema" will not be invoked.
+   Since the walk method will not be called any property or keyword listeners in the "property schema" will not be invoked.
+
+
+### Applying defaults
+
+In some use cases we may want to apply defaults while walking the schema.
+To accomplish this, create an ApplyDefaultsStrategy when creating a SchemaValidatorsConfig.
+The input object is changed in place, even if validation fails, or a fail-fast or some other exception is thrown.
+
+Here is the order of operations in walker.
+1. apply defaults
+1. run listeners
+1. validate if shouldValidateSchema is true
+
+Suppose the JSON schema is
+```json
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Schema with default values ",
+  "type": "object",
+  "properties": {
+    "intValue": {
+      "type": "integer",
+      "default": 15, 
+      "minimum": 20
+    }
+  },
+  "required": ["intValue"]
+}
+```
+
+A JSON file like
+```json
+{
+}
+```
+
+would normally fail validation as "intValue" is required.
+But if we apply defaults while walking, then required validation passes, and the object is changed in place.
+
+```java
+        JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V4);
+        SchemaValidatorsConfig schemaValidatorsConfig = new SchemaValidatorsConfig();
+        schemaValidatorsConfig.setApplyDefaultsStrategy(new ApplyDefaultsStrategy(true, true, true));
+        JsonSchema jsonSchema =  schemaFactory.getSchema(getClass().getClassLoader().getResourceAsStream("schema.json"), schemaValidatorsConfig);
+
+        JsonNode inputNode = objectMapper.readTree(getClass().getClassLoader().getResourceAsStream("data.json"));
+        ValidationResult result = jsonSchema.walk(inputNode, true);
+        assertThat(result.getValidationMessages(), Matchers.empty());
+        assertEquals("{\"intValue\":15}", inputNode.toString());
+        assertThat(result.getValidationMessages().stream().map(ValidationMessage::getMessage).collect(Collectors.toList()),
+                   Matchers.containsInAnyOrder("$.intValue: must have a minimum value of 20."));
+```
diff --git a/src/main/java/com/networknt/schema/ApplyDefaultsStrategy.java b/src/main/java/com/networknt/schema/ApplyDefaultsStrategy.java
@@ -0,0 +1,43 @@
+package com.networknt.schema;
+
+public class ApplyDefaultsStrategy {
+    static final ApplyDefaultsStrategy EMPTY_APPLY_DEFAULTS_STRATEGY = new ApplyDefaultsStrategy(false, false, false);
+
+    private final boolean applyPropertyDefaults;
+    private final boolean applyPropertyDefaultsIfNull;
+    private final boolean applyArrayDefaults;
+
+    /**
+     * Specify which default values to apply.
+     * We can apply property defaults only if they are missing or if they are declared to be null in the input json,
+     * and we can apply array defaults if they are declared to be null in the input json.
+     *
+     * <p>Note that the walker changes the input object in place.
+     * If validation fails, the input object will be changed.
+     *
+     * @param applyPropertyDefaults if true then apply defaults inside json objects if the attribute is missing
+     * @param applyPropertyDefaultsIfNull if true then apply defaults inside json objects if the attribute is explicitly null
+     * @param applyArrayDefaults if true then apply defaults inside json arrays if the attribute is explicitly null
+     * @throws IllegalArgumentException if applyPropertyDefaults is false and applyPropertyDefaultsIfNull is true
+     */
+    public ApplyDefaultsStrategy(boolean applyPropertyDefaults, boolean applyPropertyDefaultsIfNull, boolean applyArrayDefaults) {
+        if (!applyPropertyDefaults && applyPropertyDefaultsIfNull) {
+            throw new IllegalArgumentException();
+        }
+        this.applyPropertyDefaults = applyPropertyDefaults;
+        this.applyPropertyDefaultsIfNull = applyPropertyDefaultsIfNull;
+        this.applyArrayDefaults = applyArrayDefaults;
+    }
+
+    public boolean shouldApplyPropertyDefaults() {
+        return applyPropertyDefaults;
+    }
+
+    public boolean shouldApplyPropertyDefaultsIfNull() {
+        return applyPropertyDefaultsIfNull;
+    }
+
+    public boolean shouldApplyArrayDefaults() {
+        return applyArrayDefaults;
+    }
+}
diff --git a/src/main/java/com/networknt/schema/BaseJsonValidator.java b/src/main/java/com/networknt/schema/BaseJsonValidator.java
@@ -38,23 +38,37 @@ public abstract class BaseJsonValidator implements JsonValidator {
     private ErrorMessageType errorMessageType;
 
     protected final boolean failFast;
+    protected final ApplyDefaultsStrategy applyDefaultsStrategy;
 
     public BaseJsonValidator(String schemaPath, JsonNode schemaNode, JsonSchema parentSchema,
                              ValidatorTypeCode validatorType, ValidationContext validationContext) {
         this(schemaPath, schemaNode, parentSchema, validatorType, false,
-             validationContext.getConfig() != null && validationContext.getConfig().isFailFast());
+             validationContext.getConfig() != null && validationContext.getConfig().isFailFast(),
+             validationContext.getConfig() != null ? validationContext.getConfig().getApplyDefaultsStrategy() : null);
     }
 
+    // TODO: can this be made package private?
+    @Deprecated // use the BaseJsonValidator below
     public BaseJsonValidator(String schemaPath, JsonNode schemaNode, JsonSchema parentSchema,
                              ValidatorTypeCode validatorType, boolean suppressSubSchemaRetrieval, boolean failFast) {
+        this(schemaPath, schemaNode, parentSchema, validatorType, false, failFast, null);
+    }
 
+    public BaseJsonValidator(String schemaPath,
+                             JsonNode schemaNode,
+                             JsonSchema parentSchema,
+                             ValidatorTypeCode validatorType,
+                             boolean suppressSubSchemaRetrieval,
+                             boolean failFast,
+                             ApplyDefaultsStrategy applyDefaultsStrategy) {
         this.errorMessageType = validatorType;
         this.schemaPath = schemaPath;
         this.schemaNode = schemaNode;
         this.parentSchema = parentSchema;
         this.validatorType = validatorType;
         this.suppressSubSchemaRetrieval = suppressSubSchemaRetrieval;
         this.failFast = failFast;
+        this.applyDefaultsStrategy = applyDefaultsStrategy != null ? applyDefaultsStrategy : ApplyDefaultsStrategy.EMPTY_APPLY_DEFAULTS_STRATEGY;
     }
 
     public String getSchemaPath() {
diff --git a/src/main/java/com/networknt/schema/ItemsValidator.java b/src/main/java/com/networknt/schema/ItemsValidator.java
@@ -17,6 +17,7 @@
 package com.networknt.schema;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.networknt.schema.walk.DefaultItemWalkListenerRunner;
 import com.networknt.schema.walk.WalkListenerRunner;
 
@@ -118,9 +119,21 @@ private void doValidate(Set<ValidationMessage> errors, int i, JsonNode node, Jso
     @Override
     public Set<ValidationMessage> walk(JsonNode node, JsonNode rootNode, String at, boolean shouldValidateSchema) {
         HashSet<ValidationMessage> validationMessages = new LinkedHashSet<ValidationMessage>();
-        if (node != null && node.isArray()) {
+        if (node instanceof ArrayNode) {
+            ArrayNode arrayNode = (ArrayNode) node;
+            JsonNode defaultNode = null;
+            if (applyDefaultsStrategy.shouldApplyArrayDefaults() && schema != null) {
+                defaultNode = schema.getSchemaNode().get("default");
+                if (defaultNode != null && defaultNode.isNull()) {
+                    defaultNode = null;
+                }
+            }
             int i = 0;
-            for (JsonNode n : node) {
+            for (JsonNode n : arrayNode) {
+                if (n.isNull() && defaultNode != null) {
+                    arrayNode.set(i, defaultNode);
+                    n = defaultNode;
+                }
                 doWalk(validationMessages, i, n, rootNode, at, shouldValidateSchema);
                 i++;
             }
diff --git a/src/main/java/com/networknt/schema/JsonSchema.java b/src/main/java/com/networknt/schema/JsonSchema.java
@@ -19,14 +19,15 @@
 import java.io.UnsupportedEncodingException;
 import java.net.URI;
 import java.net.URLDecoder;
-import java.sql.Ref;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -37,8 +38,6 @@
 import com.networknt.schema.walk.JsonSchemaWalker;
 import com.networknt.schema.walk.WalkListenerRunner;
 
-import javax.xml.validation.Schema;
-
 /**
  * This is the core of json constraint implementation. It parses json constraint
  * file and generates JsonValidators. The class is thread safe, once it is
@@ -80,7 +79,8 @@ public JsonSchema(ValidationContext validationContext, URI baseUri, JsonNode sch
     private JsonSchema(ValidationContext validationContext, String schemaPath, URI currentUri, JsonNode schemaNode,
                        JsonSchema parent, boolean suppressSubSchemaRetrieval) {
         super(schemaPath, schemaNode, parent, null, suppressSubSchemaRetrieval,
-                validationContext.getConfig() != null && validationContext.getConfig().isFailFast());
+              validationContext.getConfig() != null && validationContext.getConfig().isFailFast(),
+              validationContext.getConfig() != null ? validationContext.getConfig().getApplyDefaultsStrategy() : null);
         this.validationContext = validationContext;
         this.idKeyword = validationContext.getMetaSchema().getIdKeyword();
         this.currentUri = this.combineCurrentUriWithIds(currentUri, schemaNode);
@@ -208,7 +208,7 @@ private boolean nodeContainsRef(String ref, JsonNode node) {
      * used in {@link com.networknt.schema.walk.DefaultKeywordWalkListenerRunner} to derive the keyword.
      */
     private Map<String, JsonValidator> read(JsonNode schemaNode) {
-        Map<String, JsonValidator> validators = new HashMap<String, JsonValidator>();
+        Map<String, JsonValidator> validators = new TreeMap<>(VALIDATOR_SORT);
         if (schemaNode.isBoolean()) {
             if (schemaNode.booleanValue()) {
                 final String customMessage = getCustomMessage(schemaNode, "true");
@@ -239,6 +239,20 @@ private Map<String, JsonValidator> read(JsonNode schemaNode) {
         return validators;
     }
 
+    /**
+     * A comparator that sorts validators, such such that 'properties' comes before 'required',
+     * so that we can apply default values before validating required.
+     */
+    private static Comparator<String> VALIDATOR_SORT = (lhs, rhs) -> {
+        if (lhs.endsWith("/properties")) {
+            return -1;
+        }
+        if (rhs.endsWith("/properties")) {
+            return 1;
+        }
+        return lhs.compareTo(rhs);
+    };
+
     private String getCustomMessage(JsonNode schemaNode, String pname) {
         final JsonSchema parentSchema = getParentSchema();
         final JsonNode message = getMessageNode(schemaNode, parentSchema);
@@ -320,7 +334,7 @@ protected ValidationResult validateAndCollect(JsonNode jsonNode, JsonNode rootNo
             SchemaValidatorsConfig config = validationContext.getConfig();
             // Get the collector context from the thread local.
             CollectorContext collectorContext = getCollectorContext();
-            // Valdiate.
+            // Validate.
             Set<ValidationMessage> errors = validate(jsonNode, rootNode, at);
             // When walk is called in series of nested call we don't want to load the collectors every time. Leave to the API to decide when to call collectors.
             if (config.doLoadCollectors()) {
@@ -374,7 +388,7 @@ public Set<ValidationMessage> walk(JsonNode node, JsonNode rootNode, String at,
             JsonSchemaWalker jsonWalker = entry.getValue();
             String schemaPathWithKeyword = entry.getKey();
             try {
-                // Call all the pre-walk listeners. If atleast one of the pre walk listeners
+                // Call all the pre-walk listeners. If at least one of the pre walk listeners
                 // returns SKIP, then skip the walk.
                 if (keywordWalkListenerRunner.runPreWalkListeners(schemaPathWithKeyword,
                         node,
diff --git a/src/main/java/com/networknt/schema/PropertiesValidator.java b/src/main/java/com/networknt/schema/PropertiesValidator.java
@@ -17,6 +17,7 @@
 package com.networknt.schema;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.networknt.schema.walk.DefaultPropertyWalkListenerRunner;
 import com.networknt.schema.walk.WalkListenerRunner;
 import org.slf4j.Logger;
@@ -52,7 +53,6 @@ public Set<ValidationMessage> validate(JsonNode node, JsonNode rootNode, String
         for (Map.Entry<String, JsonSchema> entry : schemas.entrySet()) {
             JsonSchema propertySchema = entry.getValue();
             JsonNode propertyNode = node.get(entry.getKey());
-
             if (propertyNode != null) {
                 // check whether this is a complex validator. save the state
                 boolean isComplex = state.isComplexValidator();
@@ -102,6 +102,9 @@ public Set<ValidationMessage> validate(JsonNode node, JsonNode rootNode, String
     @Override
     public Set<ValidationMessage> walk(JsonNode node, JsonNode rootNode, String at, boolean shouldValidateSchema) {
         HashSet<ValidationMessage> validationMessages = new LinkedHashSet<ValidationMessage>();
+        if (applyDefaultsStrategy.shouldApplyPropertyDefaults()) {
+            applyPropertyDefaults((ObjectNode) node);
+        }
         if (shouldValidateSchema) {
             validationMessages.addAll(validate(node, rootNode, at));
         } else {
@@ -113,6 +116,21 @@ public Set<ValidationMessage> walk(JsonNode node, JsonNode rootNode, String at,
         return validationMessages;
     }
 
+    private void applyPropertyDefaults(ObjectNode node) {
+        for (Map.Entry<String, JsonSchema> entry : schemas.entrySet()) {
+            JsonNode propertyNode = node.get(entry.getKey());
+
+            if (propertyNode == null || (applyDefaultsStrategy.shouldApplyPropertyDefaultsIfNull() && propertyNode.isNull())) {
+                JsonSchema propertySchema = entry.getValue();
+                JsonNode defaultNode = propertySchema.getSchemaNode().get("default");
+                if (defaultNode != null && !defaultNode.isNull()) {
+                    // mutate the input json
+                    node.set(entry.getKey(), defaultNode);
+                }
+            }
+        }
+    }
+
     private void walkSchema(Map.Entry<String, JsonSchema> entry, JsonNode node, JsonNode rootNode, String at,
                             boolean shouldValidateSchema, Set<ValidationMessage> validationMessages, WalkListenerRunner propertyWalkListenerRunner) {
         JsonSchema propertySchema = entry.getValue();
diff --git a/src/main/java/com/networknt/schema/SchemaValidatorsConfig.java b/src/main/java/com/networknt/schema/SchemaValidatorsConfig.java
@@ -37,6 +37,11 @@ public class SchemaValidatorsConfig {
      */
     private boolean failFast;
 
+    /**
+     * When set to true, walker sets nodes that are missing or NullNode to the default value, if any, and mutate the input json.
+     */
+    private ApplyDefaultsStrategy applyDefaultsStrategy;
+
     /**
      * When set to true, use ECMA-262 compatible validator
      */
@@ -112,6 +117,14 @@ public boolean isFailFast() {
         return this.failFast;
     }
 
+    public void setApplyDefaultsStrategy(ApplyDefaultsStrategy applyDefaultsStrategy) {
+        this.applyDefaultsStrategy = applyDefaultsStrategy;
+    }
+
+    public ApplyDefaultsStrategy getApplyDefaultsStrategy() {
+        return applyDefaultsStrategy;
+    }
+
     public Map<String, String> getUriMappings() {
         // return a copy of the mappings
         return new HashMap<String, String>(uriMappings);
diff --git a/src/test/java/com/networknt/schema/JsonWalkApplyDefaultsTest.java b/src/test/java/com/networknt/schema/JsonWalkApplyDefaultsTest.java
diff --git a/src/test/resources/data/walk-data-default.json b/src/test/resources/data/walk-data-default.json
diff --git a/src/test/resources/schema/walk-schema-default.json b/src/test/resources/schema/walk-schema-default.json
