Merge branch '2.5.0dev' into update/examples

Author: nvnieuwk

CHANGELOG.md

@@ -15,12 +15,15 @@
 4. Updated the error message when the JSON schema file is invalid to include the full file name.
 5. The ANSI log setting of Nextflow is now used to determine whether or not the log should be monochrome. This setting will take priority over the `validation.monochromeLogs` configuration option.
 6. Updated the examples in the [examples](examples/) directory and added automatic checking for validity of these examples.
+7. Refactored logic for parsing the `jsonschema` validation result into a new `ValidationResult` class.
 
 ## Bug fixes
 
 1. CSV and TSV files with trailing commas or tabs will now be properly sanitized. This fixes issues with CSV and TSV files that contained empty header columns.
 2. Unidentified parameters are no longer printed out on failure of the parameter validation. This is to prevent a bug where all parameters would be printed out on failure.
 3. Fixed an issue where default values of `null` weren't being set correctly in `samplesheetToList()`.
+4. Fixed an undocumented limit of `3MB` for `YAML` format samplesheets (new limit is `50MB`).
+5. Fixed issue where an empty string in a yaml for a file type string format would throw a Java error instead of reporting a proper validation error.
 
 ## Logging configuration
 

src/main/groovy/nextflow/validation/ValidationExtension.groovy

@@ -19,6 +19,7 @@ import nextflow.validation.samplesheet.SamplesheetConverter
 import nextflow.validation.summary.SummaryCreator
 import nextflow.validation.parameters.ParameterValidator
 import nextflow.validation.validators.JsonSchemaValidator
+import nextflow.validation.validators.ValidationResult
 import static nextflow.validation.utils.Colors.getLogColors
 import static nextflow.validation.utils.Common.getBasePath
 import static nextflow.validation.utils.Common.getLongestKeyLength
@@ -132,7 +133,8 @@ class ValidationExtension extends PluginExtensionPoint {
         } else {
             jsonObj = input
         }
-        def List<String> errors = validator.validateObj(jsonObj, getBasePath(session.baseDir.toString(), schema))[0]
+        def ValidationResult result = validator.validate(jsonObj, getBasePath(session.baseDir.toString(), schema))
+        def List<String> errors = result.getErrors('object')
         if(exitOnError && errors != []) {
             def colors = getLogColors(config.monochromeLogs)
             def String msg = "${colors.red}${errors.join('\n')}${colors.reset}\n"

src/main/groovy/nextflow/validation/parameters/ParameterValidator.groovy

@@ -14,6 +14,7 @@ import nextflow.validation.validators.JsonSchemaValidator
 import static nextflow.validation.utils.Colors.getLogColors
 import static nextflow.validation.utils.Common.getBasePath
 import static nextflow.validation.utils.Common.getValueFromJsonPointer
+import nextflow.validation.validators.ValidationResult
 
 /**
  * @author : mirpedrol <[email protected]>
@@ -138,22 +139,23 @@ class ParameterValidator {
         def colors = getLogColors(config.monochromeLogs)
 
         // Validate
-        Tuple2<List<String>,List<String>> validationResult = validator.validate(paramsJSON, getBasePath(baseDir, schemaFilename))
-        def validationErrors = validationResult[0]
-        def unevaluatedParams = validationResult[1]
-        this.errors.addAll(validationErrors)
+        def ValidationResult validationResult = validator.validate(paramsJSON, getBasePath(baseDir, schemaFilename))
+        def List<String> paramErrors = validationResult.getErrors('parameter')
+        this.errors.addAll(paramErrors)
 
         //=====================================================================//
         // Check for nextflow core params and unexpected params
         //=====================================================================//
         def List<String> unexpectedParams = []
-        unevaluatedParams.each{ param ->
-            def String dotParam = param.replaceAll("/", ".")
-            if (NF_OPTIONS.contains(param)) {
-                errors << "You used a core Nextflow option with two hyphens: '--${param}'. Please resubmit with '-${param}'".toString()
-            }
-            else if (!config.ignoreParams.any { dotParam == it || dotParam.startsWith(it + ".") } ) { // Check if an ignore param is present
-                unexpectedParams << "* --${param.replaceAll("/", ".")}: ${getValueFromJsonPointer("/"+param, paramsJSON)}".toString()
+        if(paramErrors.size() == 0) {
+            validationResult.getUnevaluated().each{ param ->
+                def String dotParam = param.replaceAll("/", ".")
+                if (NF_OPTIONS.contains(param)) {
+                    errors << "You used a core Nextflow option with two hyphens: '--${param}'. Please resubmit with '-${param}'".toString()
+                }
+                else if (!config.ignoreParams.any { dotParam == it || dotParam.startsWith(it + ".") } ) { // Check if an ignore param is present
+                    unexpectedParams << "* --${param.replaceAll("/", ".")}: ${getValueFromJsonPointer("/"+param, paramsJSON)}".toString()
+                }
             }
         }
 

src/main/groovy/nextflow/validation/samplesheet/SamplesheetConverter.groovy

@@ -16,6 +16,7 @@ import static nextflow.validation.utils.Common.hasDeepKey
 import nextflow.validation.config.ValidationConfig
 import nextflow.validation.exceptions.SchemaValidationException
 import nextflow.validation.validators.JsonSchemaValidator
+import nextflow.validation.validators.ValidationResult
 
 /**
  * @author : mirpedrol <[email protected]>
@@ -98,8 +99,8 @@ class SamplesheetConverter {
         // Validate
         final validator = new JsonSchemaValidator(config)
         def JSONArray samplesheet = fileToJson(samplesheetFile, schemaFile) as JSONArray
-        def Tuple2<List<String>,List<String>> validationResults = validator.validate(samplesheet, schemaFile.toString())
-        def validationErrors = validationResults[0]
+        def ValidationResult validationResult = validator.validate(samplesheet, schemaFile.toString())
+        def validationErrors = validationResult.getErrors('field')
         if (validationErrors) {
             def msg = "${colors.red}The following errors have been detected in ${samplesheetFile.toString()}:\n\n" + validationErrors.join('\n').trim() + "\n${colors.reset}\n"
             log.error("Validation of samplesheet failed!")

src/main/groovy/nextflow/validation/utils/Files.groovy

@@ -1,6 +1,7 @@
 package nextflow.validation.utils
 
 import org.yaml.snakeyaml.Yaml
+import org.yaml.snakeyaml.LoaderOptions
 import org.json.JSONArray
 import org.json.JSONObject
 import org.json.JSONPointer
@@ -85,7 +86,9 @@ public class Files {
         }
 
         if(fileType == "yaml"){
-            return new Yaml().load((file.text))
+            def LoaderOptions yamlLoaderOptions = new LoaderOptions()
+            yamlLoaderOptions.setCodePointLimit(50 * 1024 * 1024)
+            return new Yaml(yamlLoaderOptions).load((file.text))
         }
         else if(fileType == "json"){
             return new JsonSlurper().parseText(file.text)

src/main/groovy/nextflow/validation/validators/JsonSchemaValidator.groovy

@@ -12,16 +12,11 @@ import dev.harrel.jsonschema.FormatEvaluatorFactory
 import dev.harrel.jsonschema.JsonNode
 import dev.harrel.jsonschema.providers.OrgJsonNode
 
-import java.util.regex.Pattern
-import java.util.regex.Matcher
-
-import static nextflow.validation.utils.Common.getValueFromJsonPointer
-import static nextflow.validation.utils.Common.findAllKeys
-import static nextflow.validation.utils.Common.kebabToCamel
-import static nextflow.validation.utils.Types.isInteger
 import nextflow.validation.config.ValidationConfig
 import nextflow.validation.exceptions.SchemaValidationException
 import nextflow.validation.validators.evaluators.CustomEvaluatorFactory
+import nextflow.validation.validators.ValidationResult
+import static nextflow.validation.utils.Common.getValueFromJsonPointer
 
 /**
  * @author : nvnieuwk <[email protected]>
@@ -31,7 +26,6 @@ import nextflow.validation.validators.evaluators.CustomEvaluatorFactory
 public class JsonSchemaValidator {
 
     private ValidatorFactory validator
-    private Pattern uriPattern = Pattern.compile('^#/(\\d*)?/?(.*)$')
     private ValidationConfig config
 
     JsonSchemaValidator(ValidationConfig config) {
@@ -42,10 +36,12 @@ public class JsonSchemaValidator {
         this.config = config
     }
 
-    private Tuple2<List<String>,List<String>> validateObject(JsonNode input, String validationType, Object rawJson, String schemaFileName) {
+    private ValidationResult validateObject(JsonNode input, Object rawJson, String schemaFileName) {
         def JSONObject schema
+        def String schemaString
         try {
-            schema = new JSONObject(Files.readString(Path.of(schemaFileName)))
+            schemaString = Files.readString(Path.of(schemaFileName))
+            schema = new JSONObject(schemaString)
         } catch (org.json.JSONException e) {
             throw new SchemaValidationException("""Failed to load JSON schema (${schemaFileName}):
     ${e.message}
@@ -64,106 +60,13 @@ public class JsonSchemaValidator {
             """)
             throw new SchemaValidationException("", [])
         }
-        
         def Validator.Result result = this.validator.validate(schema, input)
-        def List<String> errors = []
-        result.getErrors().each { error ->
-            def String errorString = error.getError()
-
-            // Skip double error in the parameter schema
-            if (errorString.startsWith("Value does not match against the schemas at indexes") && validationType == "parameter") {
-                return
-            }
-
-            def String instanceLocation = error.getInstanceLocation()
-            def String value = getValueFromJsonPointer(instanceLocation, rawJson)
-            if(config.maxErrValSize >= 1 && value.size() > config.maxErrValSize) {
-                value = "${value[0..(config.maxErrValSize/2-1)]}...${value[-config.maxErrValSize/2..-1]}" as String
-            }
-
-            // Return a standard error message for object validation
-            if (validationType == "object") {
-                errors.add("${instanceLocation ? instanceLocation + ' ' : ''}(${value}): ${errorString}" as String)
-                return
-            }
-
-            // Get the custom errorMessage if there is one and the validation errors are not about the content of the file
-            def String schemaLocation = error.getSchemaLocation().replaceFirst(/^[^#]+/, "")
-            def String customError = ""
-            if (!errorString.startsWith("Validation of file failed:")) {
-                customError = getValueFromJsonPointer("${schemaLocation}/errorMessage", schema) as String
-            }
-
-            // Change some error messages to make them more clear
-            def String keyword = error.getKeyword()
-            if (keyword == "required") {
-                def Matcher matcher = errorString =~ ~/\[\[([^\[\]]*)\]\]$/
-                def String missingKeywords = matcher.findAll().flatten().last()
-                errorString = "Missing required ${validationType}(s): ${missingKeywords}"
-            }
-
-            def List<String> locationList = instanceLocation.split("/").findAll { it != "" } as List
-
-            def String printableError = "${validationType == 'field' ? '->' : '*'} ${errorString}" as String
-            if (locationList.size() > 0 && isInteger(locationList[0]) && validationType == "field") {
-                def Integer entryInteger = locationList[0] as Integer
-                def String entryString = "Entry ${entryInteger + 1}" as String
-                def String fieldError = "${errorString}" as String
-                if(locationList.size() > 1) {
-                    fieldError = "Error for ${validationType} '${locationList[1..-1].join("/")}' (${value}): ${errorString}"
-                }
-                printableError = "-> ${entryString}: ${fieldError}" as String
-            } else if (validationType == "parameter") {
-                def String fieldName = locationList.join(".")
-                if(fieldName != "") {
-                    printableError = "* --${fieldName} (${value}): ${errorString}" as String
-                }
-            }
-
-            if(customError != "") {
-                printableError = printableError + " (${customError})"
-            }
-
-            errors.add(printableError)
-
-        }
-        def List<String> unevaluated = []
-        if(errors.size() == 0) { 
-            unevaluated = getUnevaluated(result, rawJson)
-        }
-        return Tuple.tuple(errors, unevaluated)
-    }
-
-    public Tuple2<List<String>,List<String>> validate(JSONArray input, String schemaFileName) {
-        def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input)
-        return this.validateObject(jsonInput, "field", input, schemaFileName)
+        return new ValidationResult(result, rawJson, schemaString, config)
     }
 
-    public Tuple2<List<String>,List<String>> validate(JSONObject input, String schemaFileName) {
+    public ValidationResult validate(Object input, String schemaFileName) {
         def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input)
-        return this.validateObject(jsonInput, "parameter", input, schemaFileName)
+        return this.validateObject(jsonInput, input, schemaFileName)
     }
 
-    public Tuple2<List<String>,List<String>> validateObj(Object input, String schemaFileName) {
-        def JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input)
-        return this.validateObject(jsonInput, "object", input, schemaFileName)
-    }
-
-    public static List<String> getUnevaluated(Validator.Result result, Object rawJson) {
-        def Set<String> evaluated = []
-        result.getAnnotations().each{ anno ->
-            if(anno.keyword in ["properties", "patternProperties", "additionalProperties"]){
-                evaluated.addAll(
-                    anno.annotation.collect{ it ->
-                    "${anno.instanceLocation.toString()}/${it.toString()}".replaceAll("^/+", "")
-                    }
-                )
-            }
-        }
-        def Set<String> all_keys = []
-        findAllKeys(rawJson, null, all_keys, '/')
-        def unevaluated_ = all_keys - evaluated
-        def unevaluated = unevaluated_.collect{ it -> !evaluated.contains(kebabToCamel(it)) ? it : null }
-        return unevaluated - null
-    }
 }

src/main/groovy/nextflow/validation/validators/ValidationResult.groovy

@@ -0,0 +1,118 @@
+package nextflow.validation.validators
+
+import groovy.util.logging.Slf4j
+
+import java.util.regex.Matcher
+
+import org.json.JSONObject
+import dev.harrel.jsonschema.ValidatorFactory
+import dev.harrel.jsonschema.Validator
+
+import nextflow.validation.config.ValidationConfig
+import static nextflow.validation.utils.Common.getValueFromJsonPointer
+import static nextflow.validation.utils.Common.findAllKeys
+import static nextflow.validation.utils.Common.kebabToCamel
+import static nextflow.validation.utils.Types.isInteger
+
+
+@Slf4j
+public class ValidationResult {
+
+    public Validator.Result result
+    private Object rawInput 
+    private String schemaString
+    private JSONObject schemaJson
+    private ValidationConfig config
+
+    ValidationResult(Validator.Result result, Object rawInput, String schemaString, ValidationConfig config ) {
+        this.result = result
+        this.rawInput = rawInput
+        this.schemaString = schemaString
+        this.schemaJson = new JSONObject(schemaString)
+        this.config = config
+    }
+
+    public List<String> getUnevaluated() {
+        def Set<String> evaluated = []
+        this.result.getAnnotations().each{ anno ->
+            if(anno.keyword in ["properties", "patternProperties", "additionalProperties"]){
+                evaluated.addAll(
+                    anno.annotation.collect{ it ->
+                    "${anno.instanceLocation.toString()}/${it.toString()}".replaceAll("^/+", "")
+                    }
+                )
+            }
+        }
+        def Set<String> all_keys = []
+        findAllKeys(this.rawInput, null, all_keys, '/')
+        def unevaluated_ = all_keys - evaluated
+        def unevaluated = unevaluated_.collect{ it -> !evaluated.contains(kebabToCamel(it)) ? it : null }
+        return unevaluated - null
+    }
+
+    public List<String> getErrors(String validationType) {
+        def List<String> errors = []
+        this.result.getErrors().each { error ->
+            def String errorString = error.getError()
+
+            // Skip double error in the parameter schema
+            if (errorString.startsWith("Value does not match against the schemas at indexes") && validationType == "parameter") {
+                return
+            }
+
+            def String instanceLocation = error.getInstanceLocation()
+            def String value = getValueFromJsonPointer(instanceLocation, this.rawInput)
+            if(config.maxErrValSize >= 1 && value.size() > config.maxErrValSize) {
+                value = "${value[0..(config.maxErrValSize/2-1)]}...${value[-config.maxErrValSize/2..-1]}" as String
+            }
+
+            // Return a standard error message for object validation
+            if (validationType == "object") {
+                errors.add("${instanceLocation ? instanceLocation + ' ' : ''}(${value}): ${errorString}" as String)
+                return
+            }
+
+            // Get the custom errorMessage if there is one and the validation errors are not about the content of the file
+            def String schemaLocation = error.getSchemaLocation().replaceFirst(/^[^#]+/, "")
+            def String customError = ""
+            if (!errorString.startsWith("Validation of file failed:")) {
+                customError = getValueFromJsonPointer("${schemaLocation}/errorMessage", this.schemaJson) as String
+            }
+
+            // Change some error messages to make them more clear
+            def String keyword = error.getKeyword()
+            if (keyword == "required") {
+                def Matcher matcher = errorString =~ ~/\[\[([^\[\]]*)\]\]$/
+                def String missingKeywords = matcher.findAll().flatten().last()
+                errorString = "Missing required ${validationType}(s): ${missingKeywords}"
+            }
+
+            def List<String> locationList = instanceLocation.split("/").findAll { it != "" } as List
+
+            def String printableError = "${validationType == 'field' ? '->' : '*'} ${errorString}" as String
+            if (locationList.size() > 0 && isInteger(locationList[0]) && validationType == "field") {
+                def Integer entryInteger = locationList[0] as Integer
+                def String entryString = "Entry ${entryInteger + 1}" as String
+                def String fieldError = "${errorString}" as String
+                if(locationList.size() > 1) {
+                    fieldError = "Error for ${validationType} '${locationList[1..-1].join("/")}' (${value}): ${errorString}"
+                }
+                printableError = "-> ${entryString}: ${fieldError}" as String
+            } else if (validationType == "parameter") {
+                def String fieldName = locationList.join(".")
+                if(fieldName != "") {
+                    printableError = "* --${fieldName} (${value}): ${errorString}" as String
+                }
+            }
+
+            if(customError != "") {
+                printableError = printableError + " (${customError})"
+            }
+
+            errors.add(printableError)
+
+        }
+        return errors
+    }
+
+}

src/main/groovy/nextflow/validation/validators/evaluators/ExistsEvaluator.groovy

@@ -30,8 +30,8 @@ class ExistsEvaluator implements Evaluator {
         }
 
         def String value = node.asString()
-        def Path file = Nextflow.file(value) as Path
         def Boolean exists
+        def Path file
 
         try {
             file = Nextflow.file(value) as Path