schema: Add 'test' target to the Makefile

And fill in some known-good and known-bad examples.  We can make this
as detailed as we want, but this commit just adds enough to know that:

* The full-file spec examples are valid.
* The JSON Schema can distinguish valid examples from invalid JSON.

This will help catch JSON Schema typos like those being addressed by
[1].

[1]: #784

Signed-off-by: W. Trevor King <[email protected]>

Author: wking

.travis.yml

@@ -22,3 +22,4 @@ script:
   - echo "${TRAVIS_COMMIT_RANGE} -> ${TRAVIS_COMMIT_RANGE/.../..} (travis-ci/travis-ci#4596)"
   - TRAVIS_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE/.../..}" make .gitvalidation
   - make docs
+  - make -C schema test

schema/Makefile

@@ -1,3 +1,5 @@
+GOOD_TESTS = $(wildcard test/good/*.json)
+BAD_TESTS = $(wildcard test/bad/*.json)
 
 .PHONY: default
 default: validate
@@ -12,13 +14,42 @@ help:
 
 .PHONY: fmt
 fmt:
-	for i in *.json ; do jq --indent 4 -M . "$${i}" > xx && cat xx > "$${i}" && rm xx ; done
+	find . -name '*.json' -exec bash -c 'jq --indent 4 -M . {} > xx && mv xx {} || echo "skipping invalid {}"' \;
 
 .PHONY: validate
 validate: validate.go
 	go get -d ./...
 	go build ./validate.go
 
+.PHONY: test
+test: validate $(TESTS)
+	for TYPE in $$(ls test); \
+	do \
+		echo "testing $${TYPE}"; \
+		for FILE in $$(ls "test/$${TYPE}/good"); \
+		do \
+			echo "  testing test/$${TYPE}/good/$${FILE}"; \
+			if ./validate "$${TYPE}-schema.json" "test/$${TYPE}/good/$${FILE}" ; \
+			then \
+				echo "    received expected validation success" ; \
+			else \
+				echo "    received unexpected validation failure" ; \
+				exit 1; \
+			fi \
+		done; \
+		for FILE in $$(ls "test/$${TYPE}/bad"); \
+		do \
+			echo "  testing test/$${TYPE}/bad/$${FILE}"; \
+			if ./validate "$${TYPE}-schema.json" "test/$${TYPE}/good/$${FILE}" ; \
+			then \
+				echo "    received unexpected validation success" ; \
+				exit 1; \
+			else \
+				echo "    received expected validation failure" ; \
+			fi \
+		done; \
+	done
+
 .PHONY: clean
 clean:
 	rm -f validate

schema/test/config/bad/invalid-json.json

@@ -0,0 +1 @@
+{]

schema/test/config/good/minimal.json

@@ -0,0 +1,18 @@
+{
+    "ociVersion": "1.0.0",
+    "platform": {
+        "os": "linux",
+        "arch": "amd64"
+    },
+    "root": {
+        "path": "rootfs"
+    },
+    "process": {
+        "cwd": "/",
+        "args": ["sh"],
+        "user": {
+            "uid": 0,
+            "gid": 0
+        }
+    }
+}