Issue 1102: Fixup test suite, fix broken tests

A recent PR broken the JsonChecker tests by adding support for trailing
commas. This didn't end up breaking the build, because those tests
aren't run, except locally and only using CMake.

This patch fixes the tests by adding exclusions for trailing comma
tests, as well as updates Meson to run these tests as part of `ninja
test`.

See issue #1102.

Author: baylesj

meson.build

@@ -105,3 +105,13 @@ test(
     jsontestrunner,
     join_paths(meson.current_source_dir(), 'test/data')]
   )
+test(
+  'jsonchecker_jsontestrunner',
+  python,
+  args : [
+    '-B',
+    join_paths(meson.current_source_dir(), 'test/runjsontests.py'),
+    '--with-json-checker',
+    jsontestrunner,
+    join_paths(meson.current_source_dir(), 'test/data')]
+  )

test/runjsontests.py

@@ -73,45 +73,26 @@ def runAllTests(jsontest_executable_path, input_dir = None,
         input_dir = os.path.join(os.getcwd(), 'data')
     tests = glob(os.path.join(input_dir, '*.json'))
     if with_json_checker:
-        all_test_jsonchecker = glob(os.path.join(input_dir, '../jsonchecker', '*.json'))
-        # These tests fail with strict json support, but pass with jsoncpp extra lieniency
-        """
-        Failure details:
-        * Test ../jsonchecker/fail25.json
-        Parsing should have failed:
-        ["	tab	character	in	string	"]
-
-        * Test ../jsonchecker/fail13.json
-        Parsing should have failed:
-        {"Numbers cannot have leading zeroes": 013}
-
-        * Test ../jsonchecker/fail18.json
-        Parsing should have failed:
-        [[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]
-
-        * Test ../jsonchecker/fail8.json
-        Parsing should have failed:
-        ["Extra close"]]
-
-        * Test ../jsonchecker/fail7.json
-        Parsing should have failed:
-        ["Comma after the close"],
-
-        * Test ../jsonchecker/fail10.json
-        Parsing should have failed:
-        {"Extra value after close": true} "misplaced quoted value"
-
-        * Test ../jsonchecker/fail27.json
-        Parsing should have failed:
-        ["line
-        break"]
-        """
-        known_differences_withjsonchecker = [ "fail25.json", "fail13.json", "fail18.json", "fail8.json",
-                                              "fail7.json", "fail10.json", "fail27.json" ]
-        test_jsonchecker = [ test for test in all_test_jsonchecker if os.path.basename(test) not in known_differences_withjsonchecker ]
+        all_tests = glob(os.path.join(input_dir, '../jsonchecker', '*.json'))
+        # These tests fail with strict json support, but pass with JsonCPP's
+        # extra leniency features. When adding a new exclusion to this list,
+        # remember to add the test's number and reasoning here:
+        known = ["fail{}.json".format(n) for n in [
+            4, 9, # fail because we allow trailing commas
+            7,    # fails because we allow commas after close
+            8,    # fails because we allow extra close
+            10,   # fails because we allow extra values after close
+            13,   # fails because we allow leading zeroes in numbers
+            18,   # fails because we allow deeply nested values
+            25,   # fails because we allow tab characters in strings.
+            27,   # fails because we allow string line breaks
+        ]]
+        test_jsonchecker = [ test for test in all_tests
+                             if os.path.basename(test) not in known]
 
     else:
         test_jsonchecker = []
+
     failed_tests = []
     valgrind_path = use_valgrind and VALGRIND_CMD or ''
     for input_path in tests + test_jsonchecker: