Update to jsonschema>=4.18 and 'referencing'

There's a new implementation of ref resolution in newer versions of
'jsonschema'. Upgrade to use this, plus add support for loading YAML,
JSON5, and TOML schemas as references (so long as they have the
filetype suffixes which make them easily detectable).

Author: sirosen

setup.cfg

@@ -20,7 +20,7 @@ python_requires = >=3.7
 install_requires =
     importlib-resources>=1.4.0;python_version<"3.9"
     ruamel.yaml==0.17.32
-    jsonschema>=4.5.1,<5.0
+    jsonschema>=4.18.0,<5.0
     requests<3.0
     click>=8,<9
 package_dir=

src/check_jsonschema/checker.py

@@ -5,6 +5,7 @@
 
 import click
 import jsonschema
+import referencing.exceptions
 
 from . import utils
 from .formats import FormatOptions
@@ -75,7 +76,11 @@ def _build_result(self) -> CheckResult:
     def _run(self) -> None:
         try:
             result = self._build_result()
-        except jsonschema.RefResolutionError as e:
+        except (
+            referencing.exceptions.NoSuchResource,
+            referencing.exceptions.Unretrievable,
+            referencing.exceptions.Unresolvable,
+        ) as e:
             self._fail("Failure resolving $ref within schema\n", e)
 
         self._reporter.report_result(result)

src/check_jsonschema/identify_filetype.py

@@ -18,8 +18,11 @@
 }
 
 
-def path_to_type(path: pathlib.Path, *, default_type: str = "json") -> str:
-    ext = path.suffix.lstrip(".")
+def path_to_type(path: str | pathlib.Path, *, default_type: str = "json") -> str:
+    if isinstance(path, str):
+        ext = path.rpartition(".")[2]
+    else:
+        ext = path.suffix.lstrip(".")
 
     if ext in _EXTENSION_MAP:
         return _EXTENSION_MAP[ext]

src/check_jsonschema/parsers/__init__.py

@@ -82,10 +82,15 @@ def get(
             + ",".join(self._by_tag.keys())
         )
 
-    def parse_file(self, path: pathlib.Path, default_filetype: str) -> t.Any:
+    def parse_data_with_path(
+        self, data: t.BinaryIO, path: pathlib.Path | str, default_filetype: str
+    ) -> t.Any:
         loadfunc = self.get(path, default_filetype)
         try:
-            with open(path, "rb") as fp:
-                return loadfunc(fp)
+            return loadfunc(data)
         except LOADING_FAILURE_ERROR_TYPES as e:
             raise FailedFileLoadError(f"Failed to parse {path}") from e
+
+    def parse_file(self, path: pathlib.Path | str, default_filetype: str) -> t.Any:
+        with open(path, "rb") as fp:
+            return self.parse_data_with_path(fp, path, default_filetype)

src/check_jsonschema/schema_loader/main.py

@@ -9,10 +9,11 @@
 
 from ..builtin_schemas import get_builtin_schema
 from ..formats import FormatOptions, make_format_checker
+from ..parsers import ParserSet
 from ..utils import is_url_ish
 from .errors import UnsupportedUrlScheme
 from .readers import HttpSchemaReader, LocalSchemaReader
-from .resolver import make_ref_resolver
+from .resolver import make_reference_registry
 
 
 def _extend_with_default(
@@ -71,6 +72,9 @@ def __init__(
         if is_url_ish(self.schemafile):
             self.url_info = urllib.parse.urlparse(self.schemafile)
 
+        # setup a parser collection
+        self._parsers = ParserSet()
+
         # setup a schema reader lazily, when needed
         self._reader: LocalSchemaReader | HttpSchemaReader | None = None
 
@@ -117,11 +121,9 @@ def get_validator(
         # format checker (which may be None)
         format_checker = make_format_checker(format_opts, schema_dialect)
 
-        # ref resolver which may be built from the schema path
-        # if the location is a URL, there's no change, but if it's a file path
-        # it's made absolute and URI-ized
-        # the resolver should use `$id` if there is one present in the schema
-        ref_resolver = make_ref_resolver(schema_uri, schema)
+        # reference resolution
+        # with support for YAML, TOML, and other formats from the parsers
+        reference_registry = make_reference_registry(self._parsers, schema_uri, schema)
 
         # get the correct validator class and check the schema under its metaschema
         validator_cls = jsonschema.validators.validator_for(schema)
@@ -134,7 +136,7 @@ def get_validator(
         # now that we know it's safe to try to create the validator instance, do it
         validator = validator_cls(
             schema,
-            resolver=ref_resolver,
+            registry=reference_registry,
             format_checker=format_checker,
         )
         return t.cast(jsonschema.Validator, validator)
@@ -143,6 +145,7 @@ def get_validator(
 class BuiltinSchemaLoader(SchemaLoader):
     def __init__(self, schema_name: str) -> None:
         self.schema_name = schema_name
+        self._parsers = ParserSet()
 
     def get_schema_ref_base(self) -> str | None:
         return None

src/check_jsonschema/schema_loader/readers.py

@@ -25,12 +25,10 @@ def _run_load_callback(schema_location: str, callback: t.Callable) -> dict:
 
 
 class LocalSchemaReader:
-    FORMATS = ("json", "json5", "yaml")
-
     def __init__(self, filename: str) -> None:
         self.path = filename2path(filename)
         self.filename = str(self.path)
-        self.parsers = ParserSet(supported_formats=self.FORMATS)
+        self.parsers = ParserSet()
 
     def get_ref_base(self) -> str:
         return self.path.as_uri()

src/check_jsonschema/schema_loader/resolver.py

@@ -1,42 +1,63 @@
 from __future__ import annotations
 
+import pathlib
 import typing as t
-
-import click
-import jsonschema
-
-
-class _CliRefResolver(jsonschema.RefResolver):
-    def resolve_remote(self, uri: str) -> t.Any:
-        if uri.endswith(".yaml") or uri.endswith(".yml"):
-            click.secho(
-                """\
-WARNING: You appear to be using a schema which references a YAML file.
-
-This is not supported by check-jsonschema and may result in errors.
-""",
-                err=True,
-                fg="yellow",
-            )
-        elif uri.endswith(".json5"):
-            click.secho(
-                """\
-WARNING: You appear to be using a schema which references a JSON5 file.
-
-This is not supported by check-jsonschema and may result in errors.
-""",
-                err=True,
-                fg="yellow",
-            )
-        return super().resolve_remote(uri)
-
-
-def make_ref_resolver(
-    schema_uri: str | None, schema: dict
-) -> jsonschema.RefResolver | None:
-    if not schema_uri:
-        return None
-
-    base_uri = schema.get("$id", schema_uri)
-    # FIXME: temporary type-ignore because typeshed has the type wrong
-    return _CliRefResolver(base_uri, schema)  # type: ignore[arg-type]
+import urllib.parse
+
+import referencing
+import requests
+from referencing.jsonschema import DRAFT202012
+
+from ..parsers import ParserSet
+from ..utils import filename2path
+
+
+def make_reference_registry(
+    parsers: ParserSet, schema_uri: str | None, schema: dict
+) -> referencing.Registry:
+    schema_resource = referencing.Resource.from_contents(
+        schema, default_specification=DRAFT202012
+    )
+    registry = referencing.Registry(
+        retrieve=create_retrieve_callable(parsers, schema_uri)
+    )
+
+    if schema_uri is not None:
+        registry = registry.with_resource(uri=schema_uri, resource=schema_resource)
+
+    id_attribute = schema.get("$id")
+    if id_attribute is not None:
+        registry = registry.with_resource(uri=id_attribute, resource=schema_resource)
+
+    return registry
+
+
+def create_retrieve_callable(
+    parser_set: ParserSet, schema_uri: str | None
+) -> t.Callable[[str], referencing.Resource]:
+    def get_local_file(uri: str):
+        path = pathlib.Path(uri)
+        if not path.is_absolute():
+            if schema_uri is None:
+                raise referencing.exceptions.Unretrievable(
+                    f"Cannot retrieve schema reference data for '{uri}' from "
+                    "local filesystem. "
+                    "The path appears relative, but there is no known local base path."
+                )
+            schema_path = filename2path(schema_uri)
+            path = schema_path.parent / path
+        return parser_set.parse_file(path, "json")
+
+    def retrieve_reference(uri: str) -> referencing.Resource:
+        scheme = urllib.parse.urlsplit(uri).scheme
+        if scheme in ("http", "https"):
+            data = requests.get(uri, stream=True)
+            parsed_object = parser_set.parse_data_with_path(data.raw, uri, "json")
+        else:
+            parsed_object = get_local_file(uri)
+
+        return referencing.Resource.from_contents(
+            parsed_object, default_specification=DRAFT202012
+        )
+
+    return retrieve_reference

tests/acceptance/conftest.py

@@ -1,9 +1,21 @@
+import textwrap
+
 import pytest
 from click.testing import CliRunner
 
 from check_jsonschema import main as cli_main
 
 
+def _render_result(result):
+    return f"""
+output:
+{textwrap.indent(result.output, "  ")}
+
+stderr:
+{textwrap.indent(result.stderr, "  ")}
+"""
+
+
 @pytest.fixture
 def cli_runner():
     return CliRunner(mix_stderr=False)
@@ -22,8 +34,14 @@ def func(cli_args, *args, **kwargs):
 
 @pytest.fixture
 def run_line_simple(run_line):
-    def func(cli_args, *args, **kwargs):
-        res = run_line(["check-jsonschema"] + cli_args, *args, **kwargs)
-        assert res.exit_code == 0
+    def func(cli_args, *args, full_traceback: bool = True, **kwargs):
+        res = run_line(
+            ["check-jsonschema"]
+            + (["--traceback-mode", "full"] if full_traceback else [])
+            + cli_args,
+            *args,
+            **kwargs,
+        )
+        assert res.exit_code == 0, _render_result(res)
 
     return func

tests/acceptance/test_nonjson_schema_handling.py

@@ -31,9 +31,10 @@
 
 
 @pytest.mark.parametrize("passing_data", [True, False])
-def test_warning_on_yaml_reference_passes(run_line, tmp_path, passing_data):
+def test_yaml_reference(run_line, tmp_path, passing_data):
     main_schemafile = tmp_path / "main_schema.json"
     main_schemafile.write_text(json.dumps(YAML_REF_MAIN_SCHEMA))
+    # JSON is a subset of YAML, so this works for generated YAML
     ref_schema = tmp_path / "title_schema.yaml"
     ref_schema.write_text(json.dumps(TITLE_SCHEMA))
 
@@ -47,14 +48,11 @@ def test_warning_on_yaml_reference_passes(run_line, tmp_path, passing_data):
         ["check-jsonschema", "--schemafile", str(main_schemafile), str(doc)]
     )
     assert result.exit_code == (0 if passing_data else 1)
-    assert (
-        "WARNING: You appear to be using a schema which references a YAML file"
-        in result.stderr
-    )
 
 
+@pytest.mark.skipif(not JSON5_ENABLED, reason="test requires json5")
 @pytest.mark.parametrize("passing_data", [True, False])
-def test_warning_on_json5_reference(run_line, tmp_path, passing_data):
+def test_json5_reference(run_line, tmp_path, passing_data):
     main_schemafile = tmp_path / "main_schema.json"
     main_schemafile.write_text(json.dumps(JSON5_REF_MAIN_SCHEMA))
     ref_schema = tmp_path / "title_schema.json5"
@@ -70,10 +68,6 @@ def test_warning_on_json5_reference(run_line, tmp_path, passing_data):
         ["check-jsonschema", "--schemafile", str(main_schemafile), str(doc)]
     )
     assert result.exit_code == (0 if passing_data else 1)
-    assert (
-        "WARNING: You appear to be using a schema which references a JSON5 file"
-        in result.stderr
-    )
 
 
 @pytest.mark.skipif(not JSON5_ENABLED, reason="test requires json5")

tox.ini

@@ -87,4 +87,3 @@ commands = python ./scripts/generate-hooks-config.py
 [pytest]
 filterwarnings =
     error
-    ignore:jsonschema\.(RefResolver|exceptions\.RefResolutionError) is deprecated as of (version |v)4\.18\.0:DeprecationWarning