test: Use in-memory resolvers for JSON Schema test suite

Signed-off-by: Dmitry Dygalo <[email protected]>

Author: Stranger6667

.github/workflows/ci.yml

@@ -110,6 +110,8 @@ jobs:
           tool: '[email protected]'
 
       - name: Run tests
+        env:
+          JSONSCHEMA_SUITE_DRAFT_FILTER: draft2020-12
         run: |
           if [ -n "${{ matrix.features }}" ]; then
             cargo test --target ${{ matrix.target }} --no-default-features --features "${{ matrix.features }}" -p jsonschema

crates/jsonschema-py/tests-py/test_suite.py

@@ -1,53 +1,47 @@
 import json
 import os
-import random
-import socket
-import subprocess
-import sys
 from pathlib import Path
-from time import sleep
-from urllib.parse import urlparse
 
 import pytest
 
 import jsonschema_rs
 
 TEST_SUITE_PATH = Path(__file__).parent.parent.parent / "jsonschema/tests/suite"
-EXPONENTIAL_BASE = 2
-JITTER = (0.0, 0.5)
-INITIAL_RETRY_DELAY = 0.05
-MAX_WAITING_RETRIES = 10
+REMOTE_PREFIXES = ("http://localhost:1234", "https://localhost:1234")
 
 
-def is_available(url: str) -> bool:
-    """Whether the `url` is available for connection or not."""
-    parsed = urlparse(url)
-    try:
-        with socket.create_connection((parsed.hostname, parsed.port or 80)):
-            return True
-    except ConnectionError:
-        return False
-
-
-def wait_until_responsive(url: str, retries: int = MAX_WAITING_RETRIES, delay: float = INITIAL_RETRY_DELAY) -> None:
-    while retries > 0:
-        if is_available(url):
-            return
-        retries -= 1
-        delay *= EXPONENTIAL_BASE
-        delay += random.uniform(*JITTER)
-        sleep(delay)
-    raise RuntimeError(f"{url} is not available")
-
-
-@pytest.fixture(scope="session", autouse=True)
-def mock_server():
-    process = subprocess.Popen(args=[sys.executable, TEST_SUITE_PATH / "bin/jsonschema_suite", "serve"])
-    wait_until_responsive("http://127.0.0.1:1234")
-    try:
-        yield
-    finally:
-        process.terminate()
+def load_remote_documents():
+    remotes = TEST_SUITE_PATH / "remotes"
+    documents = {}
+    for root, _, files in os.walk(remotes):
+        for filename in files:
+            path = Path(root) / filename
+            relative = path.relative_to(remotes).as_posix()
+            contents = path.read_text(encoding="utf-8")
+            for prefix in REMOTE_PREFIXES:
+                documents[f"{prefix}/{relative}"] = contents
+    return documents
+
+
+REMOTE_DOCUMENTS = load_remote_documents()
+
+
+def make_testsuite_retriever():
+    localhost_alias = "http://127.0.0.1:1234/"
+
+    def _retriever(uri: str):
+        normalized = uri
+        if normalized.startswith(localhost_alias):
+            normalized = f"http://localhost:1234/{normalized[len(localhost_alias):]}"
+        try:
+            return json.loads(REMOTE_DOCUMENTS[normalized])
+        except KeyError as exc:
+            raise ValueError(f"Unknown remote schema: {uri}") from exc
+
+    return _retriever
+
+
+TESTSUITE_RETRIEVER = make_testsuite_retriever()
 
 
 SUPPORTED_DRAFTS = ("4", "6", "7", "2019-09", "2020-12")
@@ -62,8 +56,7 @@ def mock_server():
 
 def load_file(path):
     with open(path, mode="r", encoding="utf-8") as fd:
-        raw = fd.read().replace("https://localhost:1234", "http://127.0.0.1:1234")
-        for block in json.loads(raw):
+        for block in json.load(fd):
             yield block
 
 
@@ -98,7 +91,7 @@ def test_draft(filename, draft, schema, instance, expected, description, is_opti
             "2019-09": jsonschema_rs.Draft201909Validator,
             "2020-12": jsonschema_rs.Draft202012Validator,
         }[draft]
-        kwargs = {}
+        kwargs = {"retriever": TESTSUITE_RETRIEVER}
         if is_optional:
             kwargs["validate_formats"] = True
         result = cls(schema, **kwargs).is_valid(instance)

crates/jsonschema-testsuite-codegen/src/generator.rs

@@ -74,7 +74,8 @@ fn generate_nested_structure(
 
                         quote! {
                             #ignore_attr
-                            #[test]
+                            #[cfg_attr(not(all(target_arch = "wasm32", target_os = "unknown")), test)]
+                            #[cfg_attr(all(target_arch = "wasm32", target_os = "unknown"), wasm_bindgen_test::wasm_bindgen_test)]
                             fn #test_ident() {
                                 let test = testsuite::Test {
                                     draft: #draft,

crates/jsonschema-testsuite-codegen/src/lib.rs

@@ -1,13 +1,12 @@
 use std::collections::HashSet;
 
 use proc_macro::TokenStream;
-
 use quote::{format_ident, quote};
 use syn::{parse_macro_input, ItemFn};
 mod generator;
 mod idents;
 mod loader;
-mod mocks;
+mod remotes;
 
 /// A procedural macro that generates tests from
 /// [JSON-Schema-Test-Suite](https://github.com/json-schema-org/JSON-Schema-Test-Suite).
@@ -17,8 +16,8 @@ pub fn suite(args: TokenStream, input: TokenStream) -> TokenStream {
     let test_func = parse_macro_input!(input as ItemFn);
     let test_func_ident = &test_func.sig.ident;
 
-    let mocks = match mocks::generate(&config.path) {
-        Ok(mocks) => mocks,
+    let remotes = match remotes::generate(&config.path) {
+        Ok(remotes) => remotes,
         Err(e) => {
             let err = e.to_string();
             return TokenStream::from(quote! {
@@ -30,15 +29,42 @@ pub fn suite(args: TokenStream, input: TokenStream) -> TokenStream {
     let mut output = quote! {
         #test_func
 
-        static MOCK: std::sync::LazyLock<mockito::Server> = std::sync::LazyLock::new(|| {
-            #mocks
-        });
+        #remotes
+
+        struct TestsuiteRetriever;
+
+        impl jsonschema::Retrieve for TestsuiteRetriever {
+            fn retrieve(
+                &self,
+                uri: &jsonschema::Uri<String>,
+            ) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
+                static REMOTE_MAP: std::sync::LazyLock<std::collections::HashMap<&'static str, &'static str>> =
+                    std::sync::LazyLock::new(|| {
+                        let mut map = std::collections::HashMap::with_capacity(REMOTE_DOCUMENTS.len());
+                        for (uri, contents) in REMOTE_DOCUMENTS {
+                            map.insert(*uri, *contents);
+                        }
+                        map
+                    });
+                match REMOTE_MAP.get(uri.as_str()) {
+                    Some(contents) => Ok(serde_json::from_str(contents)
+                        .expect("Failed to parse remote schema")),
+                    None => Err(format!("Unknown remote: {}", uri).into()),
+                }
+            }
+        }
+
+        fn testsuite_retriever() -> TestsuiteRetriever {
+            TestsuiteRetriever
+        }
+
+        #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
+        use wasm_bindgen_test::wasm_bindgen_test;
     };
     // There are a lot of tests in the test suite
     let mut functions = HashSet::with_capacity(7200);
-
-    for draft in config.drafts {
-        let suite_tree = match loader::load_suite(&config.path, &draft) {
+    for draft in &config.drafts {
+        let suite_tree = match loader::load_suite(&config.path, draft) {
             Ok(tree) => tree,
             Err(e) => {
                 let err = e.to_string();
@@ -48,18 +74,17 @@ pub fn suite(args: TokenStream, input: TokenStream) -> TokenStream {
             }
         };
         let modules =
-            generator::generate_modules(&suite_tree, &mut functions, &config.xfail, &draft);
-        let draft = format_ident!("{}", &draft.replace('-', "_"));
+            generator::generate_modules(&suite_tree, &mut functions, &config.xfail, draft);
+        let module_ident = format_ident!("{}", &draft.replace('-', "_"));
         output = quote! {
             #output
 
-            mod #draft {
+            mod #module_ident {
                 use testsuite::Test;
-                use super::{#test_func_ident, MOCK};
+                use super::#test_func_ident;
 
                 #[inline]
                 fn inner_test(test: Test) {
-                    let _ = &*MOCK;
                     #test_func_ident(test);
                 }
                 #modules

crates/jsonschema-testsuite-codegen/src/mocks.rs

@@ -0,0 +1,48 @@
+use proc_macro2::TokenStream as TokenStream2;
+use quote::quote;
+use std::{
+    fs::read_to_string,
+    path::{Path, MAIN_SEPARATOR},
+};
+
+pub(crate) fn generate(suite_path: &str) -> Result<TokenStream2, Box<dyn std::error::Error>> {
+    let remotes = Path::new(suite_path).join("remotes");
+    if !remotes.exists() || !remotes.is_dir() {
+        return Err(format!(
+            "Path does not exist or is not a directory: {}. Run `git submodule init && git submodule update`",
+            remotes.display()
+        )
+        .into());
+    }
+
+    let mut resources = Vec::new();
+    for entry in walkdir::WalkDir::new(&remotes)
+        .into_iter()
+        .filter_map(Result::ok)
+        .filter(|entry| entry.file_type().is_file())
+    {
+        let path = entry.path().to_path_buf();
+        let relative_path = path.strip_prefix(&remotes).expect("Invalid path");
+        let url_path = relative_path
+            .to_str()
+            .expect("Invalid filename")
+            .replace(MAIN_SEPARATOR, "/");
+        let uri = format!("http://localhost:1234/{url_path}");
+        let contents = read_to_string(path).expect("Failed to read a file");
+        resources.push((uri, contents));
+    }
+
+    resources.sort_by(|(left_uri, _), (right_uri, _)| left_uri.cmp(right_uri));
+
+    let entries = resources.iter().map(|(uri, contents)| {
+        let uri_literal = proc_macro2::Literal::string(uri);
+        let contents_literal = proc_macro2::Literal::string(contents);
+        quote! { (#uri_literal, #contents_literal) }
+    });
+
+    Ok(quote! {
+        static REMOTE_DOCUMENTS: &[(&str, &str)] = &[
+            #(#entries),*
+        ];
+    })
+}

crates/jsonschema-testsuite-codegen/src/remotes.rs

@@ -60,6 +60,7 @@ uuid-simd = { version = "0.8", default-features = false, features = ["std", "det
 [dev-dependencies]
 test-case = "3"
 futures = { version = "0.3", default-features = false, features = ["std"] }
+testsuite = { package = "jsonschema-testsuite", path = "../jsonschema-testsuite" }
 
 [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dev-dependencies]
 wasm-bindgen-test = "0.3.55"
@@ -68,10 +69,8 @@ wasm-bindgen-test = "0.3.55"
 benchmark = { path = "../benchmark/" }
 codspeed-criterion-compat = { version = "4.1", default-features = false }
 criterion = { version = "0.7", default-features = false }
-mockito = "1.5"
 tempfile = "3.13.0"
 test-case = "3"
-testsuite = { package = "jsonschema-testsuite", path = "../jsonschema-testsuite" }
 tokio = { version = "1", features = ["macros", "rt"] }
 
 [lints.clippy]