rewrite jsondocck from JSONPath to jq

i tried to keep the original code structure, but it's an almost entire rewrite. thankfully, there are more deletions than insertions, i expect jq to be flexible enough to supersede almost the entire previous set of directives. no tests yet, but it seems to be working. jaq's guts are a bit arcane to me, but it seems to be only related to its setup, a resulted API turned out to be pretty straightforward

Author: fluiderson

Cargo.lock

@@ -246,6 +246,12 @@ version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
+[[package]]
+name = "base64"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+
 [[package]]
 name = "basic-toml"
 version = "0.1.10"
@@ -1107,6 +1113,12 @@ version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8975ffdaa0ef3661bfe02dbdcc06c9f829dfafe6a3c474de366a8d5e44276921"
 
+[[package]]
+name = "dyn-clone"
+version = "1.0.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005"
+
 [[package]]
 name = "either"
 version = "1.15.0"
@@ -1306,6 +1318,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "fs-err"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88d7be93788013f265201256d58f04936a8079ad5dc898743aa20525f503b683"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "fs_extra"
 version = "1.3.0"
@@ -1998,6 +2019,46 @@ version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
 
+[[package]]
+name = "jaq-core"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77526a72eb79412c29fd141767a6549bbfcb1cb40e00556fe16532d5e878e098"
+dependencies = [
+ "dyn-clone",
+ "once_cell",
+ "typed-arena",
+]
+
+[[package]]
+name = "jaq-json"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01dbdbd07b076e8403abac68ce7744d93e2ecd953bbc44bf77bf00e1e81172bc"
+dependencies = [
+ "foldhash",
+ "indexmap",
+ "jaq-core",
+ "jaq-std",
+ "serde_json",
+]
+
+[[package]]
+name = "jaq-std"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c264fe397c981705976c71f1bfe020382b9eda52ae950e57fe885e147bdd67d"
+dependencies = [
+ "aho-corasick",
+ "base64 0.22.1",
+ "chrono",
+ "jaq-core",
+ "libm",
+ "log",
+ "regex-lite",
+ "urlencoding",
+]
+
 [[package]]
 name = "jiff"
 version = "0.2.15"
@@ -2046,12 +2107,15 @@ dependencies = [
 name = "jsondocck"
 version = "0.0.0"
 dependencies = [
- "fs-err",
+ "foldhash",
+ "fs-err 3.1.1",
  "getopts",
- "jsonpath-rust",
+ "indexmap",
+ "jaq-core",
+ "jaq-json",
+ "jaq-std",
  "regex",
  "serde_json",
- "shlex",
 ]
 
 [[package]]
@@ -2060,26 +2124,13 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "clap",
- "fs-err",
+ "fs-err 2.11.0",
  "rustc-hash 2.1.1",
  "rustdoc-json-types",
  "serde",
  "serde_json",
 ]
 
-[[package]]
-name = "jsonpath-rust"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d057f8fd19e20c3f14d3663983397155739b6bc1148dc5cd4c4a1a5b3130eb0"
-dependencies = [
- "pest",
- "pest_derive",
- "regex",
- "serde_json",
- "thiserror 2.0.12",
-]
-
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -2780,50 +2831,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "pest"
-version = "2.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323"
-dependencies = [
- "memchr",
- "thiserror 2.0.12",
- "ucd-trie",
-]
-
-[[package]]
-name = "pest_derive"
-version = "2.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb056d9e8ea77922845ec74a1c4e8fb17e7c218cc4fc11a15c5d25e189aa40bc"
-dependencies = [
- "pest",
- "pest_generator",
-]
-
-[[package]]
-name = "pest_generator"
-version = "2.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87e404e638f781eb3202dc82db6760c8ae8a1eeef7fb3fa8264b2ef280504966"
-dependencies = [
- "pest",
- "pest_meta",
- "proc-macro2",
- "quote",
- "syn 2.0.104",
-]
-
-[[package]]
-name = "pest_meta"
-version = "2.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edd1101f170f5903fde0914f899bb503d9ff5271d7ba76bbb70bea63690cc0d5"
-dependencies = [
- "pest",
- "sha2",
-]
-
 [[package]]
 name = "phf"
 version = "0.11.3"
@@ -4753,7 +4760,7 @@ version = "0.0.0"
 dependencies = [
  "arrayvec",
  "askama",
- "base64",
+ "base64 0.21.7",
  "expect-test",
  "indexmap",
  "itertools",
@@ -5682,6 +5689,12 @@ dependencies = [
  "rustc-hash 2.1.1",
 ]
 
+[[package]]
+name = "typed-arena"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
+
 [[package]]
 name = "typenum"
 version = "1.18.0"
@@ -5697,12 +5710,6 @@ dependencies = [
  "regex-lite",
 ]
 
-[[package]]
-name = "ucd-trie"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
-
 [[package]]
 name = "ui_test"
 version = "0.29.2"
@@ -5874,6 +5881,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "urlencoding"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
+
 [[package]]
 name = "utf-8"
 version = "0.7.6"

src/tools/compiletest/src/directives.rs

@@ -790,8 +790,7 @@ const KNOWN_HTMLDOCCK_DIRECTIVE_NAMES: &[&str] = &[
     "!snapshot",
 ];
 
-const KNOWN_JSONDOCCK_DIRECTIVE_NAMES: &[&str] =
-    &["count", "!count", "has", "!has", "is", "!is", "ismany", "!ismany", "set", "!set"];
+const KNOWN_JSONDOCCK_DIRECTIVE_NAMES: &[&str] = &["jq", "arg", "eq", "ne"];
 
 /// The (partly) broken-down contents of a line containing a test directive,
 /// which [`iter_directives`] passes to its callback function.

src/tools/jsondocck/Cargo.toml

@@ -3,9 +3,16 @@ name = "jsondocck"
 edition = "2024"
 
 [dependencies]
-jsonpath-rust = "1.0.0"
 getopts = "0.2"
-regex = "1.4"
-shlex = "1.0"
-serde_json = "1.0"
-fs-err = "2.5.0"
+regex = "1"
+serde_json = "1"
+fs-err = "3"
+jaq-core = "2"
+# FIXME: Depends on `jaq-std` with default features (a ton of unused by `jsondocck` stuff). Consider
+# disabling them when it'll be possible.
+jaq-json = { version = "1", default-features = false, features = [
+    "serde_json",
+] }
+jaq-std = { version = "2", default-features = false }
+indexmap = "2"
+foldhash = "0.1"

src/tools/jsondocck/src/cache.rs

@@ -1,35 +1,115 @@
-use std::collections::HashMap;
-use std::path::Path;
+use std::borrow::Cow;
+use std::iter::{self, Empty};
+use std::path::{Path, PathBuf};
 
+use foldhash::fast::RandomState;
 use fs_err as fs;
+use indexmap::IndexMap as InnerIndexMap;
+use jaq_core::load::{Arena, File, Loader};
+use jaq_core::{Compiler, Ctx, Filter as InnerFilter, Native, RcIter};
+use jaq_json::Val;
 use serde_json::Value;
 
 use crate::config::Config;
 
+type IndexMap<K, V> = InnerIndexMap<K, V, RandomState>;
+
 #[derive(Debug)]
 pub struct Cache {
-    value: Value,
-    pub variables: HashMap<String, Value>,
+    value: Val,
+    global_vars: IndexMap<String, Val>,
 }
 
 impl Cache {
-    /// Create a new cache, used to read files only once and otherwise store their contents.
-    pub fn new(config: &Config) -> Cache {
-        let root = Path::new(&config.doc_dir);
+    /// Create a new cache, used to read a documentation JSON file only once and otherwise store its
+    /// content.
+    pub fn new(Config { doc_dir, template }: &Config) -> Self {
+        let root = Path::new(doc_dir);
         // `filename` needs to replace `-` with `_` to be sure the JSON path will always be valid.
-        let filename =
-            Path::new(&config.template).file_stem().unwrap().to_str().unwrap().replace('-', "_");
-        let file_path = root.join(&Path::with_extension(Path::new(&filename), "json"));
-        let content = fs::read_to_string(&file_path).expect("failed to read JSON file");
+        let mut filename: PathBuf =
+            Path::new(template).file_stem().unwrap().to_str().unwrap().replace('-', "_").into();
+
+        filename.set_extension("json");
+
+        let content = fs::read(root.join(filename)).expect("failed to read a JSON file");
 
         Cache {
-            value: serde_json::from_str::<Value>(&content).expect("failed to convert from JSON"),
-            variables: HashMap::from([("FILE".to_owned(), config.template.clone().into())]),
+            value: serde_json::from_slice::<Value>(&content)
+                .expect("failed to convert from JSON")
+                .into(),
+            // FIXME: Replace this with empty `HashMap` and use `input_filename` instead of `$FILE`
+            // in tests once https://github.com/01mf02/jaq/issues/144 is fixed.
+            global_vars: [("$FILE".into(), template.to_owned().into())].into_iter().collect(),
         }
     }
 
-    // FIXME: Make this failible, so jsonpath syntax error has line number.
-    pub fn select(&self, path: &str) -> Vec<&Value> {
-        jsonpath_rust::query::js_path_vals(path, &self.value).unwrap()
+    pub fn arg(&mut self, name: &str, value: Val) {
+        self.global_vars.insert(format!("${name}"), value);
+    }
+
+    pub fn filter(&self, code: &str) -> Result<Filter, Cow<'static, str>> {
+        let arena = Arena::default();
+        let modules = Loader::new(
+            // `tonumber` depends on `fromjson` that requires adding the extra "hifijson"
+            // dependency.
+            jaq_std::defs().chain(jaq_json::defs().filter(|def| !matches!(def.name, "tonumber"))),
+        )
+        .load(&arena, File { code, path: () })
+        .map_err(|e| format!("failed to parse the given filter: {e:?}"))?;
+
+        Ok(Filter {
+            inner: Compiler::default()
+                .with_funs(jaq_std::funs().chain(jaq_json::base_funs()))
+                .with_global_vars(self.global_vars.keys().map(String::as_ref))
+                .compile(modules)
+                .map_err(|e| format!("failed to compile the given filter: {e:?}"))?,
+            inputs: RcIter::new(iter::empty()),
+        })
+    }
+}
+
+pub struct Filter {
+    inner: InnerFilter<Native<Val>>,
+    inputs: RcIter<Empty<Result<Val, String>>>,
+}
+
+impl Filter {
+    pub fn run(&self, cache: &Cache) -> Values<impl Iterator<Item = Result<Val, String>> + '_> {
+        Values {
+            inner: self
+                .inner
+                .run((
+                    Ctx::new(cache.global_vars.values().cloned(), &self.inputs),
+                    cache.value.clone(),
+                ))
+                .map(|r| r.map_err(|e| format!("failed to execute the given filter: {e}"))),
+            counter: 0,
+        }
+    }
+}
+
+pub struct Values<I> {
+    inner: I,
+    counter: usize,
+}
+
+impl<I: Iterator<Item = Result<Val, String>>> Values<I> {
+    pub fn is_empty(&mut self) -> Result<(), String> {
+        if self.inner.next().is_none() {
+            Ok(())
+        } else {
+            Err(format!(
+                "expected {expected} value(s), received more than {expected}",
+                expected = self.counter
+            ))
+        }
+    }
+
+    pub fn next(&mut self) -> Result<Val, String> {
+        self.counter += 1;
+
+        self.inner
+            .next()
+            .ok_or_else(|| format!("{} value was expected, received nothing", self.counter))?
     }
 }