fix: skipping inside arrays on comma-atomic (#757)

Tail skipping was not triggered when the item matching the unitary
transition was an atomic value inside a list. For example,
selecting `$[0]` from a long list of integers would never skip,
massively degrading performance.

Skipping was added to `handle_comma` in the same vein as it was in
`handle_colon` to enable this.

Ref: #751

Author: V0ldek

crates/rsonpath-benchmarks/Cargo.toml

@@ -21,13 +21,13 @@ name = "pathimpl"
 [dependencies]
 clap = { version = "4.5.23", features = ["derive", "wrap_help"] }
 color-eyre = { version = "0.6.3", default-features = false }
-criterion = "0.6.0"
+criterion = "0.7.0"
 eyre = "0.6.12"
 flate2 = "1.0.35"
 hex-literal = "1.0.0"
 indicatif = "0.18.0"
 jni = { version = "0.21.1", features = ["invocation", "default"] }
-jsonpath-rust = "1.0.0"
+jsonpath-rust = "1.0.4"
 lazy_static = "1.5.0"
 serde_json = "1.0.134"
 sha2 = "0.10.8"

crates/rsonpath-benchmarks/src/framework.rs

@@ -329,7 +329,7 @@ impl<I: Implementation> BenchFn for PreparedQuery<I> {
         };
 
         let result = self.implementation.run(q, f).unwrap();
-        criterion::black_box(result);
+        std::hint::black_box(result);
     }
 }
 

crates/rsonpath-benchmarks/src/implementations/jsonpath_rust.rs

@@ -1,20 +1,22 @@
 use crate::framework::implementation::Implementation;
-use jsonpath_rust::{parser::JsonPath, JsonPathValue};
+use jsonpath_rust::{
+    parser::{errors::JsonPathError, model::JpQuery, parse_json_path},
+    query::{js_path_process, QueryRef},
+};
 use serde_json::Value;
 use std::{
     fmt::Display,
     fs,
     io::{self, BufReader},
-    str::FromStr,
 };
 use thiserror::Error;
 
 pub struct JsonpathRust {}
 
-pub struct JsonpathRustResult<'a>(Vec<JsonPathValue<'a, Value>>);
+pub struct JsonpathRustResult<'a>(Vec<QueryRef<'a, Value>>);
 
 impl Implementation for JsonpathRust {
-    type Query = JsonPath;
+    type Query = JpQuery;
 
     type File = Value;
 
@@ -39,11 +41,11 @@ impl Implementation for JsonpathRust {
     }
 
     fn compile_query(&self, query: &str) -> Result<Self::Query, Self::Error> {
-        JsonPath::from_str(query).map_err(JsonpathRustError::JsonPathInstError)
+        parse_json_path(query).map_err(JsonpathRustError::JsonPathInstError)
     }
 
     fn run<'a>(&self, query: &'a Self::Query, file: &'a Self::File) -> Result<Self::Result<'a>, Self::Error> {
-        let results = query.find_slice(file);
+        let results = js_path_process(query, file)?;
 
         Ok(JsonpathRustResult(results))
     }
@@ -52,8 +54,7 @@ impl Implementation for JsonpathRust {
 impl Display for JsonpathRustResult<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         for res in &self.0 {
-            let val = res.clone().to_data();
-            writeln!(f, "{}", val)?;
+            writeln!(f, "{}", res.clone().val())?;
         }
 
         Ok(())
@@ -67,5 +68,5 @@ pub enum JsonpathRustError {
     #[error("error parsing JSON with serde: '{0}'")]
     SerdeError(#[from] serde_json::Error),
     #[error("error parsing JSONPath query: '{0}'")]
-    JsonPathInstError(<JsonPath as TryFrom<&'static str>>::Error),
+    JsonPathInstError(#[from] JsonPathError),
 }

crates/rsonpath-benchmarks/src/main.rs

@@ -23,7 +23,7 @@ fn run<I: Implementation>(imp: I, query_str: &str, path_str: &str) -> Result<()>
 
     let result = imp.run(&query, &file)?;
 
-    println!("{}", result);
+    println!("{result}");
 
     Ok(())
 }

crates/rsonpath-lib/src/classification/structural/nosimd.rs

@@ -121,15 +121,13 @@ where
     #[inline]
     fn reclassify(&mut self, idx: usize) {
         if let Some(block) = self.block.take() {
+            let relative_idx = idx + 1 - self.iter.get_offset();
             let quote_classified_block = block.quote_classified;
-            let relevant_idx = idx + 1;
-            let block_idx = (idx + 1) % N;
-            debug!("relevant_idx is {relevant_idx}.");
-
-            if block_idx != 0 || relevant_idx == self.iter.get_offset() {
+            debug!("relative_idx is {relative_idx}.");
+            if relative_idx < N {
                 let new_block = Block::from_idx(
                     quote_classified_block,
-                    block_idx,
+                    relative_idx,
                     self.are_colons_on,
                     self.are_commas_on,
                 );

crates/rsonpath-lib/src/classification/structural/shared.rs

@@ -58,15 +58,14 @@ macro_rules! structural_classifier {
             #[inline]
             fn reclassify(&mut self, idx: usize) {
                 if let Some(block) = self.block.take() {
+                    let relative_idx = idx + 1 - self.iter.get_offset();
                     let quote_classified_block = block.quote_classified;
-                    let relevant_idx = idx + 1;
-                    let block_idx = (idx + 1) % $size;
-                    debug!("relevant_idx is {relevant_idx}.");
+                    debug!("relative_idx is {relative_idx}.");
 
-                    if block_idx != 0 || relevant_idx == self.iter.get_offset() {
+                    if relative_idx < $size {
                         debug!("need to reclassify.");
 
-                        let mask = <$mask_ty>::MAX << block_idx;
+                        let mask = <$mask_ty>::MAX << relative_idx;
                         // SAFETY: target_feature invariant
                         let mut new_block = unsafe { self.classifier.classify(quote_classified_block) };
                         new_block.structural_mask &= mask;

crates/rsonpath-lib/src/engine.rs

@@ -42,7 +42,7 @@ pub trait Engine {
     /// Find the starting indices of matches on the given [`Input`] and write them to the [`Sink`].
     ///
     /// The result is equivalent to using [`matches`](Engine::matches) and extracting the
-    /// [`Match::span.start_idx`],
+    /// [`Match::span`] and [`MatchSpan::start_idx`],
     /// but in general is much more time and memory efficient.
     ///
     /// # Errors

crates/rsonpath-lib/src/engine/main.rs

@@ -419,7 +419,7 @@ where
     /// This method only handles atomic values after the comma.
     /// Objects and arrays are processed at their respective opening character.
     #[inline(always)]
-    fn handle_comma(&mut self, _classifier: &mut Classifier!(), idx: usize) -> Result<(), EngineError> {
+    fn handle_comma(&mut self, classifier: &mut Classifier!(), idx: usize) -> Result<(), EngineError> {
         debug!("Comma");
 
         self.recorder.record_value_terminator(idx, self.depth)?;
@@ -448,6 +448,20 @@ where
             {
                 debug!("Accepting list item on comma.");
                 self.record_match_detected_at(idx + 1, NodeType::Atomic)?;
+
+                if self.automaton.is_unitary(self.state) {
+                    self.next_event = classifier.next()?;
+                    match self.next_event {
+                        None | Some(Structural::Closing(_, _)) => {
+                            return Ok(());
+                        }
+                        Some(Structural::Comma(idx)) => self.recorder.record_value_terminator(idx, self.depth)?,
+                        Some(Structural::Colon(_) | Structural::Opening(_, _)) => (),
+                    }
+                    debug!("Skipping unique state from {:?}", BracketType::Square);
+                    let stop_at = classifier.skip(BracketType::Square)?;
+                    self.next_event = Some(Structural::Closing(BracketType::Square, stop_at));
+                }
             }
         }
 

crates/rsonpath-test/Cargo.toml

@@ -19,7 +19,7 @@ serde_json = { workspace = true }
 
 [dev-dependencies]
 pretty_assertions = { workspace = true }
-rsonpath-lib = { workspace = true }
+rsonpath-lib = { workspace = true, features = ["simd"] }
 rsonpath-syntax = { workspace = true }
 
 [build-dependencies]