Fix conservative computation of look-behind start offsets

Author: shilangyu

regex-automata/src/nfa/thompson/builder.rs

@@ -342,9 +342,10 @@ pub struct Builder {
     start_pattern: Vec<StateID>,
     /// The starting states for each individual look-behind sub-expression.
     start_look_behind: Vec<StateID>,
-    /// The length (in bytes) of the longest string matched by any
-    /// look-behind sub-expression. If `None`, the length is unbounded.
-    maximum_look_behind_len: Option<usize>,
+    /// Among all look-behinds, this is the furthest offset (in bytes) from
+    /// the beginning of the main regex that a look-behind starts at.
+    /// If `None`, the offset is unbounded.
+    maximum_lookbehind_offset_from_start: Option<usize>,
     /// A map from pattern ID to capture group index to name. (If no name
     /// exists, then a None entry is present. Thus, all capturing groups are
     /// present in this mapping.)
@@ -377,7 +378,10 @@ pub struct Builder {
 impl Builder {
     /// Create a new builder for hand-assembling NFAs.
     pub fn new() -> Builder {
-        Builder { maximum_look_behind_len: Some(0), ..Builder::default() }
+        Builder {
+            maximum_lookbehind_offset_from_start: Some(0),
+            ..Builder::default()
+        }
     }
 
     /// Clear this builder.
@@ -456,7 +460,9 @@ impl Builder {
 
         nfa.set_starts(start_anchored, start_unanchored, &self.start_pattern);
         nfa.set_look_behind_starts(self.start_look_behind.as_slice());
-        nfa.set_maximum_look_behind_len(self.maximum_look_behind_len);
+        nfa.set_maximum_lookbehind_offset_from_start(
+            self.maximum_lookbehind_offset_from_start,
+        );
         nfa.set_captures(&self.captures).map_err(BuildError::captures)?;
         // The idea here is to convert our intermediate states to their final
         // form. The only real complexity here is the process of converting
@@ -715,21 +721,24 @@ impl Builder {
     }
 
     /// Adds the `start_id` to the set of starting states that is used when
-    /// running look-behind expressions. Additionally registers the maximum
-    /// length (in bytes) that the sub-expression of the look-behind can match.
+    /// running look-behind expressions. Additionally registers the furthest
+    /// offset (in bytes) from the start of the main regex this look-behind
+    /// starts.
     pub fn start_look_behind(
         &mut self,
         start_id: StateID,
-        maximum_len: Option<usize>,
+        offset_from_start: Option<usize>,
     ) {
         self.start_look_behind.push(start_id);
 
-        self.maximum_look_behind_len =
-            match (self.maximum_look_behind_len, maximum_len) {
-                (Some(l1), Some(l2)) => Some(usize::max(l1, l2)),
-                // A None subsumes the entire result.
-                (None, _) | (_, None) => None,
-            };
+        self.maximum_lookbehind_offset_from_start = match (
+            self.maximum_lookbehind_offset_from_start,
+            offset_from_start,
+        ) {
+            (Some(l1), Some(l2)) => Some(usize::max(l1, l2)),
+            // A None subsumes the entire result.
+            (None, _) | (_, None) => None,
+        };
     }
 
     /// Add an "empty" NFA state.

regex-automata/src/nfa/thompson/compiler.rs

@@ -713,6 +713,11 @@ pub struct Compiler {
     utf8_suffix: RefCell<Utf8SuffixMap>,
     /// The next index to use for a look-around expression.
     lookaround_index: RefCell<SmallIndex>,
+    /// How far from the beginning (in bytes) of the main regex does the
+    /// current look-behind start at. This is updated when relativizing to
+    /// the current look-behind expression. When `None`, the distance can be
+    /// seen as infinity.
+    current_lookbehind_offset_from_start: RefCell<Option<usize>>,
 }
 
 impl Compiler {
@@ -726,6 +731,7 @@ impl Compiler {
             trie_state: RefCell::new(RangeTrie::new()),
             utf8_suffix: RefCell::new(Utf8SuffixMap::new(1000)),
             lookaround_index: RefCell::new(SmallIndex::ZERO),
+            current_lookbehind_offset_from_start: RefCell::new(Some(0)),
         }
     }
 
@@ -1021,10 +1027,13 @@ impl Compiler {
         }
     }
 
+    /// Compile a look-around expression as its own sub-automaton. Its starting
+    /// state is saved.
     fn c_lookaround(
         &self,
         lookaround: &LookAround,
     ) -> Result<ThompsonRef, BuildError> {
+        // Assign a unique index for this look-around.
         let idx = *self.lookaround_index.borrow();
         *self.lookaround_index.borrow_mut() = SmallIndex::new(idx.one_more())
             .map_err(|e| {
@@ -1036,14 +1045,31 @@ impl Compiler {
         };
         let check = self.add_check_lookaround(idx, pos)?;
 
+        // Compute the furthest offset from the start of the main regex
+        // where this look-around can begin at. We offset the current start
+        // offset by the maximal match length of the subexpression.
+        let maximum_len = lookaround.sub().properties().maximum_len();
+        let relative_start =
+            *self.current_lookbehind_offset_from_start.borrow();
+        let start_offset = match (relative_start, maximum_len) {
+            (Some(s), Some(l)) => Some(s + l),
+            (None, _) | (_, None) => None,
+        };
+
         let unanchored =
             self.c_at_least(&Hir::dot(hir::Dot::AnyByte), false, 0)?;
-        let maximum_len = lookaround.sub().properties().maximum_len();
         self.builder
             .borrow_mut()
-            .start_look_behind(unanchored.start, maximum_len);
+            .start_look_behind(unanchored.start, start_offset);
 
+        // When compiling the subexpression we temporarily change the starting
+        // offset and restore it after. This way, the subexpression is relativized
+        // to our current offset.
+        *self.current_lookbehind_offset_from_start.borrow_mut() = start_offset;
         let sub = self.c(lookaround.sub())?;
+        *self.current_lookbehind_offset_from_start.borrow_mut() =
+            relative_start;
+
         let write = self.add_write_lookaround(idx)?;
         self.patch(unanchored.end, sub.start)?;
         self.patch(sub.end, write)?;

regex-automata/src/nfa/thompson/nfa.rs

@@ -1115,8 +1115,8 @@ impl NFA {
     /// Returns the length (in bytes) of the longest string matched by any
     /// look-behind sub-expression. If `None`, the length is unbounded.
     #[inline]
-    pub fn maximum_look_behind_len(&self) -> Option<usize> {
-        self.0.maximum_look_behind_len
+    pub fn maximum_lookbehind_offset_from_start(&self) -> Option<usize> {
+        self.0.maximum_lookbehind_offset_from_start
     }
 
     // FIXME: The `look_set_prefix_all` computation was not correct, and it
@@ -1285,9 +1285,10 @@ pub(super) struct Inner {
     lookaround_count: usize,
     /// Contains the start states for each of the look-behind subexpressions.
     start_look_behind: Vec<StateID>,
-    /// The length (in bytes) of the longest string matched by any
-    /// look-behind sub-expression. If `None`, the length is unbounded.
-    maximum_look_behind_len: Option<usize>,
+    /// Among all look-behinds, this is the furthest offset (in bytes) from
+    /// the beginning of the main regex that a look-behind starts at.
+    /// If `None`, the offset is unbounded.
+    maximum_lookbehind_offset_from_start: Option<usize>,
     /// Heap memory used indirectly by NFA states and other things (like the
     /// various capturing group representations above). Since each state
     /// might use a different amount of heap, we need to keep track of this
@@ -1444,11 +1445,12 @@ impl Inner {
         self.start_look_behind = look_behind_starts.to_vec();
     }
 
-    pub(super) fn set_maximum_look_behind_len(
+    pub(super) fn set_maximum_lookbehind_offset_from_start(
         &mut self,
-        maximum_look_behind_len: Option<usize>,
+        maximum_lookbehind_offset_from_start: Option<usize>,
     ) {
-        self.maximum_look_behind_len = maximum_look_behind_len;
+        self.maximum_lookbehind_offset_from_start =
+            maximum_lookbehind_offset_from_start;
     }
 
     /// Sets the UTF-8 mode of this NFA.

regex-automata/src/nfa/thompson/pikevm.rs

@@ -1258,7 +1258,8 @@ impl PikeVM {
             Some(config) => config,
         };
 
-        let maximum_look_behind_len = self.nfa.maximum_look_behind_len();
+        let maximum_lookbehind_offset_from_start =
+            self.nfa.maximum_lookbehind_offset_from_start();
 
         let pre =
             if anchored { None } else { self.get_config().get_prefilter() };
@@ -1280,7 +1281,7 @@ impl PikeVM {
             // start from 0.
             let start_position = usize::saturating_sub(
                 input.start(),
-                maximum_look_behind_len.unwrap_or(input.start()),
+                maximum_lookbehind_offset_from_start.unwrap_or(input.start()),
             );
 
             // This initializes the look-behind threads from the `start_position`
@@ -1362,7 +1363,7 @@ impl PikeVM {
                                     at,
                                     usize::saturating_sub(
                                         span.start,
-                                        maximum_look_behind_len
+                                        maximum_lookbehind_offset_from_start
                                             .unwrap_or(span.start),
                                     ),
                                 );