From 7550da834bdbcfa4a805fb03731dc93641801c6d Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Mon, 6 Oct 2025 16:56:24 -0400 Subject: [PATCH] automata: fix ID rollover bug in lazy DFA The lazy DFA has a cache of transitions that it may clear from time to time if it gets too full. One cleared, transitions are re-generated. There are two ways the cache gets full. First is if it uses too much memory. Second is if there are so many states that it exceeds `LazyStateID::MAX`. You might expect this to be `2^32`, but it's smaller than that because of some bits reserved for tagging purposes. When the cache is clearer, we have to be rather careful with our state. For example, we are careful to "save" the current state so that we know where to go next after the cache is cleared. And we need to re-map state identifiers when this happens. The abstraction for handling cache clearing is basically non-existent. The current code basically tried to look before it leaps, and if the cache *might* be cleared, then it will save the current state. (Saving the current state is costly, so we don't always want to do it.) But if the cache gets cleared and we think it definitely won't, then we don't save the current state and things get FUBAR. That's what happens in #1083 (I believe) and definitively what happens in https://github.com/BurntSushi/ripgrep/issues/3135. Specifically, the "look before we leap" logic wasn't accounting for the number of states exceeding the maximum. It was only accounting for memory usage. Ideally we could have a better abstraction that makes this harder to get wrong via a single point of truth on whether a cache gets cleared or not, but this is tricky for perf reasons. Fixes #1083 Fixes https://github.com/BurntSushi/ripgrep/issues/3135 --- CHANGELOG.md | 10 ++++++++++ regex-automata/src/hybrid/dfa.rs | 21 +++++++++++++++++++-- regex-automata/src/hybrid/id.rs | 2 +- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58f9d79f3..5e8525909 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +1.11.4 (TBD) +============ +TODO + +Bug fixes: + +* [BUG #1165](https://github.com/rust-lang/regex/issues/1083): +Fixes a panic in the lazy DFA (can only occur for especially large regexes). + + 1.11.3 (2025-09-25) =================== This is a small patch release with an improvement in memory usage in some diff --git a/regex-automata/src/hybrid/dfa.rs b/regex-automata/src/hybrid/dfa.rs index b4c9cf444..22893d7a3 100644 --- a/regex-automata/src/hybrid/dfa.rs +++ b/regex-automata/src/hybrid/dfa.rs @@ -2132,7 +2132,19 @@ impl<'i, 'c> Lazy<'i, 'c> { unit, empty_builder, ); - let save_state = !self.as_ref().state_builder_fits_in_cache(&builder); + // This is subtle, but if we *might* clear the cache, then we should + // try to save the current state so that we can re-map its ID after + // cache clearing. We *might* clear the cache when either the new + // state can't fit in the cache or when the number of transitions has + // reached the maximum. Even if either of these conditions is true, + // the cache might not be cleared if we can reuse an existing state. + // But we don't know that at this point. Moreover, we don't save the + // current state every time because it is costly. + // + // TODO: We should try to find a way to make this less subtle and error + // prone. ---AG + let save_state = !self.as_ref().state_builder_fits_in_cache(&builder) + || self.cache.trans.len() >= LazyStateID::MAX; if save_state { self.save_state(current); } @@ -2761,7 +2773,7 @@ impl<'i, 'c> LazyRef<'i, 'c> { let needed = self.cache.memory_usage() + self.memory_usage_for_one_more_state(state.memory_usage()); trace!( - "lazy DFA cache capacity check: {:?} ?<=? {:?}", + "lazy DFA cache capacity state check: {:?} ?<=? {:?}", needed, self.dfa.cache_capacity ); @@ -2773,6 +2785,11 @@ impl<'i, 'c> LazyRef<'i, 'c> { fn state_builder_fits_in_cache(&self, state: &StateBuilderNFA) -> bool { let needed = self.cache.memory_usage() + self.memory_usage_for_one_more_state(state.as_bytes().len()); + trace!( + "lazy DFA cache capacity state builder check: {:?} ?<=? {:?}", + needed, + self.dfa.cache_capacity + ); needed <= self.dfa.cache_capacity } diff --git a/regex-automata/src/hybrid/id.rs b/regex-automata/src/hybrid/id.rs index 43d5b5ba0..65d8528e7 100644 --- a/regex-automata/src/hybrid/id.rs +++ b/regex-automata/src/hybrid/id.rs @@ -180,7 +180,7 @@ impl LazyStateID { const MASK_QUIT: usize = 1 << (LazyStateID::MAX_BIT - 2); const MASK_START: usize = 1 << (LazyStateID::MAX_BIT - 3); const MASK_MATCH: usize = 1 << (LazyStateID::MAX_BIT - 4); - const MAX: usize = LazyStateID::MASK_MATCH - 1; + pub(crate) const MAX: usize = LazyStateID::MASK_MATCH - 1; /// Create a new lazy state ID. ///