Merge pull request #112 from coregx/feature/rust-inspired-optimizations

perf: Rust-inspired optimizations (anti-quadratic, DFA unrolling, 1-bit visited)

Author: kolkov

CHANGELOG.md

@@ -12,6 +12,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - ARM NEON SIMD support (waiting for Go 1.26 native SIMD)
 - SIMD prefilter for CompositeSequenceDFA (#83)
 
+## [0.12.0] - 2026-02-06
+
+### Performance
+- **Anti-quadratic guard for reverse searches** — Track `minStart` position in
+  ReverseSuffix, ReverseInner, and ReverseSuffixSet searchers to prevent O(n²)
+  degradation on high false-positive suffix workloads. Falls back to PikeVM
+  when quadratic behavior detected. Inspired by Rust regex `meta/limited.rs`.
+- **Lazy DFA 4x loop unrolling** — Process 4 state transitions per inner loop
+  iteration in forward and reverse DFA search. Check special states only between
+  batches. Direct field access for minimal per-transition overhead.
+  Expected 15-40% throughput on DFA-heavy patterns (alpha_digit, word_digit).
+- **Prefilter `IsFast()` gate** — Skip reverse search optimizations when a fast
+  SIMD-backed prefix prefilter already exists. Heuristic: Memchr/Memmem always
+  fast, Teddy fast when `minLen >= 3`. Inspired by Rust regex `is_fast()`.
+- **DFA cache clear & continue** — On cache overflow, clear and fall back to
+  PikeVM for current search instead of permanently disabling DFA. Configurable
+  `MaxCacheClears` limit (default 5). Inspired by Rust regex `try_clear_cache`.
+
+### Fixed
+- **OnePass DFA capture limit** — Tighten from 17 to 16 capture groups.
+  `uint32` slot mask (32 bits) can only track 16 groups (slots 0..31).
+  Group 17 silently dropped end position due to `slotIdx < 32` guard.
+
 ---
 
 ## [0.11.9] - 2026-02-02

dfa/lazy/cache.go

@@ -9,15 +9,16 @@ import (
 // Cache provides thread-safe storage for DFA states with bounded memory.
 //
 // The cache maps StateKey (NFA state set hash) → DFA State.
-// When the cache reaches maxStates, it stops accepting new entries and
-// the DFA must fall back to NFA for uncached transitions.
+// When the cache reaches maxStates, it can be cleared and rebuilt
+// (up to a configured limit) before falling back to NFA.
 //
 // Thread safety: All methods are safe for concurrent access via RWMutex.
 //
 // Memory management:
-//   - States are never evicted (LRU would require additional overhead)
-//   - When cache is full, new states trigger NFA fallback
-//   - This is simple and efficient for most patterns
+//   - States are never evicted individually (no LRU overhead)
+//   - When cache is full, it is cleared entirely and search continues
+//   - After too many clears, falls back to NFA
+//   - Clearing keeps allocated memory to avoid re-allocation
 type Cache struct {
 	// mu protects all fields below
 	// RWMutex allows concurrent reads (common case during search)
@@ -33,6 +34,12 @@ type Cache struct {
 	// Start at 1 (0 is reserved for StartState)
 	nextID StateID
 
+	// clearCount tracks how many times the cache has been cleared during
+	// the current search. This is used to detect pathological cache thrashing
+	// and trigger NFA fallback when clears exceed the configured limit.
+	// Inspired by Rust regex-automata's hybrid DFA cache clearing strategy.
+	clearCount int
+
 	// Statistics for cache performance tuning
 	hits   uint64 // Number of cache hits
 	misses uint64 // Number of cache misses
@@ -175,14 +182,58 @@ func (c *Cache) ResetStats() {
 }
 
 // Clear removes all states from the cache and resets statistics.
-// This is primarily for testing.
+// This also resets the clear counter. Primarily for testing.
 func (c *Cache) Clear() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
 	// Clear map (GC will reclaim memory)
 	c.states = make(map[StateKey]*State, c.maxStates)
 	c.nextID = StartState + 1
+	c.clearCount = 0
 	c.hits = 0
 	c.misses = 0
 }
+
+// ClearKeepMemory clears all states from the cache but keeps the allocated
+// map memory for reuse and increments the clear counter. This is used during
+// search when the cache is full: instead of falling back to NFA permanently,
+// we clear the cache and continue DFA search, rebuilding states on demand.
+//
+// Unlike Clear(), this method:
+//   - Increments clearCount (tracks clears during a search)
+//   - Does NOT reset hit/miss statistics (they accumulate across clears)
+//   - Reuses map memory via Go's map clearing optimization
+//
+// After calling this, all previously returned *State pointers are stale
+// and must not be used. The caller must re-obtain the start state.
+//
+// Inspired by Rust regex-automata's cache clearing strategy (hybrid/dfa.rs).
+func (c *Cache) ClearKeepMemory() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// Clear the map using Go's optimized clear-by-range idiom.
+	// This reuses the map's internal memory (buckets) instead of reallocating.
+	for k := range c.states {
+		delete(c.states, k)
+	}
+	c.nextID = StartState + 1
+	c.clearCount++
+}
+
+// ClearCount returns how many times the cache has been cleared.
+// Used to check against the MaxCacheClears limit.
+func (c *Cache) ClearCount() int {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.clearCount
+}
+
+// ResetClearCount resets the clear counter to zero.
+// Called at the start of each new search to give the DFA a fresh budget.
+func (c *Cache) ResetClearCount() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.clearCount = 0
+}

dfa/lazy/config.go

@@ -6,7 +6,9 @@ package lazy
 // performance. Larger caches provide better hit rates but consume more memory.
 type Config struct {
 	// MaxStates is the maximum number of DFA states to cache.
-	// When this limit is reached, the DFA falls back to NFA execution.
+	// When this limit is reached, the DFA clears the cache and continues
+	// DFA search (up to MaxCacheClears times per search), then falls back
+	// to NFA execution if the clear limit is exceeded.
 	//
 	// Default: 10,000 states (~1MB with 256-byte transition tables)
 	// Memory usage: ~100-200 bytes per state (depending on transitions)
@@ -17,6 +19,25 @@ type Config struct {
 	//   - Memory-constrained: 1,000 states (~100KB)
 	MaxStates uint32
 
+	// MaxCacheClears is the maximum number of times the DFA cache can be
+	// cleared and rebuilt during a single search before falling back to NFA.
+	//
+	// When the cache fills up during determinization, instead of immediately
+	// falling back to the NFA (PikeVM), the DFA clears the cache, re-creates
+	// the start state, and continues searching from the current position.
+	// This is much faster than NFA fallback for large inputs with complex
+	// patterns that generate many DFA states.
+	//
+	// After MaxCacheClears clears, the DFA gives up and falls back to NFA
+	// permanently. This prevents pathological cases where the cache thrashes
+	// endlessly (clearing and refilling every few bytes).
+	//
+	// Default: 5
+	// Set to 0 to disable cache clearing (always fall back to NFA on full cache).
+	//
+	// Inspired by Rust regex-automata's hybrid DFA cache clearing strategy.
+	MaxCacheClears int
+
 	// CacheHitThreshold is the minimum cache hit rate (0.0-1.0) to continue
 	// using DFA. If hit rate falls below this, fall back to NFA.
 	//
@@ -64,6 +85,7 @@ type Config struct {
 func DefaultConfig() Config {
 	return Config{
 		MaxStates:            10_000,
+		MaxCacheClears:       5,   // Allow 5 cache clears before NFA fallback
 		CacheHitThreshold:    0.0, // Disabled by default
 		UsePrefilter:         true,
 		MinPrefilterLen:      3,
@@ -81,6 +103,13 @@ func (c *Config) Validate() error {
 		}
 	}
 
+	if c.MaxCacheClears < 0 {
+		return &DFAError{
+			Kind:    InvalidConfig,
+			Message: "MaxCacheClears must be >= 0",
+		}
+	}
+
 	if c.CacheHitThreshold < 0.0 || c.CacheHitThreshold > 1.0 {
 		return &DFAError{
 			Kind:    InvalidConfig,
@@ -111,6 +140,13 @@ func (c Config) WithMaxStates(maxStates uint32) Config {
 	return c
 }
 
+// WithMaxCacheClears returns a new config with the specified max cache clears.
+// Set to 0 to disable cache clearing (always fall back to NFA on full cache).
+func (c Config) WithMaxCacheClears(maxClears int) Config {
+	c.MaxCacheClears = maxClears
+	return c
+}
+
 // WithCacheHitThreshold returns a new config with the specified cache hit threshold
 func (c Config) WithCacheHitThreshold(threshold float64) Config {
 	c.CacheHitThreshold = threshold

dfa/lazy/error.go

@@ -4,7 +4,8 @@ import "fmt"
 
 // Error types for Lazy DFA operations
 
-// ErrCacheFull indicates that the DFA state cache has exceeded its maximum size.
+// ErrCacheFull indicates that the DFA state cache has exceeded its maximum size
+// AND the maximum number of cache clears has been reached.
 // When this occurs, the DFA falls back to NFA execution (PikeVM) for the
 // remainder of the search.
 //
@@ -15,6 +16,17 @@ var ErrCacheFull = &DFAError{
 	Message: "DFA state cache is full",
 }
 
+// errCacheCleared is an internal sentinel error returned by determinize()
+// when the cache was cleared and rebuilt. The search loop must re-obtain
+// the current state from the start state at the current position and continue.
+//
+// This is NOT a real error - it signals that the search should restart
+// DFA processing from the current position with a fresh cache.
+var errCacheCleared = &DFAError{
+	Kind:    CacheCleared,
+	Message: "DFA cache was cleared and rebuilt",
+}
+
 // ErrStateLimitExceeded indicates that the DFA has reached the maximum number
 // of allowed states during determinization.
 //
@@ -36,8 +48,14 @@ type ErrorKind uint8
 
 const (
 	// CacheFull indicates the state cache reached its size limit
+	// and cannot be cleared further (max clears exceeded)
 	CacheFull ErrorKind = iota
 
+	// CacheCleared indicates the cache was cleared and rebuilt.
+	// This is an internal signal, not a real error. The search loop
+	// should re-obtain the current state and continue from the current position.
+	CacheCleared
+
 	// StateLimitExceeded indicates too many states were created
 	StateLimitExceeded
 
@@ -54,6 +72,8 @@ func (k ErrorKind) String() string {
 	switch k {
 	case CacheFull:
 		return "CacheFull"
+	case CacheCleared:
+		return "CacheCleared"
 	case StateLimitExceeded:
 		return "StateLimitExceeded"
 	case InvalidConfig: