Better default fuzzy search support

Author: ejsmith

docs/guide/query-syntax.md

@@ -40,6 +40,43 @@ Add `~N` to find words within N positions of each other:
 "hello world"~2    // "hello" and "world" within 2 words of each other
 ```
 
+## Fuzzy Search
+
+Add `~` to a term to perform fuzzy matching based on edit distance (Levenshtein distance):
+
+```
+roam~              // Fuzzy search with default edit distance of 2
+roam~1             // Fuzzy search with edit distance of 1
+roam~0             // Exact match (edit distance of 0)
+```
+
+The edit distance specifies the maximum number of character changes (insertions, deletions, substitutions) allowed to match a term. For example, `roam~1` would match:
+
+- `roam` (exact)
+- `foam` (1 substitution)
+- `roams` (1 insertion)
+
+::: tip
+Edit distance must be an integer. The default value is 2 when omitted. Lower values (0-1) are more restrictive and generally faster.
+:::
+
+::: info AST Representation
+The AST preserves whether the fuzzy distance was explicitly specified:
+- `term~` → `FuzzyDistance = TermNode.DefaultFuzzyDistance` (sentinel value -1)
+- `term~2` → `FuzzyDistance = 2` (explicitly specified)
+
+Both resolve to an effective distance of 2, but you can distinguish between default and explicit using `GetEffectiveFuzzyDistance()`.
+:::
+
+### Fuzzy with Field Queries
+
+Fuzzy search works with field queries:
+
+```
+title:hello~2      // Fuzzy search in the "title" field
+user.name:john~1   // Fuzzy search in nested field
+```
+
 ## Field Queries
 
 Specify which field to search:
@@ -267,6 +304,12 @@ level:error AND timestamp:[now-1h TO now] AND (service:api OR service:web) NOT t
 name:john* AND role:(admin OR moderator) AND status:active AND lastLogin:[now-30d TO *]
 ```
 
+### Fuzzy Name Search
+
+```
+firstName:john~1 AND lastName:smith~2
+```
+
 ### Document Search
 
 ```

src/Foundatio.Lucene.Elasticsearch/ElasticsearchQueryBuilderVisitor.cs

@@ -241,25 +241,31 @@ public override Task<QueryNode> VisitAsync(TermNode node, IQueryVisitorContext c
         }
         else if (node.FuzzyDistance.HasValue)
         {
-            // Fuzzy query - use Fuzziness property which accepts a string
+            var fuzziness = node.FuzzyDistance.Value == TermNode.DefaultFuzzyDistance
+                ? "AUTO"
+                : node.FuzzyDistance.Value.ToString();
+
             if (field is not null)
             {
                 query = new FuzzyQuery((Field)field, term)
                 {
-                    Fuzziness = node.FuzzyDistance.Value.ToString()
+                    Fuzziness = fuzziness
                 };
             }
             else if (_context.DefaultFields is { Length: > 0 })
             {
                 query = new MultiMatchQuery(term)
                 {
                     Fields = Fields.FromStrings(_context.DefaultFields),
-                    Fuzziness = new Fuzziness(node.FuzzyDistance.Value.ToString())
+                    Fuzziness = new Fuzziness(fuzziness)
                 };
             }
             else
             {
-                query = new QueryStringQuery($"{term}~{node.FuzzyDistance.Value}");
+                var fuzzyString = node.FuzzyDistance.Value == TermNode.DefaultFuzzyDistance
+                    ? $"{term}~"
+                    : $"{term}~{node.FuzzyDistance.Value}";
+                query = new QueryStringQuery(fuzzyString);
             }
         }
         else if (field is null && _context.DefaultFields is { Length: > 1 })

src/Foundatio.Lucene/Ast/MultiTermNode.cs

@@ -56,6 +56,19 @@ public string CombinedText
 
     /// <summary>
     /// Optional fuzzy distance (for fuzzy queries with ~).
+    /// Use <see cref="TermNode.DefaultFuzzyDistance"/> (-1) to indicate default fuzzy distance was requested.
+    /// Use <see cref="GetEffectiveFuzzyDistance"/> to get the actual fuzzy distance to use.
     /// </summary>
     public int? FuzzyDistance { get; set; }
+
+    /// <summary>
+    /// Gets the effective fuzzy distance, resolving the default sentinel value to the actual default.
+    /// </summary>
+    /// <returns>The fuzzy distance to use, or null if not a fuzzy query.</returns>
+    public int? GetEffectiveFuzzyDistance()
+    {
+        if (FuzzyDistance == TermNode.DefaultFuzzyDistance)
+            return TermNode.DefaultFuzzyDistanceValue;
+        return FuzzyDistance;
+    }
 }

src/Foundatio.Lucene/Ast/TermNode.cs

@@ -5,6 +5,17 @@ namespace Foundatio.Lucene.Ast;
 /// </summary>
 public class TermNode : QueryNode
 {
+    /// <summary>
+    /// Sentinel value indicating the default fuzzy distance should be used.
+    /// When FuzzyDistance equals this value, it means ~ was specified without a number.
+    /// </summary>
+    public const int DefaultFuzzyDistance = -1;
+
+    /// <summary>
+    /// The actual default fuzzy distance value used when DefaultFuzzyDistance is specified.
+    /// This is the Lucene standard default of 2.
+    /// </summary>
+    public const int DefaultFuzzyDistanceValue = 2;
     private ReadOnlyMemory<char> _term;
     private ReadOnlyMemory<char> _unescapedTerm;
 
@@ -51,9 +62,22 @@ public string UnescapedTerm
 
     /// <summary>
     /// Optional fuzzy distance (for fuzzy queries with ~).
+    /// Use <see cref="DefaultFuzzyDistance"/> (-1) to indicate default fuzzy distance was requested.
+    /// Use <see cref="GetEffectiveFuzzyDistance"/> to get the actual fuzzy distance to use.
     /// </summary>
     public int? FuzzyDistance { get; set; }
 
+    /// <summary>
+    /// Gets the effective fuzzy distance, resolving the default sentinel value to the actual default.
+    /// </summary>
+    /// <returns>The fuzzy distance to use, or null if not a fuzzy query.</returns>
+    public int? GetEffectiveFuzzyDistance()
+    {
+        if (FuzzyDistance == DefaultFuzzyDistance)
+            return DefaultFuzzyDistanceValue;
+        return FuzzyDistance;
+    }
+
     /// <summary>
     /// Whether this is a prefix query (ends with *).
     /// </summary>

src/Foundatio.Lucene/LuceneParser.cs

@@ -1054,6 +1054,10 @@ private float ParseBoost()
     /// <summary>
     /// Parses a fuzzy distance value after ~.
     /// </summary>
+    /// <returns>
+    /// The explicit fuzzy distance if specified, or <see cref="TermNode.DefaultFuzzyDistance"/>
+    /// to indicate the default should be used.
+    /// </returns>
     private int ParseFuzzyDistance()
     {
         SkipWhitespace();
@@ -1067,8 +1071,8 @@ private int ParseFuzzyDistance()
             }
         }
 
-        // Default fuzzy distance
-        return 2;
+        // Return sentinel value to indicate default fuzzy distance
+        return TermNode.DefaultFuzzyDistance;
     }
 
     /// <summary>

src/Foundatio.Lucene/QueryStringBuilder.cs

@@ -203,7 +203,7 @@ private void AppendTerm(TermNode node)
         if (node.FuzzyDistance.HasValue)
         {
             _builder.Append('~');
-            if (node.FuzzyDistance.Value != 2)
+            if (node.FuzzyDistance.Value != TermNode.DefaultFuzzyDistanceValue)
             {
                 _builder.Append(node.FuzzyDistance.Value);
             }
@@ -312,7 +312,7 @@ private void AppendMultiTerm(MultiTermNode node)
         if (node.FuzzyDistance.HasValue)
         {
             _builder.Append('~');
-            if (node.FuzzyDistance.Value != 2)
+            if (node.FuzzyDistance.Value != TermNode.DefaultFuzzyDistanceValue)
             {
                 _builder.Append(node.FuzzyDistance.Value);
             }

tests/Foundatio.Lucene.Tests/ParserTests.cs

@@ -190,7 +190,8 @@ public void Parse_FuzzyTerm_ReturnsTermNodeWithFuzzy()
         Assert.IsType<TermNode>(doc.Query);
         var term = (TermNode)doc.Query;
         Assert.Equal("roam", term.Term);
-        Assert.Equal(2, term.FuzzyDistance);
+        Assert.Equal(2, term.FuzzyDistance); // Explicitly specified as 2
+        Assert.Equal(2, term.GetEffectiveFuzzyDistance());
     }
 
     [Fact]
@@ -203,7 +204,27 @@ public void Parse_DefaultFuzzy_ReturnsTermNodeWithDefaultFuzziness()
         Assert.IsType<TermNode>(doc.Query);
         var term = (TermNode)doc.Query;
         Assert.Equal("roam", term.Term);
-        Assert.Equal(2, term.FuzzyDistance); // Default fuzziness is 2
+        Assert.Equal(TermNode.DefaultFuzzyDistance, term.FuzzyDistance); // Sentinel value (-1)
+        Assert.Equal(2, term.GetEffectiveFuzzyDistance()); // Effective value is 2
+    }
+
+    [Fact]
+    public void Parse_DefaultFuzzy_DifferentFromExplicitTwo()
+    {
+        var defaultResult = LuceneQuery.Parse("roam~");
+        var explicitResult = LuceneQuery.Parse("roam~2");
+
+        var defaultTerm = (TermNode)defaultResult.Document.Query!;
+        var explicitTerm = (TermNode)explicitResult.Document.Query!;
+
+        // The raw FuzzyDistance values are different
+        Assert.Equal(TermNode.DefaultFuzzyDistance, defaultTerm.FuzzyDistance);
+        Assert.Equal(2, explicitTerm.FuzzyDistance);
+        Assert.NotEqual(defaultTerm.FuzzyDistance, explicitTerm.FuzzyDistance);
+
+        // But the effective values are the same
+        Assert.Equal(2, defaultTerm.GetEffectiveFuzzyDistance());
+        Assert.Equal(2, explicitTerm.GetEffectiveFuzzyDistance());
     }
 
     [Fact]
@@ -972,6 +993,82 @@ public void Parse_RequiredRangeOperator_ReturnsCorrectStructure()
 
     #region Fuzzy and Proximity Tests
 
+    [Fact]
+    public void Parse_FuzzyWithEditDistance1_ReturnsTermWithFuzzy()
+    {
+        var result = LuceneQuery.Parse("roam~1");
+
+        Assert.True(result.IsSuccess);
+        var doc = result.Document;
+        Assert.IsType<TermNode>(doc.Query);
+        var term = (TermNode)doc.Query;
+        Assert.Equal("roam", term.Term);
+        Assert.Equal(1, term.FuzzyDistance);
+    }
+
+    [Fact]
+    public void Parse_FuzzyWithEditDistance0_ReturnsTermWithFuzzy()
+    {
+        var result = LuceneQuery.Parse("exact~0");
+
+        Assert.True(result.IsSuccess);
+        var doc = result.Document;
+        Assert.IsType<TermNode>(doc.Query);
+        var term = (TermNode)doc.Query;
+        Assert.Equal("exact", term.Term);
+        Assert.Equal(0, term.FuzzyDistance);
+    }
+
+    [Fact]
+    public void Parse_FuzzyFieldQuery_ReturnsFieldNodeWithFuzzyTerm()
+    {
+        var result = LuceneQuery.Parse("title:hello~2");
+
+        Assert.True(result.IsSuccess);
+        var doc = result.Document;
+        Assert.IsType<FieldQueryNode>(doc.Query);
+        var field = (FieldQueryNode)doc.Query;
+        Assert.Equal("title", field.Field);
+        Assert.IsType<TermNode>(field.Query);
+        var term = (TermNode)field.Query;
+        Assert.Equal("hello", term.Term);
+        Assert.Equal(2, term.FuzzyDistance);
+    }
+
+    [Fact]
+    public void Parse_FuzzyWithBoost_ReturnsTermWithBothModifiers()
+    {
+        var result = LuceneQuery.Parse("term~2^3");
+
+        Assert.True(result.IsSuccess);
+        var doc = result.Document;
+        Assert.IsType<TermNode>(doc.Query);
+        var term = (TermNode)doc.Query;
+        Assert.Equal("term", term.Term);
+        Assert.Equal(2, term.FuzzyDistance);
+        Assert.Equal(3.0f, term.Boost);
+    }
+
+    [Fact]
+    public void Parse_FuzzyInBooleanQuery_ReturnsBooleanWithFuzzyTerm()
+    {
+        var result = LuceneQuery.Parse("hello~1 AND world~2");
+
+        Assert.True(result.IsSuccess);
+        var doc = result.Document;
+        Assert.IsType<BooleanQueryNode>(doc.Query);
+        var boolQuery = (BooleanQueryNode)doc.Query;
+        Assert.Equal(2, boolQuery.Clauses.Count);
+
+        var firstTerm = (TermNode)boolQuery.Clauses[0].Query!;
+        Assert.Equal("hello", firstTerm.Term);
+        Assert.Equal(1, firstTerm.FuzzyDistance);
+
+        var secondTerm = (TermNode)boolQuery.Clauses[1].Query!;
+        Assert.Equal("world", secondTerm.Term);
+        Assert.Equal(2, secondTerm.FuzzyDistance);
+    }
+
     [Fact]
     public void Parse_FuzzyDecimal_ReturnsTermWithFuzzy()
     {
@@ -998,6 +1095,19 @@ public void Parse_ProximitySearchWithQuotes_ReturnsPhraseWithSlop()
         Assert.Equal(1, phrase.Slop);
     }
 
+    [Fact]
+    public void Parse_ProximityWithLargeDistance_ReturnsPhraseWithSlop()
+    {
+        var result = LuceneQuery.Parse("\"hello world\"~10");
+
+        Assert.True(result.IsSuccess);
+        var doc = result.Document;
+        Assert.IsType<PhraseNode>(doc.Query);
+        var phrase = (PhraseNode)doc.Query;
+        Assert.Equal("hello world", phrase.Phrase);
+        Assert.Equal(10, phrase.Slop);
+    }
+
     #endregion
 
     #region Wildcard Tests

tests/Foundatio.Lucene.Tests/QueryStringBuilderTests.cs

@@ -103,6 +103,17 @@ public void ToQueryString_FuzzyTerm_ReturnsTilde()
         Assert.Equal("roam~", output);
     }
 
+    [Fact]
+    public void ToQueryString_FuzzyTermNonDefault_ReturnsEditDistance()
+    {
+        var result = LuceneQuery.Parse("roam~1");
+        var builder = new QueryStringBuilder();
+
+        var output = builder.Visit(result.Document);
+
+        Assert.Equal("roam~1", output);
+    }
+
     [Fact]
     public void ToQueryString_BoostedTerm_ReturnsCaret()
     {