ESQL: don't use a Literal for constant_keyword fields when used inside full-text functions (elastic#145632) (elastic#145795)

* ESQL: don't use a Literal for constant_keyword fields when used inside full-text functions (elastic#145632)

(cherry picked from commit 5e46b50)

* Update for 9.3 specifically

* Move the csv file to the right location

* Remove all-types index and all its uses since it's interfering
with stuff that not backported from "main"

Author: astefan

docs/changelog/145632.yaml

@@ -0,0 +1,7 @@
+area: ES|QL
+issues:
+ - 145570
+pr: 145632
+summary: Don't use a Literal for `constant_keyword` fields when used inside full-text
+  functions
+type: bug

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceFieldWithConstantOrNull.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField;
+import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction;
 import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext;
 import org.elasticsearch.xpack.esql.optimizer.rules.RuleUtils;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
@@ -113,9 +114,18 @@ private LogicalPlan replaceWithNullOrConstant(
             || plan instanceof OrderBy
             || plan instanceof RegexExtract
             || plan instanceof TopN) {
+
+            // full-text functions need actual index fields to construct Lucene queries
+            // Note: AttributeSet uses semanticEquals for lookups, so any FieldAttribute that refers to the same underlying field as
+            // one used inside a FullTextFunction is protected here, even if it also appears outside the function (e.g. in a plain equality
+            // check). This slight loss of constant-folding opportunity is intentional to keep the logic simple.
+            var fullTextFieldArgsBuilder = AttributeSet.builder();
+            plan.forEachExpression(FullTextFunction.class, ftf -> ftf.forEachDown(FieldAttribute.class, fullTextFieldArgsBuilder::add));
+            AttributeSet fullTextFieldArgs = fullTextFieldArgsBuilder.build();
+
             return plan.transformExpressionsOnlyUp(FieldAttribute.class, f -> {
                 if (attrToConstant.containsKey(f)) {// handle constant values field and use the value itself instead
-                    return attrToConstant.get(f);
+                    return fullTextFieldArgs.contains(f) ? f : attrToConstant.get(f);
                 } else {// handle missing fields and replace them with null
                     return shouldBeRetained.test(f) ? f : Literal.of(f, null);
                 }

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java

@@ -2126,6 +2126,136 @@ public void testKnnOnMissingField() {
         assertThat(Expressions.name(fullTextFunction.field()), equalTo("text"));
     }
 
+    public void testFullTextFunctionOnConstantField() {
+        String functionName = randomFrom("match", "match_phrase");
+        var plan = plan(String.format(Locale.ROOT, """
+            from test
+            | where %s(first_name, "John")
+            """, functionName));
+
+        var searchStats = new EsqlTestUtils.TestSearchStats() {
+            @Override
+            public String constantValue(FieldAttribute.FieldName name) {
+                if (name.string().equals("first_name")) {
+                    return "John";
+                }
+                return null;
+            }
+        };
+        var localPlan = localPlan(plan, searchStats);
+
+        var limit = as(localPlan, Limit.class);
+        var filter = as(limit.child(), Filter.class);
+        var fullTextFunction = as(filter.condition(), SingleFieldFullTextFunction.class);
+        // The field must remain a FieldAttribute — not replaced with a constant Literal
+        assertThat(fullTextFunction.field(), instanceOf(FieldAttribute.class));
+        assertThat(Expressions.name(fullTextFunction.field()), equalTo("first_name"));
+    }
+
+    public void testConstantFieldReplacedOutsideFullTextFunction() {
+        var plan = plan("""
+            from test
+            | where match_phrase(first_name, "John") and last_name == "Doe"
+            """);
+
+        var searchStats = new EsqlTestUtils.TestSearchStats() {
+            @Override
+            public String constantValue(FieldAttribute.FieldName name) {
+                if (name.string().equals("last_name")) {
+                    return "Doe";
+                }
+                return null;
+            }
+        };
+        var localPlan = localPlan(plan, searchStats);
+
+        var limit = as(localPlan, Limit.class);
+        var filter = as(limit.child(), Filter.class);
+        // last_name is a constant field with value "Doe", so last_name == "Doe" folds to true
+        // and is eliminated from the AND, leaving only the full-text function in the condition.
+        // If constant substitution had NOT happened, the condition would still be an And.
+        var matchPhrase = as(filter.condition(), SingleFieldFullTextFunction.class);
+        assertThat(matchPhrase.field(), instanceOf(FieldAttribute.class));
+        assertThat(Expressions.name(matchPhrase.field()), equalTo("first_name"));
+    }
+
+    public void testMatchOperatorOnConstantField() {
+        var plan = plan("""
+            from test
+            | where first_name : "John"
+            """);
+
+        var searchStats = new EsqlTestUtils.TestSearchStats() {
+            @Override
+            public String constantValue(FieldAttribute.FieldName name) {
+                if (name.string().equals("first_name")) {
+                    return "John";
+                }
+                return null;
+            }
+        };
+        var localPlan = localPlan(plan, searchStats);
+
+        var limit = as(localPlan, Limit.class);
+        var filter = as(limit.child(), Filter.class);
+        var matchOp = as(filter.condition(), SingleFieldFullTextFunction.class);
+        assertThat(matchOp.field(), instanceOf(FieldAttribute.class));
+        assertThat(Expressions.name(matchOp.field()), equalTo("first_name"));
+    }
+
+    public void testSameConstantFieldProtectedInFullTextAndOutside() {
+        var plan = plan("""
+            from test
+            | where match(first_name, "John") and first_name == "John"
+            """);
+
+        var searchStats = new EsqlTestUtils.TestSearchStats() {
+            @Override
+            public String constantValue(FieldAttribute.FieldName name) {
+                if (name.string().equals("first_name")) {
+                    return "John";
+                }
+                return null;
+            }
+        };
+        var localPlan = localPlan(plan, searchStats);
+
+        var limit = as(localPlan, Limit.class);
+        var filter = as(limit.child(), Filter.class);
+        // first_name is protected everywhere because it appears in a full-text function;
+        // semantic equality in AttributeSet means all references to the same field are kept.
+        filter.condition().forEachDown(FieldAttribute.class, fa -> assertThat(Expressions.name(fa), equalTo("first_name")));
+        filter.condition().forEachDown(SingleFieldFullTextFunction.class, ftf -> assertThat(ftf.field(), instanceOf(FieldAttribute.class)));
+    }
+
+    public void testMultipleFullTextFunctionsOnConstantFields() {
+        var plan = plan("""
+            from test
+            | where match(first_name, "John") and match_phrase(last_name, "Doe")
+            """);
+
+        var searchStats = new EsqlTestUtils.TestSearchStats() {
+            @Override
+            public String constantValue(FieldAttribute.FieldName name) {
+                if (name.string().equals("first_name") || name.string().equals("last_name")) {
+                    return "constant";
+                }
+                return null;
+            }
+        };
+        var localPlan = localPlan(plan, searchStats);
+
+        var limit = as(localPlan, Limit.class);
+        var filter = as(limit.child(), Filter.class);
+        var and = as(filter.condition(), And.class);
+        var match = as(and.left(), SingleFieldFullTextFunction.class);
+        assertThat(match.field(), instanceOf(FieldAttribute.class));
+        assertThat(Expressions.name(match.field()), equalTo("first_name"));
+        var matchPhrase = as(and.right(), SingleFieldFullTextFunction.class);
+        assertThat(matchPhrase.field(), instanceOf(FieldAttribute.class));
+        assertThat(Expressions.name(matchPhrase.field()), equalTo("last_name"));
+    }
+
     private static PhysicalPlan physicalPlan(LogicalPlan logicalPlan, Analyzer analyzer) {
         var mapper = new Mapper();
         return mapper.map(new Versioned<>(logicalPlan, analyzer.context().minimumVersion()));