elastic · nik9000 · May 8, 2025 · Apr 23, 2025 · Apr 24, 2025 · Apr 24, 2025
diff --git a/docs/changelog/127355.yaml b/docs/changelog/127355.yaml
@@ -0,0 +1,5 @@
+pr: 127355
+summary: '`text ==` and `text !=` pushdown'
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv
@@ -3,3 +3,4 @@
 2023-10-23T13:55:01.544Z,Connected to 10.1.0.1
 2023-10-23T13:55:01.545Z,[Connected to 10.1.0.1, More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100]
 2023-10-23T13:55:01.546Z,More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100
+2023-10-23T13:55:01.547Z,[More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100,Second than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100]
diff --git a/...plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java b/...plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java
@@ -20,7 +20,14 @@ public interface TranslationAware {
      * Indicates whether the expression can be translated or not.
      * Usually checks whether the expression arguments are actual fields that exist in Lucene.
      */
-    boolean translatable(LucenePushdownPredicates pushdownPredicates);
+    Translatable translatable(LucenePushdownPredicates pushdownPredicates);
+
+    static TranslationAware.Translatable translatable(Expression exp, LucenePushdownPredicates lucenePushdownPredicates) {
+        if (exp instanceof TranslationAware aware) {
+            return aware.translatable(lucenePushdownPredicates);
+        }
+        return TranslationAware.Translatable.NO;
+    }
 
     /**
      * Translates the implementing expression into a Query.
@@ -42,4 +49,56 @@ interface SingleValueTranslationAware extends TranslationAware {
          */
         Expression singleValueField();
     }
+
+    enum Translatable {
+        NO(FinishedTranslatable.NO),
+        YES(FinishedTranslatable.YES),
+        RECHECK(FinishedTranslatable.RECHECK),
+        YES_BUT_RECHECK_NEGATED(FinishedTranslatable.YES);
+
+        private final FinishedTranslatable finish;
+
+        Translatable(FinishedTranslatable finish) {
+            this.finish = finish;
+        }
+
+        public FinishedTranslatable finish() {
+            return finish;
+        }
+
+        public Translatable negate() {
+            if (this == YES_BUT_RECHECK_NEGATED) {
+                return RECHECK;
+            }
+            return this;
+        }
+
+        public Translatable and(Translatable rhs) {
+            return switch (this) {
+                case NO -> NO;
+                case YES -> switch (rhs) {
+                    case NO -> NO;
+                    case YES -> YES;
+                    case RECHECK -> RECHECK;
+                    case YES_BUT_RECHECK_NEGATED -> YES_BUT_RECHECK_NEGATED;
+                };
+                case RECHECK -> switch (rhs) {
+                    case NO -> NO;
+                    case YES, RECHECK, YES_BUT_RECHECK_NEGATED -> RECHECK;
+                };
+                case YES_BUT_RECHECK_NEGATED -> switch (rhs) {
+                    case NO -> NO;
+                    case YES, YES_BUT_RECHECK_NEGATED -> YES_BUT_RECHECK_NEGATED;
+                    case RECHECK -> RECHECK;
+                };
+            };
+        }
+
+    }
+
+    enum FinishedTranslatable {
+        YES,
+        NO,
+        RECHECK;
+    }
 }
diff --git a/...main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/...main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java
@@ -156,9 +156,9 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
         // In isolation, full text functions are pushable to source. We check if there are no disjunctions in Or conditions
-        return true;
+        return Translatable.YES;
     }
 
     @Override

diff --git a/...l/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java b/...l/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java
@@ -179,8 +179,8 @@ protected NodeInfo<? extends Expression> info() {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
-        return pushdownPredicates.isPushableFieldAttribute(ipField) && Expressions.foldable(matches);
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
+        return pushdownPredicates.isPushableFieldAttribute(ipField) && Expressions.foldable(matches) ? Translatable.YES : Translatable.NO;
     }
 
     @Override

diff --git a/...rg/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java b/...rg/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.lucene.spatial.CoordinateEncoder;
+import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
@@ -273,12 +274,14 @@ protected Geometry fromBytesRef(BytesRef bytesRef) {
     /**
      * Push-down to Lucene is only possible if one field is an indexed spatial field, and the other is a constant spatial or string column.
      */
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
+    public TranslationAware.Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
         // The use of foldable here instead of SpatialEvaluatorFieldKey.isConstant is intentional to match the behavior of the
         // Lucene pushdown code in EsqlTranslationHandler::SpatialRelatesTranslator
         // We could enhance both places to support ReferenceAttributes that refer to constants, but that is a larger change
         return isPushableSpatialAttribute(left(), pushdownPredicates) && right().foldable()
-            || isPushableSpatialAttribute(right(), pushdownPredicates) && left().foldable();
+            || isPushableSpatialAttribute(right(), pushdownPredicates) && left().foldable()
+                ? TranslationAware.Translatable.YES
+                : TranslationAware.Translatable.NO;
 
     }
 

diff --git a/...g/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/...g/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
@@ -181,7 +181,7 @@ protected void processPointDocValuesAndSource(
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
         return super.translatable(pushdownPredicates); // only for the explicit Override, as only this subclass implements TranslationAware
     }
 

diff --git a/...rc/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java b/...rc/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java
@@ -139,8 +139,8 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
-        return pushdownPredicates.isPushableAttribute(str) && suffix.foldable();
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
+        return pushdownPredicates.isPushableAttribute(str) && suffix.foldable() ? Translatable.YES : Translatable.NO;
     }
 
     @Override

diff --git a/...l/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java b/...l/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java
@@ -107,8 +107,8 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
-        return pushdownPredicates.isPushableFieldAttribute(field());
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
+        return pushdownPredicates.isPushableFieldAttribute(field()) ? Translatable.YES : Translatable.NO;
     }
 
     @Override

diff --git a/.../main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java b/.../main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java
@@ -136,8 +136,8 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
-        return pushdownPredicates.isPushableAttribute(str) && prefix.foldable();
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
+        return pushdownPredicates.isPushableAttribute(str) && prefix.foldable() ? Translatable.YES : Translatable.NO;
     }
 
     @Override

diff --git a/...ain/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/...ain/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java
@@ -119,8 +119,8 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
-        return pushdownPredicates.isPushableAttribute(field());
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
+        return pushdownPredicates.isPushableAttribute(field()) ? Translatable.YES : Translatable.NO;
     }
 
     @Override

diff --git a/...ck/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java b/...ck/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java
@@ -215,8 +215,8 @@ public boolean equals(Object obj) {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
-        return pushdownPredicates.isPushableAttribute(value) && lower.foldable() && upper.foldable();
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
+        return pushdownPredicates.isPushableAttribute(value) && lower.foldable() && upper.foldable() ? Translatable.YES : Translatable.NO;
     }
 
     @Override

diff --git a/.../src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java b/.../src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java
@@ -82,11 +82,8 @@ protected boolean isCommutative() {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
-        return left() instanceof TranslationAware leftAware
-            && leftAware.translatable(pushdownPredicates)
-            && right() instanceof TranslationAware rightAware
-            && rightAware.translatable(pushdownPredicates);
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
+        return TranslationAware.translatable(left(), pushdownPredicates).and(TranslationAware.translatable(right(), pushdownPredicates));
     }
 
     @Override

diff --git a/...gin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java b/...gin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java
@@ -100,8 +100,8 @@ static Expression negate(Expression exp) {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
-        return field() instanceof TranslationAware aware && aware.translatable(pushdownPredicates);
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
+        return TranslationAware.translatable(field(), pushdownPredicates).negate();
     }
 
     @Override

diff --git a/...esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java b/...esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java
@@ -75,7 +75,7 @@ public UnaryScalarFunction negate() {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
         return IsNull.isTranslatable(field(), pushdownPredicates);
     }
 

diff --git a/...in/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java b/...in/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java
@@ -72,12 +72,14 @@ public UnaryScalarFunction negate() {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
         return isTranslatable(field(), pushdownPredicates);
     }
 
-    protected static boolean isTranslatable(Expression field, LucenePushdownPredicates pushdownPredicates) {
-        return LucenePushdownPredicates.isPushableTextFieldAttribute(field) || pushdownPredicates.isPushableFieldAttribute(field);
+    protected static Translatable isTranslatable(Expression field, LucenePushdownPredicates pushdownPredicates) {
+        return LucenePushdownPredicates.isPushableTextFieldAttribute(field) || pushdownPredicates.isPushableFieldAttribute(field)
+            ? Translatable.YES
+            : Translatable.NO;
     }
 
     @Override

diff --git a/...in/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/...in/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java
@@ -128,11 +128,11 @@ public Equals(Source source, Expression left, Expression right, ZoneId zoneId) {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
         if (right() instanceof Literal lit) {
-            if (false && left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
+            if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
                 if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, ((BytesRef) lit.value()).utf8ToString())) {
-                    return true;
+                    return Translatable.YES_BUT_RECHECK_NEGATED;
                 }
             }
         }
@@ -142,8 +142,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     @Override
     public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         if (right() instanceof Literal lit) {
-            // Disabled because it cased a bug with !=. Fix incoming shortly.
-            if (false && left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
+            if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
                 String value = ((BytesRef) lit.value()).utf8ToString();
                 if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, value)) {
                     String name = handler.nameOf(fa);

diff --git a/...asticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/...asticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java
@@ -328,16 +328,16 @@ public static String formatIncompatibleTypesMessage(DataType leftType, DataType
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
         if (right().foldable()) {
             if (pushdownPredicates.isPushableFieldAttribute(left())) {
-                return true;
+                return Translatable.YES;
             }
             if (LucenePushdownPredicates.isPushableMetadataAttribute(left())) {
-                return this instanceof Equals || this instanceof NotEquals;
+                return this instanceof Equals || this instanceof NotEquals ? Translatable.YES : Translatable.NO;
             }
         }
-        return false;
+        return Translatable.NO;
     }
 
     /**

diff --git a/...c/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/...c/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java
@@ -461,8 +461,8 @@ static boolean process(BitSet nulls, BitSet mvs, BytesRef lhs, BytesRef[] rhs) {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
-        return pushdownPredicates.isPushableAttribute(value) && Expressions.foldable(list());
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
+        return pushdownPredicates.isPushableAttribute(value) && Expressions.foldable(list()) ? Translatable.YES : Translatable.NO;
     }
 
     @Override

diff --git a/.../elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java b/.../elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java
@@ -97,8 +97,8 @@ public Boolean fold(FoldContext ctx) {
     }
 
     @Override
-    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
-        return pushdownPredicates.isPushableFieldAttribute(left()) && right().foldable();
+    public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
+        return pushdownPredicates.isPushableFieldAttribute(left()) && right().foldable() ? Translatable.YES : Translatable.NO;
     }
 
     @Override

diff --git a/...lasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java b/...lasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.Point;
 import org.elasticsearch.geometry.utils.WellKnownBinary;
+import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
 import org.elasticsearch.xpack.esql.core.expression.Alias;
 import org.elasticsearch.xpack.esql.core.expression.Attribute;
 import org.elasticsearch.xpack.esql.core.expression.AttributeMap;
@@ -42,8 +43,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.capabilities.TranslationAware.translatable;
 import static org.elasticsearch.xpack.esql.expression.predicate.Predicates.splitAnd;
-import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.canPushToSource;
 import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.getAliasReplacedBy;
 
 /**
@@ -106,7 +107,8 @@ private FilterExec rewrite(
             }
             return comparison;
         });
-        if (rewritten.equals(filterExec.condition()) == false && canPushToSource(rewritten, lucenePushdownPredicates)) {
+        if (rewritten.equals(filterExec.condition()) == false
+            && translatable(rewritten, lucenePushdownPredicates).finish() == TranslationAware.FinishedTranslatable.YES) {
             return new FilterExec(filterExec.source(), esQueryExec, rewritten);
         }
         return filterExec;
@@ -156,7 +158,8 @@ private PhysicalPlan rewriteBySplittingFilter(
             // Find and rewrite any binary comparisons that involve a distance function and a literal
             var rewritten = rewriteDistanceFilters(ctx, resExp, distances);
             // If all pushable StDistance functions were found and re-written, we need to re-write the FILTER/EVAL combination
-            if (rewritten.equals(resExp) == false && canPushToSource(rewritten, lucenePushdownPredicates)) {
+            if (rewritten.equals(resExp) == false
+                && translatable(rewritten, lucenePushdownPredicates).finish() == TranslationAware.FinishedTranslatable.YES) {
                 pushable.add(rewritten);
             } else {
                 nonPushable.add(exp);
@@ -183,7 +186,8 @@ private PhysicalPlan rewriteBySplittingFilter(
     private Map<NameId, StDistance> getPushableDistances(List<Alias> aliases, LucenePushdownPredicates lucenePushdownPredicates) {
         Map<NameId, StDistance> distances = new LinkedHashMap<>();
         aliases.forEach(alias -> {
-            if (alias.child() instanceof StDistance distance && distance.translatable(lucenePushdownPredicates)) {
+            if (alias.child() instanceof StDistance distance
+                && distance.translatable(lucenePushdownPredicates).finish() == TranslationAware.FinishedTranslatable.YES) {
                 distances.put(alias.id(), distance);
             } else if (alias.child() instanceof ReferenceAttribute ref && distances.containsKey(ref.id())) {
                 StDistance distance = distances.get(ref.id());