Tests

Author: nik9000

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -829,6 +829,9 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
             if (blContext.blockLoaderFunctionConfig() != null) {
                 throw new UnsupportedOperationException("function fusing only supported for doc values");
             }
+            if (isStored()) {
+                return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name());
+            }
 
             // Multi fields don't have fallback synthetic source.
             if (isSyntheticSourceEnabled() && blContext.parentField(name()) == null) {

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java

@@ -252,6 +252,13 @@ public EsField getExactField() {
         return this;
     }
 
+    /**
+     * Can this field be pushed <strong>if</strong> it is indexed?
+     */
+    public boolean pushable() {
+        return true;
+    }
+
     /**
      * Returns and {@link Exact} object with all the necessary info about the field:
      * <ul>

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java

@@ -70,4 +70,10 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(super.hashCode(), functionConfig);
     }
+
+    @Override
+    public boolean pushable() {
+        // These fields are *never* pushable to the lucene index.
+        return false;
+    }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
 import org.elasticsearch.xpack.esql.core.expression.TypedAttribute;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.FunctionEsField;
 import org.elasticsearch.xpack.esql.core.util.Check;
 import org.elasticsearch.xpack.esql.plugin.EsqlFlags;
 import org.elasticsearch.xpack.esql.stats.SearchStats;
@@ -194,6 +195,9 @@ public boolean isIndexedAndHasDocValues(FieldAttribute attr) {
                 // We still consider the value of isAggregatable here, because some fields like ScriptFieldTypes are always aggregatable
                 // But this could hide issues with fields that are not indexed but are aggregatable
                 // This is the original behaviour for ES|QL, but is it correct?
+                if (attr.field().pushable() == false) {
+                    return false;
+                }
                 return attr.field().isAggregatable()
                     || stats.isIndexed(new FieldAttribute.FieldName(attr.name()))
                         && stats.hasDocValues(new FieldAttribute.FieldName(attr.name()));

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java

@@ -1129,7 +1129,6 @@ public void testVectorFunctionsReplaced() {
         // Check replaced field attribute
         FieldAttribute fieldAttr = (FieldAttribute) alias.child();
         assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector"));
-        assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct"));
         var field = as(fieldAttr.field(), FunctionEsField.class);
         var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class);
         assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION));
@@ -1185,7 +1184,6 @@ public void testVectorFunctionsReplacedWithTopN() {
         // Check replaced field attribute
         FieldAttribute fieldAttr = (FieldAttribute) alias.child();
         assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector"));
-        assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct"));
         var field = as(fieldAttr.field(), FunctionEsField.class);
         var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class);
         assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION));
@@ -1300,7 +1298,6 @@ public void testVectorFunctionsInWhere() {
         // Check left side is the replaced field attribute
         var fieldAttr = as(greaterThan.left(), FieldAttribute.class);
         assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector"));
-        assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct"));
         var field = as(fieldAttr.field(), FunctionEsField.class);
         var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class);
         assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION));
@@ -1349,7 +1346,6 @@ public void testVectorFunctionsInStats() {
         // Check left side is the replaced field attribute
         var fieldAttr = as(filterCondition.left(), FieldAttribute.class);
         assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector"));
-        assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct"));
         var field = as(fieldAttr.field(), FunctionEsField.class);
         var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class);
         assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION));
@@ -1392,7 +1388,6 @@ public void testVectorFunctionsUpdateIntermediateProjections() {
         // Check replaced field attribute
         var fieldAttr = as(alias.child(), FieldAttribute.class);
         assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector"));
-        assertThat(fieldAttr.name(), startsWith("$$dense_vector$CosineSimilarity"));
         var field = as(fieldAttr.field(), FunctionEsField.class);
         var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class);
         assertThat(blockLoaderFunctionConfig.similarityFunction(), is(CosineSimilarity.SIMILARITY_FUNCTION));
@@ -1406,9 +1401,7 @@ public void testVectorFunctionsUpdateIntermediateProjections() {
         var innerProject = as(mvExpand.child(), EsqlProject.class);
         assertTrue(Expressions.names(innerProject.projections()).contains("keyword"));
         assertTrue(
-            innerProject.projections()
-                .stream()
-                .anyMatch(p -> (p instanceof FieldAttribute fa) && fa.name().startsWith("$$dense_vector$CosineSimilarity"))
+            innerProject.projections().stream().anyMatch(p -> (p instanceof FieldAttribute fa) && fa.name().startsWith("$$dense_vector$"))
         );
 
         // EsRelation[test_all][$$dense_vector$CosineSimilarity$33{f}#33, !alias_in..]
@@ -1441,7 +1434,6 @@ public void testVectorFunctionsWithDuplicateFunctions() {
         assertThat(s1Alias.name(), equalTo("s1"));
         var s1FieldAttr = as(s1Alias.child(), FieldAttribute.class);
         assertThat(s1FieldAttr.fieldName().string(), equalTo("dense_vector"));
-        assertThat(s1FieldAttr.name(), startsWith("$$dense_vector$DotProduct"));
         var s1Field = as(s1FieldAttr.field(), FunctionEsField.class);
         var s1Config = as(s1Field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class);
         assertThat(s1Config.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION));
@@ -1460,7 +1452,6 @@ public void testVectorFunctionsWithDuplicateFunctions() {
         assertThat(r1Alias.name(), equalTo("r1"));
         var r1FieldAttr = as(r1Alias.child(), FieldAttribute.class);
         assertThat(r1FieldAttr.fieldName().string(), equalTo("dense_vector"));
-        assertThat(r1FieldAttr.name(), startsWith("$$dense_vector$DotProduct"));
         var r1Field = as(r1FieldAttr.field(), FunctionEsField.class);
         var r1Config = as(r1Field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class);
         assertThat(r1Config.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION));
@@ -1478,7 +1469,6 @@ public void testVectorFunctionsWithDuplicateFunctions() {
         // Right side: CosineSimilarity field
         var r2CosineFieldAttr = as(r2Add.right(), FieldAttribute.class);
         assertThat(r2CosineFieldAttr.fieldName().string(), equalTo("dense_vector"));
-        assertThat(r2CosineFieldAttr.name(), startsWith("$$dense_vector$CosineSimilarity"));
         var r2CosineField = as(r2CosineFieldAttr.field(), FunctionEsField.class);
         var r2CosineConfig = as(r2CosineField.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class);
         assertThat(r2CosineConfig.similarityFunction(), is(CosineSimilarity.SIMILARITY_FUNCTION));