From a3c526e5732f08302db18209c02a9e2add41a23b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Oct 2025 10:23:17 -0400 Subject: [PATCH 01/18] ESQL: Make field fusion generic Speeds up queries like ``` FROM foo | STATS SUM(LENGTH(field)) ``` by fusing the `LENGTH` into the loading of the `field` if it has doc values. Running a fairly simple test: https://gist.github.com/nik9000/9dac067f8ce29875a4fb0f0359a75091 I'm seeing that query drop from 48ms to 28ms. So, like, 40% faster. More importantly, this makes the mechanism for fusing functions into field loading generic. All you have to do is implement `BlockLoaderExpression` on your expression and return non-null from `tryFuse`. --- server/src/main/java/module-info.java | 1 + .../index/mapper/KeywordFieldMapper.java | 17 ++++- .../index/mapper/MappedFieldType.java | 8 +-- .../BlockLoaderFunctionConfig.java | 23 +++++++ .../blockloader/{docvalues => }/Warnings.java | 4 +- .../Utf8CodePointsFromOrdsBlockLoader.java | 3 +- .../vectors/DenseVectorFieldMapper.java | 6 +- .../blockloader/docvalues/MockWarnings.java | 2 + .../xpack/esql/core/type/FunctionEsField.java | 12 ++-- .../compute/lucene/ShardContext.java | 3 +- .../lucene/LuceneSourceOperatorTests.java | 3 +- .../blockloader/BlockLoaderExpression.java | 45 ++++++++++-- .../function/scalar/string/Length.java | 19 ++++- .../vector/VectorSimilarityFunction.java | 38 +++++++--- .../optimizer/LocalLogicalPlanOptimizer.java | 4 +- ...ns.java => FuseExpressionToFieldLoad.java} | 69 ++++++------------- .../planner/EsPhysicalOperationProviders.java | 7 +- .../ConstantShardContextIndexedByShardId.java | 3 +- .../xpack/esql/type/FunctionEsFieldTests.java | 4 +- 19 files changed, 173 insertions(+), 98 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java rename server/src/main/java/org/elasticsearch/index/mapper/blockloader/{docvalues => }/Warnings.java (95%) rename x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/{PushDownVectorSimilarityFunctions.java => FuseExpressionToFieldLoad.java} (62%) diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 6a4d447c8067d..92e60b309fd3b 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -501,4 +501,5 @@ exports org.elasticsearch.index.codec.vectors.es93 to org.elasticsearch.test.knn; exports org.elasticsearch.search.crossproject; exports org.elasticsearch.index.mapper.blockloader.docvalues; + exports org.elasticsearch.index.mapper.blockloader; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 5ddb98699808b..3e3db558d34a0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -55,7 +55,9 @@ import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.StoredFieldSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromOrdsBlockLoader; +import org.elasticsearch.index.mapper.blockloader.docvalues.Utf8CodePointsFromOrdsBlockLoader; import org.elasticsearch.index.query.AutomatonQueryWithDescription; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; @@ -813,10 +815,19 @@ NamedAnalyzer normalizer() { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues() && (blContext.fieldExtractPreference() != FieldExtractPreference.STORED || isSyntheticSourceEnabled())) { - return new BytesRefsFromOrdsBlockLoader(name()); + return switch (blContext.blockLoaderFunctionConfig()) { + case null -> new BytesRefsFromOrdsBlockLoader(name()); + case BlockLoaderFunctionConfig.Named named -> switch (named.name()) { + case "LENGTH" -> new Utf8CodePointsFromOrdsBlockLoader(named.warnings(), name()); + default -> throw new UnsupportedOperationException("unknown fusion config [" + named.name() + "]"); + }; + default -> throw new UnsupportedOperationException( + "unknown fusion config [" + blContext.blockLoaderFunctionConfig() + "]" + ); + }; } - if (isStored()) { - return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name()); + if (blContext.blockLoaderFunctionConfig() != null) { + throw new UnsupportedOperationException("function fusing only supported for doc values"); } // Multi fields don't have fallback synthetic source. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 1c5d0cc2dfa15..14281a72e88bf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -35,6 +35,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.query.DistanceFeatureQueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardException; @@ -710,11 +711,4 @@ default BlockLoaderFunctionConfig blockLoaderFunctionConfig() { } } - /** - * Marker interface that contains the configuration needed to transform loaded values into blocks. - * Is retrievable from the {@link BlockLoaderContext}. The {@link MappedFieldType} can use this configuration to choose the appropriate - * implementation for transforming loaded values into blocks. - */ - public interface BlockLoaderFunctionConfig {} - } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java new file mode 100644 index 0000000000000..1eded2f5a27c0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader; + +import org.elasticsearch.index.mapper.MappedFieldType; + +/** + * Configuration needed to transform loaded values into blocks. + * {@link MappedFieldType}s will find me in + * {@link MappedFieldType.BlockLoaderContext#blockLoaderFunctionConfig()} and + * use this configuration to choose the appropriate implementation for + * transforming loaded values into blocks. + */ +public interface BlockLoaderFunctionConfig { + record Named(String name, Warnings warnings) implements BlockLoaderFunctionConfig {} +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/Warnings.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/Warnings.java similarity index 95% rename from server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/Warnings.java rename to server/src/main/java/org/elasticsearch/index/mapper/blockloader/Warnings.java index 67fcb543d48c7..78ca012e3654f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/Warnings.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/Warnings.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.index.mapper.blockloader.docvalues; +package org.elasticsearch.index.mapper.blockloader; /** * Warnings returned when loading values for ESQL. These are returned as HTTP 299 headers like so: @@ -17,7 +17,7 @@ * < Warning: 299 Elasticsearch-${ver} "Line 1:27: java.lang.IllegalArgumentException: single-value function encountered multi-value" * } */ -interface Warnings { +public interface Warnings { /** * Register a warning. ESQL deduplicates and limits the number of warnings returned so it should * be fine to blast as many warnings into this as you encounter. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoader.java index 730ef2a58a6f3..df58a47f8e471 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoader.java @@ -16,11 +16,12 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.index.mapper.blockloader.Warnings; import java.io.IOException; import java.util.Arrays; -import static org.elasticsearch.index.mapper.blockloader.docvalues.Warnings.registerSingleValueWarning; +import static org.elasticsearch.index.mapper.blockloader.Warnings.registerSingleValueWarning; /** * A count of utf-8 code points for {@code keyword} style fields that are stored as a lookup table. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index a67ffd77e97f2..114173b6abf58 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -59,6 +59,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.ArraySourceValueFetcher; import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.BlockSourceReader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; @@ -3173,10 +3174,10 @@ public interface SimilarityFunction { } /** - * Configuration for a {@link MappedFieldType.BlockLoaderFunctionConfig} that calculates vector similarity. + * Configuration for a {@link BlockLoaderFunctionConfig} that calculates vector similarity. * Functions that use this config should use SIMILARITY_FUNCTION_NAME as their name. */ - public static class VectorSimilarityFunctionConfig implements MappedFieldType.BlockLoaderFunctionConfig { + public static class VectorSimilarityFunctionConfig implements BlockLoaderFunctionConfig { private final SimilarityFunction similarityFunction; private final float[] vector; @@ -3185,7 +3186,6 @@ public static class VectorSimilarityFunctionConfig implements MappedFieldType.Bl public VectorSimilarityFunctionConfig(SimilarityFunction similarityFunction, float[] vector) { this.similarityFunction = similarityFunction; this.vector = vector; - } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MockWarnings.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MockWarnings.java index cfb9ffc93fc76..74664c3ca278f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MockWarnings.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MockWarnings.java @@ -9,6 +9,8 @@ package org.elasticsearch.index.mapper.blockloader.docvalues; +import org.elasticsearch.index.mapper.blockloader.Warnings; + import java.util.ArrayList; import java.util.List; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java index ec4e2831524c8..3ef4f4bc2fc00 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.esql.core.type; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import java.io.IOException; import java.util.Map; @@ -16,15 +16,15 @@ /** * EsField that represents a function being applied to a field on extraction. It receives a - * {@link org.elasticsearch.index.mapper.MappedFieldType.BlockLoaderFunctionConfig} that will be passed down to the block loading process + * {@link BlockLoaderFunctionConfig} that will be passed down to the block loading process * to apply the function at data load time. */ public class FunctionEsField extends EsField { // Not serialized as it will be created on the data node - private final transient MappedFieldType.BlockLoaderFunctionConfig functionConfig; + private final transient BlockLoaderFunctionConfig functionConfig; - public FunctionEsField(EsField esField, DataType dataType, MappedFieldType.BlockLoaderFunctionConfig functionConfig) { + public FunctionEsField(EsField esField, DataType dataType, BlockLoaderFunctionConfig functionConfig) { this( esField.getName(), dataType, @@ -43,7 +43,7 @@ private FunctionEsField( boolean aggregatable, boolean isAlias, TimeSeriesFieldType timeSeriesFieldType, - MappedFieldType.BlockLoaderFunctionConfig functionConfig + BlockLoaderFunctionConfig functionConfig ) { super(name, esDataType, properties, aggregatable, isAlias, timeSeriesFieldType); this.functionConfig = functionConfig; @@ -54,7 +54,7 @@ public void writeTo(StreamOutput out) throws IOException { throw new UnsupportedOperationException("FunctionEsField is not serializable, should be created on data nodes"); } - public MappedFieldType.BlockLoaderFunctionConfig functionConfig() { + public BlockLoaderFunctionConfig functionConfig() { return functionConfig; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java index 2d7ed5e0c6cee..09d43aaf842a0 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java @@ -11,6 +11,7 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.core.RefCounted; import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.search.sort.SortAndFormats; @@ -58,7 +59,7 @@ BlockLoader blockLoader( String name, boolean asUnsupportedSource, MappedFieldType.FieldExtractPreference fieldExtractPreference, - MappedFieldType.BlockLoaderFunctionConfig blockLoaderFunctionConfig + BlockLoaderFunctionConfig blockLoaderFunctionConfig ); /** diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java index aaa2ceb97000a..b99aa7e8c20bc 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy; import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; @@ -467,7 +468,7 @@ public BlockLoader blockLoader( String name, boolean asUnsupportedSource, MappedFieldType.FieldExtractPreference fieldExtractPreference, - MappedFieldType.BlockLoaderFunctionConfig blockLoaderFunctionConfig + BlockLoaderFunctionConfig blockLoaderFunctionConfig ) { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java index 61ed67dae1222..1560a8037d78d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java @@ -7,17 +7,50 @@ package org.elasticsearch.xpack.esql.expression.function.blockloader; -import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.stats.SearchStats; /** - * {@link org.elasticsearch.xpack.esql.core.expression.Expression}s that can be implemented as part of value loading implement this - * interface to provide the {@link MappedFieldType.BlockLoaderFunctionConfig} that will be used to load and - * transform the value of the field. + * {@link Expression} that can be "fused" into value loading. Most of the time + * we load values into {@link Block}s and then run the expressions on them, but + * sometimes it's worth short-circuiting this process and running the expression + * in the tight loop we use for loading: + * */ public interface BlockLoaderExpression { + /** + * The field and loading configuration that replaces this expression, effectively + * "fusing" the expression into the load. Or null if the fusion isn't possible. + */ + @Nullable + Fuse tryFuse(SearchStats stats); /** - * Returns the configuration that will be used to load the value of the field and transform it + * Fused load configuration. + * @param field the field whose load we're fusing into + * @param config the fusion configuration */ - MappedFieldType.BlockLoaderFunctionConfig getBlockLoaderFunctionConfig(); + record Fuse(FieldAttribute field, BlockLoaderFunctionConfig config) {} } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java index 3b442a8583a0a..182a5dd3ffa9d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java @@ -13,14 +13,18 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.blockloader.BlockLoaderExpression; import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.esql.stats.SearchStats; import java.io.IOException; import java.util.List; @@ -28,7 +32,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; -public class Length extends UnaryScalarFunction { +public class Length extends UnaryScalarFunction implements BlockLoaderExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Length", Length::new); @FunctionInfo( @@ -90,4 +94,17 @@ protected NodeInfo info() { public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new LengthEvaluator.Factory(source(), toEvaluator.apply(field())); } + + @Override + public Fuse tryFuse(SearchStats stats) { + if (field instanceof FieldAttribute f) { + if (stats.hasDocValues(f.fieldName()) == false) { + return null; + } + return new Fuse(f, new BlockLoaderFunctionConfig.Named("LENGTH", (exceptionClass, message) -> { + throw new IllegalStateException("NOCOMMIT"); + })); + } + return null; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java index 94c3c54ec29fb..b8932e3e606c6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java @@ -18,9 +18,9 @@ import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; @@ -29,6 +29,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.blockloader.BlockLoaderExpression; +import org.elasticsearch.xpack.esql.stats.SearchStats; import java.io.IOException; import java.util.ArrayList; @@ -207,17 +208,34 @@ public void close() { } @Override - public MappedFieldType.BlockLoaderFunctionConfig getBlockLoaderFunctionConfig() { - // PushDownVectorSimilarityFunctions checks that one of the sides is a literal - Literal literal = (Literal) (left() instanceof Literal ? left() : right()); - @SuppressWarnings("unchecked") - List numberList = (List) literal.value(); - float[] vector = new float[numberList.size()]; - for (int i = 0; i < numberList.size(); i++) { - vector[i] = numberList.get(i).floatValue(); + public Fuse tryFuse(SearchStats stats) { + // Bail if we're not directly comparing a field with a literal. + Literal literal; + FieldAttribute field; + if (left() instanceof Literal lit && right() instanceof FieldAttribute f) { + literal = lit; + field = f; + } else if (left() instanceof FieldAttribute f && right() instanceof Literal lit) { + literal = lit; + field = f; + } else { + return null; + } + + // Bail if the field isn't indexed. + if (stats.isIndexed(field.fieldName()) == false) { + return null; + } + + List vectorList = (List) literal.value(); + float[] vectorArray = new float[vectorList.size()]; + int arrayHashCode = 0; + for (int i = 0; i < vectorList.size(); i++) { + vectorArray[i] = ((Number) vectorList.get(i)).floatValue(); + arrayHashCode = 31 * arrayHashCode + Float.floatToIntBits(vectorArray[i]); } - return new DenseVectorFieldMapper.VectorSimilarityFunctionConfig(getSimilarityFunction(), vector); + return new Fuse(field, new DenseVectorFieldMapper.VectorSimilarityFunctionConfig(getSimilarityFunction(), vectorArray)); } interface VectorValueProvider extends Releasable { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index 9e852a9c0d595..3b556aae85a7b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -15,7 +15,7 @@ import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.IgnoreNullMetrics; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferIsNotNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferNonNullAggConstraint; -import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.PushDownVectorSimilarityFunctions; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.FuseExpressionToFieldLoad; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceDateTruncBucketWithRoundTo; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceFieldWithConstantOrNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceTopNWithLimitAndSort; @@ -71,7 +71,7 @@ private static Batch localRewrites() { ) ); if (EsqlCapabilities.Cap.VECTOR_SIMILARITY_FUNCTIONS_PUSHDOWN.isEnabled()) { - rules.add(new PushDownVectorSimilarityFunctions()); + rules.add(new FuseExpressionToFieldLoad()); } return new Batch<>("Local rewrite", Limiter.ONCE, rules.toArray(Rule[]::new)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/PushDownVectorSimilarityFunctions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java similarity index 62% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/PushDownVectorSimilarityFunctions.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java index a5dc214b442c9..f51a765fdfc72 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/PushDownVectorSimilarityFunctions.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java @@ -11,12 +11,11 @@ import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; -import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.FunctionEsField; -import org.elasticsearch.xpack.esql.expression.function.vector.VectorSimilarityFunction; +import org.elasticsearch.xpack.esql.expression.function.blockloader.BlockLoaderExpression; import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; @@ -38,11 +37,9 @@ * the similarity function during value loading, when one side of the function is a literal. * It also adds the new field function attribute to the EsRelation output, and adds a projection after it to remove it from the output. */ -public class PushDownVectorSimilarityFunctions extends OptimizerRules.ParameterizedOptimizerRule< - LogicalPlan, - LocalLogicalOptimizerContext> { +public class FuseExpressionToFieldLoad extends OptimizerRules.ParameterizedOptimizerRule { - public PushDownVectorSimilarityFunctions() { + public FuseExpressionToFieldLoad() { super(OptimizerRules.TransformDirection.DOWN); } @@ -50,10 +47,15 @@ public PushDownVectorSimilarityFunctions() { protected LogicalPlan rule(LogicalPlan plan, LocalLogicalOptimizerContext context) { if (plan instanceof Eval || plan instanceof Filter || plan instanceof Aggregate) { Map addedAttrs = new HashMap<>(); - LogicalPlan transformedPlan = plan.transformExpressionsOnly( - VectorSimilarityFunction.class, - similarityFunction -> replaceFieldsForFieldTransformations(similarityFunction, addedAttrs, context) - ); + LogicalPlan transformedPlan = plan.transformExpressionsOnly(Expression.class, e -> { + if (e instanceof BlockLoaderExpression ble) { + BlockLoaderExpression.Fuse fuse = ble.tryFuse(context.searchStats()); + if (fuse != null) { + return replaceFieldsForFieldTransformations(e, addedAttrs, fuse); + } + } + return e; + }); if (addedAttrs.isEmpty()) { return plan; @@ -81,57 +83,26 @@ protected LogicalPlan rule(LogicalPlan plan, LocalLogicalOptimizerContext contex } private static Expression replaceFieldsForFieldTransformations( - VectorSimilarityFunction similarityFunction, + Expression e, Map addedAttrs, - LocalLogicalOptimizerContext context + BlockLoaderExpression.Fuse fuse ) { - // Only replace if exactly one side is a literal and the other a field attribute - if ((similarityFunction.left() instanceof Literal ^ similarityFunction.right() instanceof Literal) == false) { - return similarityFunction; - } - - Literal literal = (Literal) (similarityFunction.left() instanceof Literal ? similarityFunction.left() : similarityFunction.right()); - FieldAttribute fieldAttr = null; - if (similarityFunction.left() instanceof FieldAttribute fa) { - fieldAttr = fa; - } else if (similarityFunction.right() instanceof FieldAttribute fa) { - fieldAttr = fa; - } - // We can push down also for doc values, requires handling that case on the field mapper - if (fieldAttr == null || context.searchStats().isIndexed(fieldAttr.fieldName()) == false) { - return similarityFunction; - } - - @SuppressWarnings("unchecked") - List vectorList = (List) literal.value(); - float[] vectorArray = new float[vectorList.size()]; - int arrayHashCode = 0; - for (int i = 0; i < vectorList.size(); i++) { - vectorArray[i] = vectorList.get(i).floatValue(); - arrayHashCode = 31 * arrayHashCode + Float.floatToIntBits(vectorArray[i]); - } - // Change the similarity function to a reference of a transformation on the field - FunctionEsField functionEsField = new FunctionEsField( - fieldAttr.field(), - similarityFunction.dataType(), - similarityFunction.getBlockLoaderFunctionConfig() - ); - var name = rawTemporaryName(fieldAttr.name(), similarityFunction.nodeName(), String.valueOf(arrayHashCode)); + FunctionEsField functionEsField = new FunctionEsField(fuse.field().field(), e.dataType(), fuse.config()); + var name = rawTemporaryName(fuse.field().name(), String.valueOf(functionEsField.hashCode())); // TODO: Check if exists before adding, retrieve the previous one var newFunctionAttr = new FieldAttribute( - fieldAttr.source(), - fieldAttr.parentName(), - fieldAttr.qualifier(), + fuse.field().source(), + fuse.field().parentName(), + fuse.field().qualifier(), name, functionEsField, - fieldAttr.nullable(), + fuse.field().nullable(), new NameId(), true ); Attribute.IdIgnoringWrapper key = newFunctionAttr.ignoreId(); if (addedAttrs.containsKey(key)) { - ; return addedAttrs.get(key); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 281788d29f759..59abaac375895 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -194,7 +195,7 @@ private BlockLoader getBlockLoaderFor(int shardId, Attribute attr, MappedFieldTy } // Apply any block loader function if present - MappedFieldType.BlockLoaderFunctionConfig functionConfig = null; + BlockLoaderFunctionConfig functionConfig = null; if (attr instanceof FieldAttribute fieldAttr && fieldAttr.field() instanceof FunctionEsField functionEsField) { functionConfig = functionEsField.functionConfig(); } @@ -488,7 +489,7 @@ public BlockLoader blockLoader( String name, boolean asUnsupportedSource, MappedFieldType.FieldExtractPreference fieldExtractPreference, - MappedFieldType.BlockLoaderFunctionConfig blockLoaderFunctionConfig + BlockLoaderFunctionConfig blockLoaderFunctionConfig ) { if (asUnsupportedSource) { return BlockLoader.CONSTANT_NULLS; @@ -535,7 +536,7 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { } @Override - public MappedFieldType.BlockLoaderFunctionConfig blockLoaderFunctionConfig() { + public BlockLoaderFunctionConfig blockLoaderFunctionConfig() { return blockLoaderFunctionConfig; } }); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java index 4d39782c23e58..01e024524ab21 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.Query; import org.elasticsearch.compute.lucene.IndexedByShardId; import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.QueryBuilder; @@ -70,7 +71,7 @@ public BlockLoader blockLoader( String name, boolean asUnsupportedSource, MappedFieldType.FieldExtractPreference fieldExtractPreference, - MappedFieldType.BlockLoaderFunctionConfig blockLoaderFunctionConfig + BlockLoaderFunctionConfig blockLoaderFunctionConfig ) { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/FunctionEsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/FunctionEsFieldTests.java index 4a31bb4298769..b860c6975c1e3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/FunctionEsFieldTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/FunctionEsFieldTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.esql.type; import org.elasticsearch.TransportVersion; -import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; @@ -34,7 +34,7 @@ protected FunctionEsField createTestInstance() { protected FunctionEsField mutateInstance(FunctionEsField instance) throws IOException { EsField esField = instance.getExactField(); DataType dataType = instance.getDataType(); - MappedFieldType.BlockLoaderFunctionConfig functionConfig = instance.functionConfig(); + BlockLoaderFunctionConfig functionConfig = instance.functionConfig(); switch (between(0, 2)) { case 0 -> esField = randomValueOtherThan(esField, () -> randomAnyEsField(4)); case 1 -> dataType = randomValueOtherThan(dataType, () -> randomFrom(DataType.types())); From 7758ab968dab1a8bec26718d096f5bd94c851b97 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Oct 2025 10:39:21 -0400 Subject: [PATCH 02/18] Update docs/changelog/137382.yaml --- docs/changelog/137382.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/137382.yaml diff --git a/docs/changelog/137382.yaml b/docs/changelog/137382.yaml new file mode 100644 index 0000000000000..1e044843e9242 --- /dev/null +++ b/docs/changelog/137382.yaml @@ -0,0 +1,5 @@ +pr: 137382 +summary: Make field fusion generic +area: ES|QL +type: enhancement +issues: [] From 47c874ec3b1e3c22dfe2890e02e6785e762c56e4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 30 Oct 2025 14:46:34 +0000 Subject: [PATCH 03/18] [CI] Auto commit changes from spotless --- .../index/mapper/vectors/DenseVectorFieldMapper.java | 2 +- .../java/org/elasticsearch/compute/lucene/ShardContext.java | 2 +- .../elasticsearch/compute/lucene/LuceneSourceOperatorTests.java | 2 +- .../xpack/esql/optimizer/LocalLogicalPlanOptimizer.java | 2 +- .../xpack/esql/planner/EsPhysicalOperationProviders.java | 2 +- .../esql/planner/ConstantShardContextIndexedByShardId.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 114173b6abf58..ac8fc242a18eb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -59,7 +59,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.ArraySourceValueFetcher; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.BlockSourceReader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; @@ -74,6 +73,7 @@ import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.blockloader.docvalues.DenseVectorBlockLoader; import org.elasticsearch.index.mapper.blockloader.docvalues.DenseVectorBlockLoaderProcessor; import org.elasticsearch.index.mapper.blockloader.docvalues.DenseVectorFromBinaryBlockLoader; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java index 09d43aaf842a0..89440a440f23b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java @@ -11,9 +11,9 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.core.RefCounted; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java index b99aa7e8c20bc..ba753c96c5222 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java @@ -40,10 +40,10 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.sort.SortAndFormats; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index 3b556aae85a7b..fea93bd1ee1cd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -12,10 +12,10 @@ import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveRegexMatch; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.FuseExpressionToFieldLoad; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.IgnoreNullMetrics; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferIsNotNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferNonNullAggConstraint; -import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.FuseExpressionToFieldLoad; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceDateTruncBucketWithRoundTo; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceFieldWithConstantOrNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceTopNWithLimitAndSort; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 59abaac375895..55750adffd207 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -40,12 +40,12 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.ExistsQueryBuilder; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java index 01e024524ab21..8af35d05843ee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java @@ -11,9 +11,9 @@ import org.apache.lucene.search.Query; import org.elasticsearch.compute.lucene.IndexedByShardId; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; From 6b0fead45dbad7b32449ba1d35720c20120fa565 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Oct 2025 12:05:38 -0400 Subject: [PATCH 04/18] More tests --- .../BlockLoaderFunctionConfig.java | 16 ++++++++- .../vectors/DenseVectorFieldMapper.java | 2 +- .../{docvalues => }/MockWarnings.java | 9 ++--- ...tf8CodePointsFromOrdsBlockLoaderTests.java | 1 + .../compute/lucene/ShardContext.java | 2 +- .../lucene/LuceneSourceOperatorTests.java | 2 +- .../function/BlockLoaderWarnings.java | 36 +++++++++++++++++++ .../function/scalar/string/Length.java | 7 ++-- .../vector/VectorSimilarityFunction.java | 4 +-- .../optimizer/LocalLogicalPlanOptimizer.java | 2 +- .../planner/EsPhysicalOperationProviders.java | 2 +- .../ConstantShardContextIndexedByShardId.java | 2 +- .../xpack/esql/stats/DisabledSearchStats.java | 3 +- 13 files changed, 68 insertions(+), 20 deletions(-) rename server/src/test/java/org/elasticsearch/index/mapper/blockloader/{docvalues => }/MockWarnings.java (73%) create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/BlockLoaderWarnings.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java index 1eded2f5a27c0..c417266e0a1a2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java @@ -19,5 +19,19 @@ * transforming loaded values into blocks. */ public interface BlockLoaderFunctionConfig { - record Named(String name, Warnings warnings) implements BlockLoaderFunctionConfig {} + record Named(String name, Warnings warnings) implements BlockLoaderFunctionConfig { + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + Named named = (Named) o; + return name.equals(named.name); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 114173b6abf58..ac8fc242a18eb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -59,7 +59,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.ArraySourceValueFetcher; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.BlockSourceReader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; @@ -74,6 +73,7 @@ import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.blockloader.docvalues.DenseVectorBlockLoader; import org.elasticsearch.index.mapper.blockloader.docvalues.DenseVectorBlockLoaderProcessor; import org.elasticsearch.index.mapper.blockloader.docvalues.DenseVectorFromBinaryBlockLoader; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MockWarnings.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/MockWarnings.java similarity index 73% rename from server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MockWarnings.java rename to server/src/test/java/org/elasticsearch/index/mapper/blockloader/MockWarnings.java index 74664c3ca278f..37331d1fdfb86 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MockWarnings.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/MockWarnings.java @@ -7,16 +7,13 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.index.mapper.blockloader.docvalues; - -import org.elasticsearch.index.mapper.blockloader.Warnings; +package org.elasticsearch.index.mapper.blockloader; import java.util.ArrayList; import java.util.List; -// TODO move me once we have more of these loaders -class MockWarnings implements Warnings { - record MockWarning(Class exceptionClass, String message) {} +public class MockWarnings implements Warnings { + public record MockWarning(Class exceptionClass, String message) {} private final List warnings = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java index 5613abd02f4e5..f8e78ee46a981 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.TestBlock; +import org.elasticsearch.index.mapper.blockloader.MockWarnings; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matcher; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java index 09d43aaf842a0..89440a440f23b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ShardContext.java @@ -11,9 +11,9 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.core.RefCounted; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java index b99aa7e8c20bc..ba753c96c5222 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSourceOperatorTests.java @@ -40,10 +40,10 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.sort.SortAndFormats; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/BlockLoaderWarnings.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/BlockLoaderWarnings.java new file mode 100644 index 0000000000000..6a6fa5413d6e8 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/BlockLoaderWarnings.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function; + +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class BlockLoaderWarnings implements org.elasticsearch.index.mapper.blockloader.Warnings { + private final DriverContext.WarningsMode warningsMode; + private final Source source; + private Warnings delegate; + + public BlockLoaderWarnings(DriverContext.WarningsMode warningsMode, Source source) { + this.warningsMode = warningsMode; + this.source = source; + } + + @Override + public void registerException(Class exceptionClass, String message) { + if (delegate == null) { + delegate = Warnings.createOnlyWarnings( + warningsMode, + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + delegate.registerException(exceptionClass, message); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java index 182a5dd3ffa9d..362986c4bcb26 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -19,6 +20,7 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.BlockLoaderWarnings; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; @@ -101,9 +103,8 @@ public Fuse tryFuse(SearchStats stats) { if (stats.hasDocValues(f.fieldName()) == false) { return null; } - return new Fuse(f, new BlockLoaderFunctionConfig.Named("LENGTH", (exceptionClass, message) -> { - throw new IllegalStateException("NOCOMMIT"); - })); + BlockLoaderWarnings warnings = new BlockLoaderWarnings(DriverContext.WarningsMode.COLLECT, source()); + return new Fuse(f, new BlockLoaderFunctionConfig.Named("LENGTH", warnings)); } return null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java index b8932e3e606c6..ed9f8df5a7f99 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java @@ -208,7 +208,7 @@ public void close() { } @Override - public Fuse tryFuse(SearchStats stats) { + public final Fuse tryFuse(SearchStats stats) { // Bail if we're not directly comparing a field with a literal. Literal literal; FieldAttribute field; @@ -229,10 +229,8 @@ public Fuse tryFuse(SearchStats stats) { List vectorList = (List) literal.value(); float[] vectorArray = new float[vectorList.size()]; - int arrayHashCode = 0; for (int i = 0; i < vectorList.size(); i++) { vectorArray[i] = ((Number) vectorList.get(i)).floatValue(); - arrayHashCode = 31 * arrayHashCode + Float.floatToIntBits(vectorArray[i]); } return new Fuse(field, new DenseVectorFieldMapper.VectorSimilarityFunctionConfig(getSimilarityFunction(), vectorArray)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index 3b556aae85a7b..fea93bd1ee1cd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -12,10 +12,10 @@ import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveRegexMatch; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.FuseExpressionToFieldLoad; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.IgnoreNullMetrics; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferIsNotNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferNonNullAggConstraint; -import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.FuseExpressionToFieldLoad; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceDateTruncBucketWithRoundTo; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceFieldWithConstantOrNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceTopNWithLimitAndSort; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 59abaac375895..55750adffd207 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -40,12 +40,12 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.ExistsQueryBuilder; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java index 01e024524ab21..8af35d05843ee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/ConstantShardContextIndexedByShardId.java @@ -11,9 +11,9 @@ import org.apache.lucene.search.Query; import org.elasticsearch.compute.lucene.IndexedByShardId; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/DisabledSearchStats.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/DisabledSearchStats.java index c63c7aff03909..f726b941b0bb7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/DisabledSearchStats.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/DisabledSearchStats.java @@ -25,7 +25,8 @@ public boolean isIndexed(FieldName field) { @Override public boolean hasDocValues(FieldName field) { - return true; + // Some geo tests assume doc values and the loader emulates it. Nothing else does. + return field.string().endsWith("location") || field.string().endsWith("centroid") || field.string().equals("subset"); } @Override From 1dd6d96f1f252f92ff292441b33d006d279bbd31 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Oct 2025 15:42:25 -0400 Subject: [PATCH 05/18] Tests --- .../index/mapper/KeywordFieldMapper.java | 3 +++ .../elasticsearch/xpack/esql/core/type/EsField.java | 7 +++++++ .../xpack/esql/core/type/FunctionEsField.java | 6 ++++++ .../physical/local/LucenePushdownPredicates.java | 4 ++++ .../optimizer/LocalLogicalPlanOptimizerTests.java | 12 +----------- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 3e3db558d34a0..de97283ecf00b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -829,6 +829,9 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { if (blContext.blockLoaderFunctionConfig() != null) { throw new UnsupportedOperationException("function fusing only supported for doc values"); } + if (isStored()) { + return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name()); + } // Multi fields don't have fallback synthetic source. if (isSyntheticSourceEnabled() && blContext.parentField(name()) == null) { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java index a6b8e432fc205..df62f7dc6b170 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java @@ -252,6 +252,13 @@ public EsField getExactField() { return this; } + /** + * Can this field be pushed if it is indexed? + */ + public boolean pushable() { + return true; + } + /** * Returns and {@link Exact} object with all the necessary info about the field: *
    diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java index 3ef4f4bc2fc00..63e3de5b13b39 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java @@ -70,4 +70,10 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(super.hashCode(), functionConfig); } + + @Override + public boolean pushable() { + // These fields are *never* pushable to the lucene index. + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index aa9ea3b0e004b..efffdbdcd6f43 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.FunctionEsField; import org.elasticsearch.xpack.esql.core.util.Check; import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.stats.SearchStats; @@ -194,6 +195,9 @@ public boolean isIndexedAndHasDocValues(FieldAttribute attr) { // We still consider the value of isAggregatable here, because some fields like ScriptFieldTypes are always aggregatable // But this could hide issues with fields that are not indexed but are aggregatable // This is the original behaviour for ES|QL, but is it correct? + if (attr.field().pushable() == false) { + return false; + } return attr.field().isAggregatable() || stats.isIndexed(new FieldAttribute.FieldName(attr.name())) && stats.hasDocValues(new FieldAttribute.FieldName(attr.name())); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index 7925fe5ef9242..2a7e462b9e170 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -1129,7 +1129,6 @@ public void testVectorFunctionsReplaced() { // Check replaced field attribute FieldAttribute fieldAttr = (FieldAttribute) alias.child(); assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector")); - assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var field = as(fieldAttr.field(), FunctionEsField.class); var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1185,7 +1184,6 @@ public void testVectorFunctionsReplacedWithTopN() { // Check replaced field attribute FieldAttribute fieldAttr = (FieldAttribute) alias.child(); assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector")); - assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var field = as(fieldAttr.field(), FunctionEsField.class); var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1300,7 +1298,6 @@ public void testVectorFunctionsInWhere() { // Check left side is the replaced field attribute var fieldAttr = as(greaterThan.left(), FieldAttribute.class); assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector")); - assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var field = as(fieldAttr.field(), FunctionEsField.class); var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1349,7 +1346,6 @@ public void testVectorFunctionsInStats() { // Check left side is the replaced field attribute var fieldAttr = as(filterCondition.left(), FieldAttribute.class); assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector")); - assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var field = as(fieldAttr.field(), FunctionEsField.class); var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1392,7 +1388,6 @@ public void testVectorFunctionsUpdateIntermediateProjections() { // Check replaced field attribute var fieldAttr = as(alias.child(), FieldAttribute.class); assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector")); - assertThat(fieldAttr.name(), startsWith("$$dense_vector$CosineSimilarity")); var field = as(fieldAttr.field(), FunctionEsField.class); var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(blockLoaderFunctionConfig.similarityFunction(), is(CosineSimilarity.SIMILARITY_FUNCTION)); @@ -1406,9 +1401,7 @@ public void testVectorFunctionsUpdateIntermediateProjections() { var innerProject = as(mvExpand.child(), EsqlProject.class); assertTrue(Expressions.names(innerProject.projections()).contains("keyword")); assertTrue( - innerProject.projections() - .stream() - .anyMatch(p -> (p instanceof FieldAttribute fa) && fa.name().startsWith("$$dense_vector$CosineSimilarity")) + innerProject.projections().stream().anyMatch(p -> (p instanceof FieldAttribute fa) && fa.name().startsWith("$$dense_vector$")) ); // EsRelation[test_all][$$dense_vector$CosineSimilarity$33{f}#33, !alias_in..] @@ -1441,7 +1434,6 @@ public void testVectorFunctionsWithDuplicateFunctions() { assertThat(s1Alias.name(), equalTo("s1")); var s1FieldAttr = as(s1Alias.child(), FieldAttribute.class); assertThat(s1FieldAttr.fieldName().string(), equalTo("dense_vector")); - assertThat(s1FieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var s1Field = as(s1FieldAttr.field(), FunctionEsField.class); var s1Config = as(s1Field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(s1Config.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1460,7 +1452,6 @@ public void testVectorFunctionsWithDuplicateFunctions() { assertThat(r1Alias.name(), equalTo("r1")); var r1FieldAttr = as(r1Alias.child(), FieldAttribute.class); assertThat(r1FieldAttr.fieldName().string(), equalTo("dense_vector")); - assertThat(r1FieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var r1Field = as(r1FieldAttr.field(), FunctionEsField.class); var r1Config = as(r1Field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(r1Config.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1478,7 +1469,6 @@ public void testVectorFunctionsWithDuplicateFunctions() { // Right side: CosineSimilarity field var r2CosineFieldAttr = as(r2Add.right(), FieldAttribute.class); assertThat(r2CosineFieldAttr.fieldName().string(), equalTo("dense_vector")); - assertThat(r2CosineFieldAttr.name(), startsWith("$$dense_vector$CosineSimilarity")); var r2CosineField = as(r2CosineFieldAttr.field(), FunctionEsField.class); var r2CosineConfig = as(r2CosineField.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(r2CosineConfig.similarityFunction(), is(CosineSimilarity.SIMILARITY_FUNCTION)); From d50a74b25685a7c6b1bd2f8b968b1e5ffcdea116 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 30 Oct 2025 19:49:50 +0000 Subject: [PATCH 06/18] [CI] Auto commit changes from spotless --- .../optimizer/rules/physical/local/LucenePushdownPredicates.java | 1 - .../xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index efffdbdcd6f43..24be3b71c8e91 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -15,7 +15,6 @@ import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.FunctionEsField; import org.elasticsearch.xpack.esql.core.util.Check; import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.stats.SearchStats; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index 2a7e462b9e170..617871b5db3f0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -118,7 +118,6 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.Matchers.startsWith; //@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug") public class LocalLogicalPlanOptimizerTests extends ESTestCase { From 8746cfa5da0b0253bcde45e75b882610250e9371 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 30 Oct 2025 17:19:01 -0400 Subject: [PATCH 07/18] Add names back --- .../blockloader/BlockLoaderFunctionConfig.java | 5 +++++ .../mapper/vectors/DenseVectorFieldMapper.java | 5 +++++ .../function/blockloader/BlockLoaderExpression.java | 4 ++-- .../expression/function/scalar/string/Length.java | 4 ++-- .../function/vector/CosineSimilarity.java | 5 +++++ .../esql/expression/function/vector/DotProduct.java | 5 +++++ .../esql/expression/function/vector/Hamming.java | 10 ++++++++++ .../esql/expression/function/vector/L1Norm.java | 5 +++++ .../esql/expression/function/vector/L2Norm.java | 5 +++++ .../function/vector/VectorSimilarityFunction.java | 4 ++-- .../logical/local/FuseExpressionToFieldLoad.java | 6 +++--- .../optimizer/LocalLogicalPlanOptimizerTests.java | 13 ++++++++++++- 12 files changed, 61 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java index c417266e0a1a2..1d908b9db5b28 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/BlockLoaderFunctionConfig.java @@ -19,6 +19,11 @@ * transforming loaded values into blocks. */ public interface BlockLoaderFunctionConfig { + /** + * Name used in descriptions. + */ + String name(); + record Named(String name, Warnings warnings) implements BlockLoaderFunctionConfig { @Override public int hashCode() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index ac8fc242a18eb..024bb4b8892d6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -3199,6 +3199,11 @@ public VectorSimilarityFunctionConfig forByteVector() { return this; } + @Override + public String name() { + return similarityFunction.toString(); + } + public byte[] vectorAsBytes() { assert vectorAsBytes != null : "vectorAsBytes is null, call forByteVector() first"; return vectorAsBytes; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java index 1560a8037d78d..92608a780e320 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java @@ -45,12 +45,12 @@ public interface BlockLoaderExpression { * "fusing" the expression into the load. Or null if the fusion isn't possible. */ @Nullable - Fuse tryFuse(SearchStats stats); + FusedExpression tryFuse(SearchStats stats); /** * Fused load configuration. * @param field the field whose load we're fusing into * @param config the fusion configuration */ - record Fuse(FieldAttribute field, BlockLoaderFunctionConfig config) {} + record FusedExpression(FieldAttribute field, BlockLoaderFunctionConfig config) {} } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java index 362986c4bcb26..ad3d06868ad10 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java @@ -98,13 +98,13 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } @Override - public Fuse tryFuse(SearchStats stats) { + public FusedExpression tryFuse(SearchStats stats) { if (field instanceof FieldAttribute f) { if (stats.hasDocValues(f.fieldName()) == false) { return null; } BlockLoaderWarnings warnings = new BlockLoaderWarnings(DriverContext.WarningsMode.COLLECT, source()); - return new Fuse(f, new BlockLoaderFunctionConfig.Named("LENGTH", warnings)); + return new FusedExpression(f, new BlockLoaderFunctionConfig.Named("LENGTH", warnings)); } return null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/CosineSimilarity.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/CosineSimilarity.java index d74aefdd0d360..011b4854c0d33 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/CosineSimilarity.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/CosineSimilarity.java @@ -40,6 +40,11 @@ public float calculateSimilarity(byte[] leftVector, byte[] rightVector) { public float calculateSimilarity(float[] leftVector, float[] rightVector) { return VectorUtil.cosine(leftVector, rightVector); } + + @Override + public String toString() { + return "Cosine"; + } }; @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/DotProduct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/DotProduct.java index e319b235e5eaa..f4697b14cbe80 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/DotProduct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/DotProduct.java @@ -41,6 +41,11 @@ public float calculateSimilarity(byte[] leftVector, byte[] rightVector) { public float calculateSimilarity(float[] leftVector, float[] rightVector) { return VectorUtil.dotProduct(leftVector, rightVector); } + + @Override + public String toString() { + return "DotProduct"; + } }; @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Hamming.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Hamming.java index 8ebd771221c48..7e49eea2554ca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Hamming.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/Hamming.java @@ -36,6 +36,11 @@ public float calculateSimilarity(byte[] leftVector, byte[] rightVector) { public float calculateSimilarity(float[] leftVector, float[] rightVector) { throw new UnsupportedOperationException("Hamming distance is not supported for float vectors"); } + + @Override + public String toString() { + return "Hamming"; + } }; public static final DenseVectorFieldMapper.SimilarityFunction EVALUATOR_SIMILARITY_FUNCTION = new DenseVectorFieldMapper.SimilarityFunction() { @@ -56,6 +61,11 @@ public float calculateSimilarity(float[] leftVector, float[] rightVector) { } return Hamming.calculateSimilarity(a, b); } + + @Override + public String toString() { + return "Hamming"; + } }; @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L1Norm.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L1Norm.java index 09bf2681f8559..6c066e7fe0bf5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L1Norm.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L1Norm.java @@ -49,6 +49,11 @@ public float calculateSimilarity(float[] leftVector, float[] rightVector) { } return result; } + + @Override + public String toString() { + return "L1Norm"; + } }; @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java index a6350fc98535c..74088b12cb86f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java @@ -37,6 +37,11 @@ public float calculateSimilarity(byte[] leftVector, byte[] rightVector) { public float calculateSimilarity(float[] leftVector, float[] rightVector) { return (float) Math.sqrt(VectorUtil.squareDistance(leftVector, rightVector)); } + + @Override + public String toString() { + return "L1Norm"; + } }; @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java index ed9f8df5a7f99..40c4469d8f6e2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java @@ -208,7 +208,7 @@ public void close() { } @Override - public final Fuse tryFuse(SearchStats stats) { + public final FusedExpression tryFuse(SearchStats stats) { // Bail if we're not directly comparing a field with a literal. Literal literal; FieldAttribute field; @@ -233,7 +233,7 @@ public final Fuse tryFuse(SearchStats stats) { vectorArray[i] = ((Number) vectorList.get(i)).floatValue(); } - return new Fuse(field, new DenseVectorFieldMapper.VectorSimilarityFunctionConfig(getSimilarityFunction(), vectorArray)); + return new FusedExpression(field, new DenseVectorFieldMapper.VectorSimilarityFunctionConfig(getSimilarityFunction(), vectorArray)); } interface VectorValueProvider extends Releasable { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java index f51a765fdfc72..ead0137326348 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java @@ -49,7 +49,7 @@ protected LogicalPlan rule(LogicalPlan plan, LocalLogicalOptimizerContext contex Map addedAttrs = new HashMap<>(); LogicalPlan transformedPlan = plan.transformExpressionsOnly(Expression.class, e -> { if (e instanceof BlockLoaderExpression ble) { - BlockLoaderExpression.Fuse fuse = ble.tryFuse(context.searchStats()); + BlockLoaderExpression.FusedExpression fuse = ble.tryFuse(context.searchStats()); if (fuse != null) { return replaceFieldsForFieldTransformations(e, addedAttrs, fuse); } @@ -85,11 +85,11 @@ protected LogicalPlan rule(LogicalPlan plan, LocalLogicalOptimizerContext contex private static Expression replaceFieldsForFieldTransformations( Expression e, Map addedAttrs, - BlockLoaderExpression.Fuse fuse + BlockLoaderExpression.FusedExpression fuse ) { // Change the similarity function to a reference of a transformation on the field FunctionEsField functionEsField = new FunctionEsField(fuse.field().field(), e.dataType(), fuse.config()); - var name = rawTemporaryName(fuse.field().name(), String.valueOf(functionEsField.hashCode())); + var name = rawTemporaryName(fuse.field().name(), fuse.config().name(), String.valueOf(fuse.config().hashCode())); // TODO: Check if exists before adding, retrieve the previous one var newFunctionAttr = new FieldAttribute( fuse.field().source(), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index 617871b5db3f0..e10d371c62cd9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -118,6 +118,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; //@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug") public class LocalLogicalPlanOptimizerTests extends ESTestCase { @@ -1128,6 +1129,7 @@ public void testVectorFunctionsReplaced() { // Check replaced field attribute FieldAttribute fieldAttr = (FieldAttribute) alias.child(); assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector")); + assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var field = as(fieldAttr.field(), FunctionEsField.class); var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1183,6 +1185,7 @@ public void testVectorFunctionsReplacedWithTopN() { // Check replaced field attribute FieldAttribute fieldAttr = (FieldAttribute) alias.child(); assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector")); + assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var field = as(fieldAttr.field(), FunctionEsField.class); var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1297,6 +1300,7 @@ public void testVectorFunctionsInWhere() { // Check left side is the replaced field attribute var fieldAttr = as(greaterThan.left(), FieldAttribute.class); assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector")); + assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var field = as(fieldAttr.field(), FunctionEsField.class); var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1345,6 +1349,7 @@ public void testVectorFunctionsInStats() { // Check left side is the replaced field attribute var fieldAttr = as(filterCondition.left(), FieldAttribute.class); assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector")); + assertThat(fieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var field = as(fieldAttr.field(), FunctionEsField.class); var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(blockLoaderFunctionConfig.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1387,6 +1392,7 @@ public void testVectorFunctionsUpdateIntermediateProjections() { // Check replaced field attribute var fieldAttr = as(alias.child(), FieldAttribute.class); assertThat(fieldAttr.fieldName().string(), equalTo("dense_vector")); + assertThat(fieldAttr.name(), startsWith("$$dense_vector$Cosine")); var field = as(fieldAttr.field(), FunctionEsField.class); var blockLoaderFunctionConfig = as(field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(blockLoaderFunctionConfig.similarityFunction(), is(CosineSimilarity.SIMILARITY_FUNCTION)); @@ -1400,7 +1406,9 @@ public void testVectorFunctionsUpdateIntermediateProjections() { var innerProject = as(mvExpand.child(), EsqlProject.class); assertTrue(Expressions.names(innerProject.projections()).contains("keyword")); assertTrue( - innerProject.projections().stream().anyMatch(p -> (p instanceof FieldAttribute fa) && fa.name().startsWith("$$dense_vector$")) + innerProject.projections() + .stream() + .anyMatch(p -> (p instanceof FieldAttribute fa) && fa.name().startsWith("$$dense_vector$Cosine")) ); // EsRelation[test_all][$$dense_vector$CosineSimilarity$33{f}#33, !alias_in..] @@ -1433,6 +1441,7 @@ public void testVectorFunctionsWithDuplicateFunctions() { assertThat(s1Alias.name(), equalTo("s1")); var s1FieldAttr = as(s1Alias.child(), FieldAttribute.class); assertThat(s1FieldAttr.fieldName().string(), equalTo("dense_vector")); + assertThat(s1FieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var s1Field = as(s1FieldAttr.field(), FunctionEsField.class); var s1Config = as(s1Field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(s1Config.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1451,6 +1460,7 @@ public void testVectorFunctionsWithDuplicateFunctions() { assertThat(r1Alias.name(), equalTo("r1")); var r1FieldAttr = as(r1Alias.child(), FieldAttribute.class); assertThat(r1FieldAttr.fieldName().string(), equalTo("dense_vector")); + assertThat(r1FieldAttr.name(), startsWith("$$dense_vector$DotProduct")); var r1Field = as(r1FieldAttr.field(), FunctionEsField.class); var r1Config = as(r1Field.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(r1Config.similarityFunction(), is(DotProduct.SIMILARITY_FUNCTION)); @@ -1468,6 +1478,7 @@ public void testVectorFunctionsWithDuplicateFunctions() { // Right side: CosineSimilarity field var r2CosineFieldAttr = as(r2Add.right(), FieldAttribute.class); assertThat(r2CosineFieldAttr.fieldName().string(), equalTo("dense_vector")); + assertThat(r2CosineFieldAttr.name(), startsWith("$$dense_vector$Cosine")); var r2CosineField = as(r2CosineFieldAttr.field(), FunctionEsField.class); var r2CosineConfig = as(r2CosineField.functionConfig(), DenseVectorFieldMapper.VectorSimilarityFunctionConfig.class); assertThat(r2CosineConfig.similarityFunction(), is(CosineSimilarity.SIMILARITY_FUNCTION)); From d01184b21a53fc6e2a41451fe2067b3c62091e9f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Oct 2025 08:55:20 -0400 Subject: [PATCH 08/18] Renam --- .../function/blockloader/BlockLoaderExpression.java | 8 ++++---- .../esql/expression/function/scalar/string/Length.java | 4 ++-- .../function/vector/VectorSimilarityFunction.java | 4 ++-- .../rules/logical/local/FuseExpressionToFieldLoad.java | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java index 92608a780e320..e819707a03ce1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java @@ -45,12 +45,12 @@ public interface BlockLoaderExpression { * "fusing" the expression into the load. Or null if the fusion isn't possible. */ @Nullable - FusedExpression tryFuse(SearchStats stats); + FusedBlockLoaderExpression tryFuse(SearchStats stats); /** - * Fused load configuration. + * Expression "fused" to the block loader. * @param field the field whose load we're fusing into - * @param config the fusion configuration + * @param config the expression's configuration */ - record FusedExpression(FieldAttribute field, BlockLoaderFunctionConfig config) {} + record FusedBlockLoaderExpression(FieldAttribute field, BlockLoaderFunctionConfig config) {} } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java index ad3d06868ad10..ed7266bcf0760 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java @@ -98,13 +98,13 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } @Override - public FusedExpression tryFuse(SearchStats stats) { + public FusedBlockLoaderExpression tryFuse(SearchStats stats) { if (field instanceof FieldAttribute f) { if (stats.hasDocValues(f.fieldName()) == false) { return null; } BlockLoaderWarnings warnings = new BlockLoaderWarnings(DriverContext.WarningsMode.COLLECT, source()); - return new FusedExpression(f, new BlockLoaderFunctionConfig.Named("LENGTH", warnings)); + return new FusedBlockLoaderExpression(f, new BlockLoaderFunctionConfig.Named("LENGTH", warnings)); } return null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java index 40c4469d8f6e2..5fd9fb2b50270 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java @@ -208,7 +208,7 @@ public void close() { } @Override - public final FusedExpression tryFuse(SearchStats stats) { + public final FusedBlockLoaderExpression tryFuse(SearchStats stats) { // Bail if we're not directly comparing a field with a literal. Literal literal; FieldAttribute field; @@ -233,7 +233,7 @@ public final FusedExpression tryFuse(SearchStats stats) { vectorArray[i] = ((Number) vectorList.get(i)).floatValue(); } - return new FusedExpression(field, new DenseVectorFieldMapper.VectorSimilarityFunctionConfig(getSimilarityFunction(), vectorArray)); + return new FusedBlockLoaderExpression(field, new DenseVectorFieldMapper.VectorSimilarityFunctionConfig(getSimilarityFunction(), vectorArray)); } interface VectorValueProvider extends Releasable { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java index ead0137326348..60fb1457c9ab0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java @@ -49,7 +49,7 @@ protected LogicalPlan rule(LogicalPlan plan, LocalLogicalOptimizerContext contex Map addedAttrs = new HashMap<>(); LogicalPlan transformedPlan = plan.transformExpressionsOnly(Expression.class, e -> { if (e instanceof BlockLoaderExpression ble) { - BlockLoaderExpression.FusedExpression fuse = ble.tryFuse(context.searchStats()); + BlockLoaderExpression.FusedBlockLoaderExpression fuse = ble.tryFuse(context.searchStats()); if (fuse != null) { return replaceFieldsForFieldTransformations(e, addedAttrs, fuse); } @@ -85,7 +85,7 @@ protected LogicalPlan rule(LogicalPlan plan, LocalLogicalOptimizerContext contex private static Expression replaceFieldsForFieldTransformations( Expression e, Map addedAttrs, - BlockLoaderExpression.FusedExpression fuse + BlockLoaderExpression.FusedBlockLoaderExpression fuse ) { // Change the similarity function to a reference of a transformation on the field FunctionEsField functionEsField = new FunctionEsField(fuse.field().field(), e.dataType(), fuse.config()); From d6897d81fb202d24252a9cf55567d7079e84b3cd Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 31 Oct 2025 13:02:11 +0000 Subject: [PATCH 09/18] [CI] Auto commit changes from spotless --- .../expression/function/vector/VectorSimilarityFunction.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java index 5fd9fb2b50270..48075b987f862 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java @@ -233,7 +233,10 @@ public final FusedBlockLoaderExpression tryFuse(SearchStats stats) { vectorArray[i] = ((Number) vectorList.get(i)).floatValue(); } - return new FusedBlockLoaderExpression(field, new DenseVectorFieldMapper.VectorSimilarityFunctionConfig(getSimilarityFunction(), vectorArray)); + return new FusedBlockLoaderExpression( + field, + new DenseVectorFieldMapper.VectorSimilarityFunctionConfig(getSimilarityFunction(), vectorArray) + ); } interface VectorValueProvider extends Releasable { From 538b72be29998ebaeaf61923e3859ca3243bb83f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Oct 2025 15:13:04 -0400 Subject: [PATCH 10/18] More tests --- .../LocalLogicalPlanOptimizerTests.java | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index e10d371c62cd9..e8a2b2f8b1ac2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -36,7 +36,11 @@ import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.ReferenceAttributeTests; +import org.elasticsearch.xpack.esql.expression.function.aggregate.Avg; +import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.function.aggregate.Min; +import org.elasticsearch.xpack.esql.expression.function.aggregate.Sum; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; @@ -118,6 +122,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; import static org.hamcrest.Matchers.startsWith; //@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug") @@ -1494,6 +1499,103 @@ public void testVectorFunctionsWithDuplicateFunctions() { assertTrue(esRelation.output().contains(r2CosineFieldAttr)); } + public void testLengthInEval() { + assumeTrue("requires similarity functions", EsqlCapabilities.Cap.VECTOR_SIMILARITY_FUNCTIONS_PUSHDOWN.isEnabled()); + String query = """ + FROM test + | EVAL l = LENGTH(last_name) + | KEEP l + """; + LogicalPlan plan = localPlan(plan(query, analyzer), TEST_SEARCH_STATS); + + var project = as(plan, EsqlProject.class); + assertThat(Expressions.names(project.projections()), contains("l")); + var eval = as(project.child(), Eval.class); + FieldAttribute lAttr = as(as(eval.fields().getFirst(), Alias.class).child(), FieldAttribute.class); + var limit = as(eval.child(), Limit.class); + var relation = as(limit.child(), EsRelation.class); + assertTrue(relation.output().contains(lAttr)); + } + + public void testLengthInWhere() { + assumeTrue("requires similarity functions", EsqlCapabilities.Cap.VECTOR_SIMILARITY_FUNCTIONS_PUSHDOWN.isEnabled()); + String query = """ + FROM test + | WHERE LENGTH(last_name) > 1 + """; + LogicalPlan plan = localPlan(plan(query, analyzer), TEST_SEARCH_STATS); + + var project = as(plan, EsqlProject.class); + var limit = as(project.child(), Limit.class); + var filter = as(limit.child(), Filter.class); + FieldAttribute lAttr = as(as(filter.condition(), GreaterThan.class).left(), FieldAttribute.class); + var relation = as(filter.child(), EsRelation.class); + assertTrue(relation.output().contains(lAttr)); + } + + public void testLengthInStats() { + assumeTrue("requires similarity functions", EsqlCapabilities.Cap.VECTOR_SIMILARITY_FUNCTIONS_PUSHDOWN.isEnabled()); + String query = """ + FROM test + | STATS l = SUM(LENGTH(last_name)) + """; + LogicalPlan plan = localPlan(plan(query, analyzer), TEST_SEARCH_STATS); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates(), hasSize(1)); + as(as(agg.aggregates().getFirst(), Alias.class).child(), Sum.class); + var eval = as(agg.child(), Eval.class); + FieldAttribute lAttr = as(as(eval.fields().getFirst(), Alias.class).child(), FieldAttribute.class); + var relation = as(eval.child(), EsRelation.class); + assertTrue(relation.output().contains(lAttr)); + } + + public void testLengthInWhereAndEval() { + assumeFalse("fix me", true); + assumeTrue("requires similarity functions", EsqlCapabilities.Cap.VECTOR_SIMILARITY_FUNCTIONS_PUSHDOWN.isEnabled()); + String query = """ + FROM test + | WHERE LENGTH(last_name) > 1 + | EVAL l = LENGTH(last_name) + """; + LogicalPlan plan = localPlan(plan(query, analyzer), TEST_SEARCH_STATS); + + var project = as(plan, EsqlProject.class); + var limit = as(project.child(), Limit.class); + var eval = as(limit.child(), Eval.class); + var filter = as(eval.child(), Filter.class); + FieldAttribute lAttr = as(as(filter.condition(), GreaterThan.class).left(), FieldAttribute.class); + var relation = as(filter.child(), EsRelation.class); + assertTrue(relation.output().contains(lAttr)); + } + + public void testLengthInStatsTwice() { + assumeTrue("requires similarity functions", EsqlCapabilities.Cap.VECTOR_SIMILARITY_FUNCTIONS_PUSHDOWN.isEnabled()); + String query = """ + FROM test + | STATS l = SUM(LENGTH(last_name)) + AVG(LENGTH(last_name)) + """; + LogicalPlan plan = localPlan(plan(query, analyzer), TEST_SEARCH_STATS); + + var project = as(plan, Project.class); + var eval1 = as(project.child(), Eval.class); + var limit = as(eval1.child(), Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(agg.aggregates(), hasSize(2)); + var sum = as(as(agg.aggregates().getFirst(), Alias.class).child(), Sum.class); + var count = as(as(agg.aggregates().get(1), Alias.class).child(), Count.class); + var eval2 = as(agg.child(), Eval.class); + assertThat(eval2.fields(), hasSize(1)); + Alias lAlias = as(eval2.fields().getFirst(), Alias.class); + FieldAttribute lAttr = as(lAlias.child(), FieldAttribute.class); + var relation = as(eval2.child(), EsRelation.class); + assertTrue(relation.output().contains(lAttr)); + + assertThat(as(sum.field(), ReferenceAttribute.class).id(), equalTo(lAlias.id())); + assertThat(as(count.field(), ReferenceAttribute.class).id(), equalTo(lAlias.id())); + } + private IsNotNull isNotNull(Expression field) { return new IsNotNull(EMPTY, field); } From e310d4bf50f1e349c4bcd88ea83ee8eeafa56fc4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 31 Oct 2025 19:58:17 +0000 Subject: [PATCH 11/18] [CI] Auto commit changes from spotless --- .../xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index e8a2b2f8b1ac2..2c7d00bcb283b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -36,8 +36,6 @@ import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; -import org.elasticsearch.xpack.esql.expression.function.ReferenceAttributeTests; -import org.elasticsearch.xpack.esql.expression.function.aggregate.Avg; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.function.aggregate.Min; import org.elasticsearch.xpack.esql.expression.function.aggregate.Sum; @@ -122,7 +120,6 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.Matchers.sameInstance; import static org.hamcrest.Matchers.startsWith; //@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug") From bb9bca22a7fdd3fcb47778da1f20cadd85c1a0c2 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 3 Nov 2025 10:24:02 -0500 Subject: [PATCH 12/18] Update x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java Co-authored-by: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> --- .../xpack/esql/expression/function/vector/L2Norm.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java index 74088b12cb86f..46c26ea7d8d11 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/L2Norm.java @@ -40,7 +40,7 @@ public float calculateSimilarity(float[] leftVector, float[] rightVector) { @Override public String toString() { - return "L1Norm"; + return "L2Norm"; } }; From d70bca9989be5fdcef16fa5f4fb3942673f04391 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 3 Nov 2025 10:41:49 -0500 Subject: [PATCH 13/18] Javadoc --- .../index/mapper/blockloader/docvalues/IntsBlockLoader.java | 2 +- .../xpack/esql/expression/function/BlockLoaderWarnings.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java index 9f0bc40f02845..50276cf1601eb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java @@ -31,7 +31,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { @Override public AllReader reader(LeafReaderContext context) throws IOException { - SortedNumericDocValues docValues = context.reader().getSortedNumericDocValues(fieldName); + SortedNumeriBlockLoaderWarningscDocValues docValues = context.reader().getSortedNumericDocValues(fieldName); if (docValues != null) { NumericDocValues singleton = DocValues.unwrapSingleton(docValues); if (singleton != null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/BlockLoaderWarnings.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/BlockLoaderWarnings.java index 6a6fa5413d6e8..278035dade015 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/BlockLoaderWarnings.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/BlockLoaderWarnings.java @@ -11,6 +11,12 @@ import org.elasticsearch.compute.operator.Warnings; import org.elasticsearch.xpack.esql.core.tree.Source; +/** + * Shim between the {@link org.elasticsearch.index.mapper.blockloader.Warnings} in server and + * our {@link Warnings}. Also adds laziness because our {@link Warnings} are a little expensive + * on creation and {@link org.elasticsearch.index.mapper.blockloader.Warnings} wants to be + * cheap to create. + */ public class BlockLoaderWarnings implements org.elasticsearch.index.mapper.blockloader.Warnings { private final DriverContext.WarningsMode warningsMode; private final Source source; From 39ae15a3f0440867442d8881742c954bedbdc500 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 3 Nov 2025 11:07:38 -0500 Subject: [PATCH 14/18] Rename --- .../function/blockloader/BlockLoaderExpression.java | 10 +++++++--- .../esql/expression/function/scalar/string/Length.java | 4 ++-- .../function/vector/VectorSimilarityFunction.java | 4 ++-- .../esql/optimizer/LocalLogicalPlanOptimizer.java | 4 ++-- ...oFieldLoad.java => PushExpressionsToFieldLoad.java} | 8 ++++---- 5 files changed, 17 insertions(+), 13 deletions(-) rename x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/{FuseExpressionToFieldLoad.java => PushExpressionsToFieldLoad.java} (93%) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java index e819707a03ce1..80abc925ee8a3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java @@ -15,7 +15,7 @@ import org.elasticsearch.xpack.esql.stats.SearchStats; /** - * {@link Expression} that can be "fused" into value loading. Most of the time + * {@link Expression} that can be "pushed" into value loading. Most of the time * we load values into {@link Block}s and then run the expressions on them, but * sometimes it's worth short-circuiting this process and running the expression * in the tight loop we use for loading: @@ -37,6 +37,10 @@ *
  • * {@code MV_COUNT(anything)} - counts are always integers. *
    • + *
  • + * {@code MV_MIN} and {@code MV_MAX} - loads much fewer data for + * multivalued fields. + *
    • *
*/ public interface BlockLoaderExpression { @@ -45,12 +49,12 @@ public interface BlockLoaderExpression { * "fusing" the expression into the load. Or null if the fusion isn't possible. */ @Nullable - FusedBlockLoaderExpression tryFuse(SearchStats stats); + PushedBlockLoaderExpression tryPushToFieldLoading(SearchStats stats); /** * Expression "fused" to the block loader. * @param field the field whose load we're fusing into * @param config the expression's configuration */ - record FusedBlockLoaderExpression(FieldAttribute field, BlockLoaderFunctionConfig config) {} + record PushedBlockLoaderExpression(FieldAttribute field, BlockLoaderFunctionConfig config) {} } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java index ed7266bcf0760..225a0006ad731 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java @@ -98,13 +98,13 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } @Override - public FusedBlockLoaderExpression tryFuse(SearchStats stats) { + public PushedBlockLoaderExpression tryPushToFieldLoading(SearchStats stats) { if (field instanceof FieldAttribute f) { if (stats.hasDocValues(f.fieldName()) == false) { return null; } BlockLoaderWarnings warnings = new BlockLoaderWarnings(DriverContext.WarningsMode.COLLECT, source()); - return new FusedBlockLoaderExpression(f, new BlockLoaderFunctionConfig.Named("LENGTH", warnings)); + return new PushedBlockLoaderExpression(f, new BlockLoaderFunctionConfig.Named("LENGTH", warnings)); } return null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java index 48075b987f862..0d3a7025eee26 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/vector/VectorSimilarityFunction.java @@ -208,7 +208,7 @@ public void close() { } @Override - public final FusedBlockLoaderExpression tryFuse(SearchStats stats) { + public final PushedBlockLoaderExpression tryPushToFieldLoading(SearchStats stats) { // Bail if we're not directly comparing a field with a literal. Literal literal; FieldAttribute field; @@ -233,7 +233,7 @@ public final FusedBlockLoaderExpression tryFuse(SearchStats stats) { vectorArray[i] = ((Number) vectorList.get(i)).floatValue(); } - return new FusedBlockLoaderExpression( + return new PushedBlockLoaderExpression( field, new DenseVectorFieldMapper.VectorSimilarityFunctionConfig(getSimilarityFunction(), vectorArray) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index fea93bd1ee1cd..3777021e5431a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -12,7 +12,7 @@ import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveRegexMatch; -import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.FuseExpressionToFieldLoad; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.PushExpressionsToFieldLoad; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.IgnoreNullMetrics; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferIsNotNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferNonNullAggConstraint; @@ -71,7 +71,7 @@ private static Batch localRewrites() { ) ); if (EsqlCapabilities.Cap.VECTOR_SIMILARITY_FUNCTIONS_PUSHDOWN.isEnabled()) { - rules.add(new FuseExpressionToFieldLoad()); + rules.add(new PushExpressionsToFieldLoad()); } return new Batch<>("Local rewrite", Limiter.ONCE, rules.toArray(Rule[]::new)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/PushExpressionsToFieldLoad.java similarity index 93% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/PushExpressionsToFieldLoad.java index 60fb1457c9ab0..7830c97331b2a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/FuseExpressionToFieldLoad.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/PushExpressionsToFieldLoad.java @@ -37,9 +37,9 @@ * the similarity function during value loading, when one side of the function is a literal. * It also adds the new field function attribute to the EsRelation output, and adds a projection after it to remove it from the output. */ -public class FuseExpressionToFieldLoad extends OptimizerRules.ParameterizedOptimizerRule { +public class PushExpressionsToFieldLoad extends OptimizerRules.ParameterizedOptimizerRule { - public FuseExpressionToFieldLoad() { + public PushExpressionsToFieldLoad() { super(OptimizerRules.TransformDirection.DOWN); } @@ -49,7 +49,7 @@ protected LogicalPlan rule(LogicalPlan plan, LocalLogicalOptimizerContext contex Map addedAttrs = new HashMap<>(); LogicalPlan transformedPlan = plan.transformExpressionsOnly(Expression.class, e -> { if (e instanceof BlockLoaderExpression ble) { - BlockLoaderExpression.FusedBlockLoaderExpression fuse = ble.tryFuse(context.searchStats()); + BlockLoaderExpression.PushedBlockLoaderExpression fuse = ble.tryPushToFieldLoading(context.searchStats()); if (fuse != null) { return replaceFieldsForFieldTransformations(e, addedAttrs, fuse); } @@ -85,7 +85,7 @@ protected LogicalPlan rule(LogicalPlan plan, LocalLogicalOptimizerContext contex private static Expression replaceFieldsForFieldTransformations( Expression e, Map addedAttrs, - BlockLoaderExpression.FusedBlockLoaderExpression fuse + BlockLoaderExpression.PushedBlockLoaderExpression fuse ) { // Change the similarity function to a reference of a transformation on the field FunctionEsField functionEsField = new FunctionEsField(fuse.field().field(), e.dataType(), fuse.config()); From 90005cae6ce195de812044635d90904654b74c6e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 3 Nov 2025 16:17:57 +0000 Subject: [PATCH 15/18] [CI] Auto commit changes from spotless --- .../xpack/esql/optimizer/LocalLogicalPlanOptimizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index 3777021e5431a..8af212ccdb6e6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -12,10 +12,10 @@ import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveRegexMatch; -import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.PushExpressionsToFieldLoad; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.IgnoreNullMetrics; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferIsNotNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferNonNullAggConstraint; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.PushExpressionsToFieldLoad; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceDateTruncBucketWithRoundTo; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceFieldWithConstantOrNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceTopNWithLimitAndSort; From c462d070adf6b2cbb7072c9c572cad953f58064d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 3 Nov 2025 11:30:52 -0500 Subject: [PATCH 16/18] fix --- .../index/mapper/blockloader/docvalues/IntsBlockLoader.java | 2 +- .../xpack/esql/optimizer/LocalLogicalPlanOptimizer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java index 50276cf1601eb..9f0bc40f02845 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java @@ -31,7 +31,7 @@ public Builder builder(BlockFactory factory, int expectedCount) { @Override public AllReader reader(LeafReaderContext context) throws IOException { - SortedNumeriBlockLoaderWarningscDocValues docValues = context.reader().getSortedNumericDocValues(fieldName); + SortedNumericDocValues docValues = context.reader().getSortedNumericDocValues(fieldName); if (docValues != null) { NumericDocValues singleton = DocValues.unwrapSingleton(docValues); if (singleton != null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index 3777021e5431a..8af212ccdb6e6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -12,10 +12,10 @@ import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveRegexMatch; -import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.PushExpressionsToFieldLoad; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.IgnoreNullMetrics; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferIsNotNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferNonNullAggConstraint; +import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.PushExpressionsToFieldLoad; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceDateTruncBucketWithRoundTo; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceFieldWithConstantOrNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.ReplaceTopNWithLimitAndSort; From bd2e10ebc04ea574c600db5e2388d025f24841f3 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 4 Nov 2025 13:34:45 -0500 Subject: [PATCH 17/18] Use exact info instead --- .../org/elasticsearch/xpack/esql/core/type/EsField.java | 7 ------- .../xpack/esql/core/type/FunctionEsField.java | 5 ++--- .../rules/physical/local/LucenePushdownPredicates.java | 3 --- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java index df62f7dc6b170..a6b8e432fc205 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java @@ -252,13 +252,6 @@ public EsField getExactField() { return this; } - /** - * Can this field be pushed if it is indexed? - */ - public boolean pushable() { - return true; - } - /** * Returns and {@link Exact} object with all the necessary info about the field: *
    diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java index 63e3de5b13b39..d8585fd4bfeac 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java @@ -72,8 +72,7 @@ public int hashCode() { } @Override - public boolean pushable() { - // These fields are *never* pushable to the lucene index. - return false; + public Exact getExactInfo() { + return new Exact(false, "merged with " + functionConfig.name()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index 24be3b71c8e91..aa9ea3b0e004b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -194,9 +194,6 @@ public boolean isIndexedAndHasDocValues(FieldAttribute attr) { // We still consider the value of isAggregatable here, because some fields like ScriptFieldTypes are always aggregatable // But this could hide issues with fields that are not indexed but are aggregatable // This is the original behaviour for ES|QL, but is it correct? - if (attr.field().pushable() == false) { - return false; - } return attr.field().isAggregatable() || stats.isIndexed(new FieldAttribute.FieldName(attr.name())) && stats.hasDocValues(new FieldAttribute.FieldName(attr.name())); From bd9c9abeebce234b7301d8f1fa03afbadcd9b14d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 4 Nov 2025 15:41:35 -0500 Subject: [PATCH 18/18] Integration test --- .../single_node/PushExpressionToLoadIT.java | 246 ++++++++++++++++++ .../esql/qa/single_node/PushQueriesIT.java | 2 +- .../blockloader/BlockLoaderExpression.java | 3 +- 3 files changed, 249 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushExpressionToLoadIT.java diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushExpressionToLoadIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushExpressionToLoadIT.java new file mode 100644 index 0000000000000..cc343ee37f27b --- /dev/null +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushExpressionToLoadIT.java @@ -0,0 +1,246 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.qa.single_node; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.test.MapMatcher; +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.esql.AssertWarnings; +import org.elasticsearch.xpack.esql.qa.rest.ProfileLogger; +import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase; +import org.hamcrest.Matcher; +import org.junit.ClassRule; +import org.junit.Rule; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.test.ListMatcher.matchesList; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.entityToMap; +import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.requestObjectBuilder; +import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.runEsql; +import static org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT.commonProfile; +import static org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT.fixTypesOnProfile; +import static org.hamcrest.Matchers.any; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.startsWith; + +/** + * Tests for pushing expressions into field loading. + */ +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class PushExpressionToLoadIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = Clusters.testCluster(spec -> spec.plugin("inference-service-test")); + + @Rule(order = Integer.MIN_VALUE) + public ProfileLogger profileLogger = new ProfileLogger(); + + public void testLength() throws IOException { + String value = "v".repeat(between(0, 256)); + test( + justType("keyword"), + b -> b.value(value), + "LENGTH(test)", + matchesList().item(value.length()), + "Utf8CodePointsFromOrds.SingletonOrdinals" + ); + } + + public void testVCosine() throws IOException { + test( + justType("dense_vector"), + b -> b.startArray().value(128).value(128).value(0).endArray(), + "V_COSINE(test, [0, 255, 255])", + matchesList().item(0.5), + "BlockDocValuesReader.FloatDenseVectorNormalizedValuesBlockReader" + ); + } + + private void test( + CheckedConsumer mapping, + CheckedConsumer value, + String functionInvocation, + Matcher expectedValue, + String expectedLoader + ) throws IOException { + indexValue(mapping, value); + RestEsqlTestCase.RequestObjectBuilder builder = requestObjectBuilder().query(""" + FROM test + | EVAL test =""" + " " + functionInvocation + """ + | STATS test = VALUES(test)"""); + /* + * TODO if you just do KEEP test then the load is in the data node reduce driver and not merged: + * \_ProjectExec[[test{f}#7]] + * \_FieldExtractExec[test{f}#7]<[],[]> + * \_EsQueryExec[test], indexMode[standard]] + * \_ExchangeSourceExec[[test{f}#7],false]}, {cluster_name=test-cluster, node_name=test-cluster-0, descrip + * \_ProjectExec[[test{r}#3]] + * \_EvalExec[[LENGTH(test{f}#7) AS test#3]] + * \_LimitExec[1000[INTEGER],50] + * \_ExchangeSourceExec[[test{f}#7],false]}], query={to + */ + builder.profile(true); + Map result = runEsql(builder, new AssertWarnings.NoWarnings(), profileLogger, RestEsqlTestCase.Mode.SYNC); + assertResultMap( + result, + getResultMatcher(result).entry( + "profile", + matchesMap() // + .entry("drivers", instanceOf(List.class)) + .entry("plans", instanceOf(List.class)) + .entry("planning", matchesMap().extraOk()) + .entry("query", matchesMap().extraOk()) + ), + matchesList().item(matchesMap().entry("name", "test").entry("type", any(String.class))), + matchesList().item(expectedValue) + ); + @SuppressWarnings("unchecked") + List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); + for (Map p : profiles) { + fixTypesOnProfile(p); + assertThat(p, commonProfile()); + List sig = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> operators = (List>) p.get("operators"); + for (Map o : operators) { + sig.add(checkOperatorProfile(o, expectedLoader)); + } + String description = p.get("description").toString(); + switch (description) { + case "data" -> assertMap( + sig, + matchesList().item("LuceneSourceOperator") + .item("ValuesSourceReaderOperator") // the real work is here, checkOperatorProfile checks the status + .item("EvalOperator") // this one just renames the field + .item("AggregationOperator") + .item("ExchangeSinkOperator") + ); + case "node_reduce" -> logger.info("node_reduce {}", sig); + case "final" -> logger.info("final {}", sig); + default -> throw new IllegalArgumentException("can't match " + description); + } + } + } + + private void indexValue(CheckedConsumer mapping, CheckedConsumer value) + throws IOException { + try { + // Delete the index if it has already been created. + client().performRequest(new Request("DELETE", "test")); + } catch (ResponseException e) { + if (e.getResponse().getStatusLine().getStatusCode() != 404) { + throw e; + } + } + + Request createIndex = new Request("PUT", "test"); + try (XContentBuilder config = JsonXContent.contentBuilder()) { + config.startObject(); + config.startObject("settings"); + { + config.startObject("index"); + config.field("number_of_shards", 1); + config.endObject(); + } + config.endObject(); + config.startObject("mappings"); + { + config.startObject("properties").startObject("test"); + mapping.accept(config); + config.endObject().endObject(); + } + config.endObject(); + + createIndex.setJsonEntity(Strings.toString(config.endObject())); + } + Response createResponse = client().performRequest(createIndex); + assertThat( + entityToMap(createResponse.getEntity(), XContentType.JSON), + matchesMap().entry("shards_acknowledged", true).entry("index", "test").entry("acknowledged", true) + ); + + Request bulk = new Request("POST", "/_bulk"); + bulk.addParameter("refresh", ""); + try (XContentBuilder doc = JsonXContent.contentBuilder()) { + doc.startObject(); + doc.field("test"); + value.accept(doc); + doc.endObject(); + bulk.setJsonEntity(""" + {"create":{"_index":"test"}} + """ + Strings.toString(doc) + "\n"); + } + Response bulkResponse = client().performRequest(bulk); + assertThat(entityToMap(bulkResponse.getEntity(), XContentType.JSON), matchesMap().entry("errors", false).extraOk()); + } + + private CheckedConsumer justType(String type) { + return b -> b.field("type", type); + } + + private static String checkOperatorProfile(Map o, String expectedLoader) { + String name = (String) o.get("operator"); + name = PushQueriesIT.TO_NAME.matcher(name).replaceAll(""); + if (name.equals("ValuesSourceReaderOperator")) { + MapMatcher expectedOp = matchesMap().entry("operator", startsWith(name)) + .entry( + "status", + matchesMap().entry("readers_built", matchesMap().entry("test:column_at_a_time:" + expectedLoader, 1)).extraOk() + ); + assertMap(o, expectedOp); + } + return name; + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected boolean preserveClusterUponCompletion() { + // Preserve the cluser to speed up the semantic_text tests + return true; + } + + private static boolean setupEmbeddings = false; + + private void setUpTextEmbeddingInferenceEndpoint() throws IOException { + setupEmbeddings = true; + Request request = new Request("PUT", "_inference/text_embedding/test"); + request.setJsonEntity(""" + { + "service": "text_embedding_test_service", + "service_settings": { + "model": "my_model", + "api_key": "abc64", + "dimensions": 128 + }, + "task_settings": { + } + } + """); + adminClient().performRequest(request); + } +} diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java index 6663880380ad3..0076ee4d9fb43 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -515,7 +515,7 @@ private String semanticTextWithKeyword() { }"""; } - private static final Pattern TO_NAME = Pattern.compile("\\[.+", Pattern.DOTALL); + static final Pattern TO_NAME = Pattern.compile("\\[.+", Pattern.DOTALL); private static String checkOperatorProfile(Map o, Matcher query) { String name = (String) o.get("operator"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java index 80abc925ee8a3..8888977e31211 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java @@ -22,7 +22,8 @@ *
      *
    • * {@code V_COSINE(vector, [constant_vector])} - vector is ~512 floats - * and V_COSINE is one double. + * and V_COSINE is one double. We can find the similarity without any + * copies if we combine. *
      • *
    • * {@code ST_CENTROID(shape)} - shapes can be quite large. Centroids