Skip to content

Commit 5e75a46

Browse files
nik9000nik9000
committed
Recheck
1 parent 06cb017 commit 5e75a46

File tree

14 files changed

+199
-21
lines changed

14 files changed

+199
-21
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -860,6 +860,14 @@ public Builder builder(BlockFactory factory, int expectedCount) {
860860
return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext));
861861
}
862862

863+
@Override
864+
public boolean supportsBlockLoaderConfig(BlockLoaderFunctionConfig config, FieldExtractPreference preference) {
865+
if (hasDocValues() && (preference != FieldExtractPreference.STORED || isSyntheticSourceEnabled())) {
866+
return config.name().equals("LENGTH");
867+
}
868+
return false;
869+
}
870+
863871
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
864872
var nullValueBytes = nullValue != null ? new BytesRef(nullValue) : null;
865873
return new FallbackSyntheticSourceBlockLoader.SingleValueReader<BytesRef>(nullValueBytes) {

‎server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -643,6 +643,10 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
643643
return null;
644644
}
645645

646+
public boolean supportsBlockLoaderConfig(BlockLoaderFunctionConfig config, FieldExtractPreference preference) {
647+
return false;
648+
}
649+
646650
public enum FieldExtractPreference {
647651
/**
648652
* Load the field from doc-values into a BlockLoader supporting doc-values.

‎server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/DenseVectorBlockLoader.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ protected void processCurrentVector(B builder) throws IOException {
178178

179179
@Override
180180
public String toString() {
181-
return "BlockDocValuesReader.FloatDenseVectorValuesBlockReader";
181+
return "FloatDenseVectorFromDocValues." + processor.name();
182182
}
183183
}
184184

@@ -216,7 +216,7 @@ protected void processCurrentVector(B builder) throws IOException {
216216

217217
@Override
218218
public String toString() {
219-
return "BlockDocValuesReader.FloatDenseVectorNormalizedValuesBlockReader";
219+
return "FloatDenseVectorFromDocValues.Normalized." + processor.name();
220220
}
221221
}
222222

@@ -237,7 +237,7 @@ protected void processCurrentVector(B builder) throws IOException {
237237

238238
@Override
239239
public String toString() {
240-
return "BlockDocValuesReader.ByteDenseVectorValuesBlockReader";
240+
return "ByteDenseVectorFromDocValues." + processor.name();
241241
}
242242
}
243243

@@ -255,7 +255,7 @@ protected void assertDimensions() {
255255

256256
@Override
257257
public String toString() {
258-
return "BlockDocValuesReader.BitDenseVectorValuesBlockReader";
258+
return "BitDenseVectorFromDocValues." + processor.name();
259259
}
260260
}
261261
}

‎server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/DenseVectorBlockLoaderProcessor.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ default void appendNull(B builder) {
4747
builder.appendNull();
4848
}
4949

50+
String name();
51+
5052
/**
5153
* Processor that appends raw float vectors to a FloatBuilder as multi values.
5254
*/
@@ -74,6 +76,11 @@ public void process(byte[] vector, BlockLoader.FloatBuilder builder) {
7476
}
7577
builder.endPositionEntry();
7678
}
79+
80+
@Override
81+
public String name() {
82+
return "Load";
83+
}
7784
}
7885

7986
/**
@@ -102,5 +109,10 @@ public void process(byte[] vector, BlockLoader.DoubleBuilder builder) {
102109
double similarity = config.similarityFunction().calculateSimilarity(vector, config.vectorAsBytes());
103110
builder.appendDouble(similarity);
104111
}
112+
113+
@Override
114+
public String name() {
115+
return config.similarityFunction().toString();
116+
}
105117
}
106118
}

‎server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2865,7 +2865,7 @@ public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) {
28652865
new DenseVectorBlockLoaderProcessor.DenseVectorLoaderProcessor()
28662866
);
28672867
} else if (functionConfig instanceof VectorSimilarityFunctionConfig similarityConfig) {
2868-
if (getElementType() == ElementType.BYTE) {
2868+
if (getElementType() == ElementType.BYTE || getElementType() == ElementType.BIT) {
28692869
similarityConfig = similarityConfig.forByteVector();
28702870
}
28712871
return new DenseVectorBlockLoader<>(
@@ -2900,6 +2900,19 @@ public BlockLoader blockLoader(MappedFieldType.BlockLoaderContext blContext) {
29002900
);
29012901
}
29022902

2903+
@Override
2904+
public boolean supportsBlockLoaderConfig(BlockLoaderFunctionConfig config, FieldExtractPreference preference) {
2905+
if (dims == null) {
2906+
// No data has been indexed yet
2907+
return true;
2908+
}
2909+
2910+
if (indexed) {
2911+
return config instanceof VectorSimilarityFunctionConfig;
2912+
}
2913+
return false;
2914+
}
2915+
29032916
private SourceValueFetcher sourceValueFetcher(Set<String> sourcePaths, IndexSettings indexSettings) {
29042917
return new SourceValueFetcher(sourcePaths, null, indexSettings.getIgnoredSourceFormat()) {
29052918
@Override

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/FunctionEsField.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,12 @@ public int hashCode() {
7373

7474
@Override
7575
public Exact getExactInfo() {
76+
/*
77+
* We force and "inexact" field info to prevent pushing
78+
* expressions like `WHERE LENGTH(kwd) > 2`. `LENGTH(kwd)`
79+
* is a `FunctionEsField` which *looks* pushable without
80+
* the inexact match.
81+
*/
7682
return new Exact(false, "merged with " + functionConfig.name());
7783
}
7884
}

‎x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushExpressionToLoadIT.java‎

Lines changed: 66 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@
2222
import org.elasticsearch.xcontent.XContentType;
2323
import org.elasticsearch.xcontent.json.JsonXContent;
2424
import org.elasticsearch.xpack.esql.AssertWarnings;
25+
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
2526
import org.elasticsearch.xpack.esql.qa.rest.ProfileLogger;
2627
import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase;
2728
import org.hamcrest.Matcher;
29+
import org.junit.Before;
2830
import org.junit.ClassRule;
2931
import org.junit.Rule;
3032

@@ -56,14 +58,51 @@ public class PushExpressionToLoadIT extends ESRestTestCase {
5658
@Rule(order = Integer.MIN_VALUE)
5759
public ProfileLogger profileLogger = new ProfileLogger();
5860

59-
public void testLength() throws IOException {
61+
@Before
62+
public void checkPushCapability() throws IOException {
63+
assumeTrue(
64+
"requires " + EsqlCapabilities.Cap.VECTOR_SIMILARITY_FUNCTIONS_PUSHDOWN.capabilityName(),
65+
clusterHasCapability(
66+
"POST",
67+
"_query",
68+
List.of(),
69+
List.of(EsqlCapabilities.Cap.VECTOR_SIMILARITY_FUNCTIONS_PUSHDOWN.capabilityName())
70+
).orElseGet(() -> false)
71+
);
72+
}
73+
74+
public void testLengthToKeyword() throws IOException {
6075
String value = "v".repeat(between(0, 256));
6176
test(
6277
justType("keyword"),
6378
b -> b.value(value),
6479
"LENGTH(test)",
6580
matchesList().item(value.length()),
66-
"Utf8CodePointsFromOrds.SingletonOrdinals"
81+
matchesMap().entry("test:column_at_a_time:Utf8CodePointsFromOrds.Singleton", 1)
82+
);
83+
}
84+
85+
public void testLengthNotPushedToWildcard() throws IOException {
86+
String value = "v".repeat(between(0, 256));
87+
test(
88+
justType("wildcard"),
89+
b -> b.value(value),
90+
"LENGTH(test)",
91+
matchesList().item(value.length()),
92+
matchesMap().entry("test:column_at_a_time:BlockDocValuesReader.BytesCustom", 1)
93+
);
94+
}
95+
96+
public void testLengthNotPushedToText() throws IOException {
97+
String value = "v".repeat(between(0, 256));
98+
test(
99+
justType("text"),
100+
b -> b.value(value),
101+
"LENGTH(test)",
102+
matchesList().item(value.length()),
103+
matchesMap().entry("test:column_at_a_time:null", 1)
104+
.entry("stored_fields[requires_source:true, fields:0, sequential: false]", 1)
105+
.entry("test:row_stride:BlockSourceReader.Bytes", 1)
67106
);
68107
}
69108

@@ -73,7 +112,27 @@ public void testVCosine() throws IOException {
73112
b -> b.startArray().value(128).value(128).value(0).endArray(),
74113
"V_COSINE(test, [0, 255, 255])",
75114
matchesList().item(0.5),
76-
"BlockDocValuesReader.FloatDenseVectorNormalizedValuesBlockReader"
115+
matchesMap().entry("test:column_at_a_time:FloatDenseVectorFromDocValues.Normalized.Cosine", 1)
116+
);
117+
}
118+
119+
public void testVHammingToByte() throws IOException {
120+
test(
121+
b -> b.field("type", "dense_vector").field("element_type", "byte"),
122+
b -> b.startArray().value(100).value(100).value(0).endArray(),
123+
"V_HAMMING(test, [0, 100, 100])",
124+
matchesList().item(6.0),
125+
matchesMap().entry("test:column_at_a_time:ByteDenseVectorFromDocValues.Hamming", 1)
126+
);
127+
}
128+
129+
public void testVHammingToBit() throws IOException {
130+
test(
131+
b -> b.field("type", "dense_vector").field("element_type", "bit"),
132+
b -> b.startArray().value(100).value(100).value(0).endArray(),
133+
"V_HAMMING(test, [0, 100, 100])",
134+
matchesList().item(6.0),
135+
matchesMap().entry("test:column_at_a_time:BitDenseVectorFromDocValues.Hamming", 1)
77136
);
78137
}
79138

@@ -82,7 +141,7 @@ private void test(
82141
CheckedConsumer<XContentBuilder, IOException> value,
83142
String functionInvocation,
84143
Matcher<?> expectedValue,
85-
String expectedLoader
144+
MapMatcher expectedLoaders
86145
) throws IOException {
87146
indexValue(mapping, value);
88147
RestEsqlTestCase.RequestObjectBuilder builder = requestObjectBuilder().query("""
@@ -124,7 +183,7 @@ private void test(
124183
@SuppressWarnings("unchecked")
125184
List<Map<String, Object>> operators = (List<Map<String, Object>>) p.get("operators");
126185
for (Map<String, Object> o : operators) {
127-
sig.add(checkOperatorProfile(o, expectedLoader));
186+
sig.add(checkOperatorProfile(o, expectedLoaders));
128187
}
129188
String description = p.get("description").toString();
130189
switch (description) {
@@ -199,15 +258,12 @@ private CheckedConsumer<XContentBuilder, IOException> justType(String type) {
199258
return b -> b.field("type", type);
200259
}
201260

202-
private static String checkOperatorProfile(Map<String, Object> o, String expectedLoader) {
261+
private static String checkOperatorProfile(Map<String, Object> o, MapMatcher expectedLoaders) {
203262
String name = (String) o.get("operator");
204263
name = PushQueriesIT.TO_NAME.matcher(name).replaceAll("");
205264
if (name.equals("ValuesSourceReaderOperator")) {
206265
MapMatcher expectedOp = matchesMap().entry("operator", startsWith(name))
207-
.entry(
208-
"status",
209-
matchesMap().entry("readers_built", matchesMap().entry("test:column_at_a_time:" + expectedLoader, 1)).extraOk()
210-
);
266+
.entry("status", matchesMap().entry("readers_built", expectedLoaders).extraOk());
211267
assertMap(o, expectedOp);
212268
}
213269
return name;

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,9 @@
5252
import org.elasticsearch.geometry.utils.Geohash;
5353
import org.elasticsearch.h3.H3;
5454
import org.elasticsearch.index.IndexMode;
55+
import org.elasticsearch.index.mapper.MappedFieldType;
5556
import org.elasticsearch.index.mapper.RoutingPathFields;
57+
import org.elasticsearch.index.mapper.blockloader.BlockLoaderFunctionConfig;
5658
import org.elasticsearch.license.XPackLicenseState;
5759
import org.elasticsearch.search.SearchService;
5860
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
@@ -348,6 +350,15 @@ public boolean hasExactSubfield(FieldName field) {
348350
return exists(field);
349351
}
350352

353+
@Override
354+
public boolean supportsLoaderConfig(
355+
FieldName name,
356+
BlockLoaderFunctionConfig config,
357+
MappedFieldType.FieldExtractPreference preference
358+
) {
359+
return true;
360+
}
361+
351362
@Override
352363
public long count() {
353364
return -1;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/blockloader/BlockLoaderExpression.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,15 @@ public interface BlockLoaderExpression {
4848
/**
4949
* The field and loading configuration that replaces this expression, effectively
5050
* "fusing" the expression into the load. Or null if the fusion isn't possible.
51+
* <p>
52+
* {@link SearchStats#supportsLoaderConfig} checks that the configuration is
53+
* supported by all field mappers. Callers to this method <strong>must</strong>
54+
* call that to confirm that configurations returned are supported. Implementations
55+
* of this method do not need to call it, though they <strong>may</strong> use
56+
* methods like {@link SearchStats#hasDocValues} and {@link SearchStats#isIndexed}
57+
* as preflight checks. They <strong>should</strong> use those methods if it is
58+
* expensive to build the {@link BlockLoaderFunctionConfig}.
59+
* </p>
5160
*/
5261
@Nullable
5362
PushedBlockLoaderExpression tryPushToFieldLoading(SearchStats stats);

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,9 +100,6 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
100100
@Override
101101
public PushedBlockLoaderExpression tryPushToFieldLoading(SearchStats stats) {
102102
if (field instanceof FieldAttribute f) {
103-
if (stats.hasDocValues(f.fieldName()) == false) {
104-
return null;
105-
}
106103
BlockLoaderWarnings warnings = new BlockLoaderWarnings(DriverContext.WarningsMode.COLLECT, source());
107104
return new PushedBlockLoaderExpression(f, new BlockLoaderFunctionConfig.Named("LENGTH", warnings));
108105
}

0 commit comments

Comments
 (0)