Skip to content

Commit b616c2d

Browse files
luigidellaquilaluigidellaquila
authored
[8.19] ES|QL: Add support for LOOKUP JOIN on aliases (#128519) (#128833)
* ES|QL: Add support for LOOKUP JOIN on aliases (#128519) * Fix test
1 parent c27de5a commit b616c2d

File tree

22 files changed

+401
-74
lines changed

22 files changed

+401
-74
lines changed

‎docs/changelog/128519.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 128519
2+
summary: Add support for LOOKUP JOIN on aliases
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,7 @@ static TransportVersion def(int id) {
231231
public static final TransportVersion INFERENCE_CUSTOM_SERVICE_ADDED_8_19 = def(8_841_0_39);
232232
public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES_ADDED_8_19 = def(8_841_0_40);
233233
public static final TransportVersion DATA_STREAM_OPTIONS_API_REMOVE_INCLUDE_DEFAULTS_8_19 = def(8_841_0_41);
234+
public static final TransportVersion JOIN_ON_ALIASES_8_19 = def(8_841_0_42);
234235
/*
235236
* STOP! READ THIS FIRST! No, really,
236237
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

‎x-pack/plugin/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
227227
task.skipTest("esql/40_unsupported_types/unsupported", "TODO: support for subset of metric fields")
228228
task.skipTest("esql/40_unsupported_types/unsupported with sort", "TODO: support for subset of metric fields")
229229
task.skipTest("esql/63_enrich_int_range/Invalid age as double", "TODO: require disable allow_partial_results")
230+
task.skipTest("esql/191_lookup_join_on_datastreams/data streams not supported in LOOKUP JOIN", "Added support for aliases in JOINs")
230231
})
231232

232233

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java‎

Lines changed: 57 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,10 @@
3434
import org.elasticsearch.index.mapper.MappedFieldType;
3535
import org.elasticsearch.index.mapper.RangeFieldMapper;
3636
import org.elasticsearch.index.query.SearchExecutionContext;
37+
import org.elasticsearch.search.internal.AliasFilter;
3738

3839
import java.io.IOException;
40+
import java.io.UncheckedIOException;
3941
import java.util.ArrayList;
4042
import java.util.List;
4143
import java.util.function.IntFunction;
@@ -45,12 +47,20 @@
4547
*/
4648
public abstract class QueryList {
4749
protected final SearchExecutionContext searchExecutionContext;
50+
protected final AliasFilter aliasFilter;
4851
protected final MappedFieldType field;
4952
protected final Block block;
5053
protected final boolean onlySingleValues;
5154

52-
protected QueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block, boolean onlySingleValues) {
55+
protected QueryList(
56+
MappedFieldType field,
57+
SearchExecutionContext searchExecutionContext,
58+
AliasFilter aliasFilter,
59+
Block block,
60+
boolean onlySingleValues
61+
) {
5362
this.searchExecutionContext = searchExecutionContext;
63+
this.aliasFilter = aliasFilter;
5464
this.field = field;
5565
this.block = block;
5666
this.onlySingleValues = onlySingleValues;
@@ -78,6 +88,17 @@ final Query getQuery(int position) {
7888

7989
Query query = doGetQuery(position, firstValueIndex, valueCount);
8090

91+
if (aliasFilter != null && aliasFilter != AliasFilter.EMPTY) {
92+
BooleanQuery.Builder builder = new BooleanQuery.Builder();
93+
builder.add(query, BooleanClause.Occur.FILTER);
94+
try {
95+
builder.add(aliasFilter.getQueryBuilder().toQuery(searchExecutionContext), BooleanClause.Occur.FILTER);
96+
query = builder.build();
97+
} catch (IOException e) {
98+
throw new UncheckedIOException("Error while building query for alias filter", e);
99+
}
100+
}
101+
81102
if (onlySingleValues) {
82103
query = wrapSingleValueQuery(query);
83104
}
@@ -121,7 +142,12 @@ private Query wrapSingleValueQuery(Query query) {
121142
* using only the {@link ElementType} of the {@link Block} to determine the
122143
* query.
123144
*/
124-
public static QueryList rawTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block) {
145+
public static QueryList rawTermQueryList(
146+
MappedFieldType field,
147+
SearchExecutionContext searchExecutionContext,
148+
AliasFilter aliasFilter,
149+
Block block
150+
) {
125151
IntFunction<Object> blockToJavaObject = switch (block.elementType()) {
126152
case BOOLEAN -> {
127153
BooleanBlock booleanBlock = (BooleanBlock) block;
@@ -153,17 +179,22 @@ public static QueryList rawTermQueryList(MappedFieldType field, SearchExecutionC
153179
case AGGREGATE_METRIC_DOUBLE -> throw new IllegalArgumentException("can't read values from [aggregate metric double] block");
154180
case UNKNOWN -> throw new IllegalArgumentException("can't read values from [" + block + "]");
155181
};
156-
return new TermQueryList(field, searchExecutionContext, block, false, blockToJavaObject);
182+
return new TermQueryList(field, searchExecutionContext, aliasFilter, block, false, blockToJavaObject);
157183
}
158184

159185
/**
160186
* Returns a list of term queries for the given field and the input block of
161187
* {@code ip} field values.
162188
*/
163-
public static QueryList ipTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, BytesRefBlock block) {
189+
public static QueryList ipTermQueryList(
190+
MappedFieldType field,
191+
SearchExecutionContext searchExecutionContext,
192+
AliasFilter aliasFilter,
193+
BytesRefBlock block
194+
) {
164195
BytesRef scratch = new BytesRef();
165196
byte[] ipBytes = new byte[InetAddressPoint.BYTES];
166-
return new TermQueryList(field, searchExecutionContext, block, false, offset -> {
197+
return new TermQueryList(field, searchExecutionContext, aliasFilter, block, false, offset -> {
167198
final var bytes = block.getBytesRef(offset, scratch);
168199
if (ipBytes.length != bytes.length) {
169200
// Lucene only support 16-byte IP addresses, even IPv4 is encoded in 16 bytes
@@ -178,10 +209,16 @@ public static QueryList ipTermQueryList(MappedFieldType field, SearchExecutionCo
178209
* Returns a list of term queries for the given field and the input block of
179210
* {@code date} field values.
180211
*/
181-
public static QueryList dateTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, LongBlock block) {
212+
public static QueryList dateTermQueryList(
213+
MappedFieldType field,
214+
SearchExecutionContext searchExecutionContext,
215+
AliasFilter aliasFilter,
216+
LongBlock block
217+
) {
182218
return new TermQueryList(
183219
field,
184220
searchExecutionContext,
221+
aliasFilter,
185222
block,
186223
false,
187224
field instanceof RangeFieldMapper.RangeFieldType rangeFieldType
@@ -193,8 +230,14 @@ public static QueryList dateTermQueryList(MappedFieldType field, SearchExecution
193230
/**
194231
* Returns a list of geo_shape queries for the given field and the input block.
195232
*/
196-
public static QueryList geoShapeQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block) {
197-
return new GeoShapeQueryList(field, searchExecutionContext, block, false);
233+
234+
public static QueryList geoShapeQueryList(
235+
MappedFieldType field,
236+
SearchExecutionContext searchExecutionContext,
237+
AliasFilter aliasFilter,
238+
Block block
239+
) {
240+
return new GeoShapeQueryList(field, searchExecutionContext, aliasFilter, block, false);
198241
}
199242

200243
private static class TermQueryList extends QueryList {
@@ -203,17 +246,18 @@ private static class TermQueryList extends QueryList {
203246
private TermQueryList(
204247
MappedFieldType field,
205248
SearchExecutionContext searchExecutionContext,
249+
AliasFilter aliasFilter,
206250
Block block,
207251
boolean onlySingleValues,
208252
IntFunction<Object> blockValueReader
209253
) {
210-
super(field, searchExecutionContext, block, onlySingleValues);
254+
super(field, searchExecutionContext, aliasFilter, block, onlySingleValues);
211255
this.blockValueReader = blockValueReader;
212256
}
213257

214258
@Override
215259
public TermQueryList onlySingleValues() {
216-
return new TermQueryList(field, searchExecutionContext, block, true, blockValueReader);
260+
return new TermQueryList(field, searchExecutionContext, aliasFilter, block, true, blockValueReader);
217261
}
218262

219263
@Override
@@ -241,18 +285,19 @@ private static class GeoShapeQueryList extends QueryList {
241285
private GeoShapeQueryList(
242286
MappedFieldType field,
243287
SearchExecutionContext searchExecutionContext,
288+
AliasFilter aliasFilter,
244289
Block block,
245290
boolean onlySingleValues
246291
) {
247-
super(field, searchExecutionContext, block, onlySingleValues);
292+
super(field, searchExecutionContext, aliasFilter, block, onlySingleValues);
248293

249294
this.blockValueReader = blockToGeometry(block);
250295
this.shapeQuery = shapeQuery();
251296
}
252297

253298
@Override
254299
public GeoShapeQueryList onlySingleValues() {
255-
return new GeoShapeQueryList(field, searchExecutionContext, block, true);
300+
return new GeoShapeQueryList(field, searchExecutionContext, aliasFilter, block, true);
256301
}
257302

258303
@Override

‎x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/EnrichQuerySourceOperatorTests.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import org.elasticsearch.index.mapper.MappedFieldType;
4242
import org.elasticsearch.index.query.SearchExecutionContext;
4343
import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
44+
import org.elasticsearch.search.internal.AliasFilter;
4445
import org.elasticsearch.test.ESTestCase;
4546
import org.junit.After;
4647
import org.junit.Before;
@@ -83,7 +84,7 @@ public void testQueries() throws Exception {
8384
var inputTerms = makeTermsBlock(List.of(List.of("b2"), List.of("c1", "a2"), List.of("z2"), List.of(), List.of("a3"), List.of()))
8485
) {
8586
MappedFieldType uidField = new KeywordFieldMapper.KeywordFieldType("uid");
86-
QueryList queryList = QueryList.rawTermQueryList(uidField, directoryData.searchExecutionContext, inputTerms);
87+
QueryList queryList = QueryList.rawTermQueryList(uidField, directoryData.searchExecutionContext, AliasFilter.EMPTY, inputTerms);
8788
assertThat(queryList.getPositionCount(), equalTo(6));
8889
assertThat(queryList.getQuery(0), equalTo(new TermQuery(new Term("uid", new BytesRef("b2")))));
8990
assertThat(queryList.getQuery(1), equalTo(new TermInSetQuery("uid", List.of(new BytesRef("c1"), new BytesRef("a2")))));
@@ -154,7 +155,12 @@ public void testRandomMatchQueries() throws Exception {
154155
}).toList();
155156

156157
try (var directoryData = makeDirectoryWith(directoryTermsList); var inputTerms = makeTermsBlock(inputTermsList)) {
157-
var queryList = QueryList.rawTermQueryList(directoryData.field, directoryData.searchExecutionContext, inputTerms);
158+
var queryList = QueryList.rawTermQueryList(
159+
directoryData.field,
160+
directoryData.searchExecutionContext,
161+
AliasFilter.EMPTY,
162+
inputTerms
163+
);
158164
int maxPageSize = between(1, 256);
159165
var warnings = Warnings.createWarnings(DriverContext.WarningsMode.IGNORE, 0, 0, "test enrich");
160166
EnrichQuerySourceOperator queryOperator = new EnrichQuerySourceOperator(
@@ -192,8 +198,12 @@ public void testQueries_OnlySingleValues() throws Exception {
192198
List.of(List.of("b2"), List.of("c1", "a2"), List.of("z2"), List.of(), List.of("a3"), List.of("a3", "a2", "z2", "xx"))
193199
)
194200
) {
195-
QueryList queryList = QueryList.rawTermQueryList(directoryData.field, directoryData.searchExecutionContext, inputTerms)
196-
.onlySingleValues();
201+
QueryList queryList = QueryList.rawTermQueryList(
202+
directoryData.field,
203+
directoryData.searchExecutionContext,
204+
AliasFilter.EMPTY,
205+
inputTerms
206+
).onlySingleValues();
197207
// pos -> terms -> docs
198208
// -----------------------------
199209
// 0 -> [b2] -> []

0 commit comments

Comments
 (0)