diff --git a/docs/changelog/128519.yaml b/docs/changelog/128519.yaml new file mode 100644 index 0000000000000..20352e7102e98 --- /dev/null +++ b/docs/changelog/128519.yaml @@ -0,0 +1,5 @@ +pr: 128519 +summary: Add support for LOOKUP JOIN on aliases +area: ES|QL +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index bc76773b86b97..e5aaaa3534302 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -276,6 +276,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_LIMIT_ROW_SIZE = def(9_085_0_00); public static final TransportVersion ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY = def(9_086_0_00); public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES = def(9_087_0_00); + public static final TransportVersion JOIN_ON_ALIASES = def(9_088_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 2ee2446ea1e9f..51a99c3d47c4f 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -133,6 +133,7 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.replaceValueInMatch("Size", 49, "Test flamegraph from test-events") task.skipTest("esql/90_non_indexed/fetch", "Temporary until backported") task.skipTest("esql/63_enrich_int_range/Invalid age as double", "TODO: require disable allow_partial_results") + task.skipTest("esql/191_lookup_join_on_datastreams/data streams not supported in LOOKUP JOIN", "Added support for aliases in JOINs") }) tasks.named('yamlRestCompatTest').configure { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java index d5e7656c637b4..14cfd09ccb21a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java @@ -34,6 +34,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.search.internal.AliasFilter; import java.io.IOException; import java.io.UncheckedIOException; @@ -46,6 +47,7 @@ */ public abstract class QueryList { protected final SearchExecutionContext searchExecutionContext; + protected final AliasFilter aliasFilter; protected final MappedFieldType field; protected final Block block; @Nullable @@ -54,10 +56,12 @@ public abstract class QueryList { protected QueryList( MappedFieldType field, SearchExecutionContext searchExecutionContext, + AliasFilter aliasFilter, Block block, OnlySingleValueParams onlySingleValueParams ) { this.searchExecutionContext = searchExecutionContext; + this.aliasFilter = aliasFilter; this.field = field; this.block = block; this.onlySingleValueParams = onlySingleValueParams; @@ -74,7 +78,7 @@ int getPositionCount() { * Returns a copy of this query list that only returns queries for single-valued positions. * That is, it returns `null` queries for either multivalued or null positions. *

- * Whenever a multi-value position is encountered, whether in the input block or in the queried index, a warning is emitted. + * Whenever a multi-value position is encountered, whether in the input block or in the queried index, a warning is emitted. *

*/ public abstract QueryList onlySingleValues(Warnings warnings, String multiValueWarningMessage); @@ -93,6 +97,17 @@ final Query getQuery(int position) { Query query = doGetQuery(position, firstValueIndex, valueCount); + if (aliasFilter != null && aliasFilter != AliasFilter.EMPTY) { + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(query, BooleanClause.Occur.FILTER); + try { + builder.add(aliasFilter.getQueryBuilder().toQuery(searchExecutionContext), BooleanClause.Occur.FILTER); + query = builder.build(); + } catch (IOException e) { + throw new UncheckedIOException("Error while building query for alias filter", e); + } + } + if (onlySingleValueParams != null) { query = wrapSingleValueQuery(query); } @@ -138,7 +153,12 @@ private Query wrapSingleValueQuery(Query query) { * using only the {@link ElementType} of the {@link Block} to determine the * query. */ - public static QueryList rawTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block) { + public static QueryList rawTermQueryList( + MappedFieldType field, + SearchExecutionContext searchExecutionContext, + AliasFilter aliasFilter, + Block block + ) { IntFunction blockToJavaObject = switch (block.elementType()) { case BOOLEAN -> { BooleanBlock booleanBlock = (BooleanBlock) block; @@ -170,17 +190,22 @@ public static QueryList rawTermQueryList(MappedFieldType field, SearchExecutionC case AGGREGATE_METRIC_DOUBLE -> throw new IllegalArgumentException("can't read values from [aggregate metric double] block"); case UNKNOWN -> throw new IllegalArgumentException("can't read values from [" + block + "]"); }; - return new TermQueryList(field, searchExecutionContext, block, null, blockToJavaObject); + return new TermQueryList(field, searchExecutionContext, aliasFilter, block, null, blockToJavaObject); } /** * Returns a list of term queries for the given field and the input block of * {@code ip} field values. */ - public static QueryList ipTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, BytesRefBlock block) { + public static QueryList ipTermQueryList( + MappedFieldType field, + SearchExecutionContext searchExecutionContext, + AliasFilter aliasFilter, + BytesRefBlock block + ) { BytesRef scratch = new BytesRef(); byte[] ipBytes = new byte[InetAddressPoint.BYTES]; - return new TermQueryList(field, searchExecutionContext, block, null, offset -> { + return new TermQueryList(field, searchExecutionContext, aliasFilter, block, null, offset -> { final var bytes = block.getBytesRef(offset, scratch); if (ipBytes.length != bytes.length) { // Lucene only support 16-byte IP addresses, even IPv4 is encoded in 16 bytes @@ -195,10 +220,16 @@ public static QueryList ipTermQueryList(MappedFieldType field, SearchExecutionCo * Returns a list of term queries for the given field and the input block of * {@code date} field values. */ - public static QueryList dateTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, LongBlock block) { + public static QueryList dateTermQueryList( + MappedFieldType field, + SearchExecutionContext searchExecutionContext, + AliasFilter aliasFilter, + LongBlock block + ) { return new TermQueryList( field, searchExecutionContext, + aliasFilter, block, null, field instanceof RangeFieldMapper.RangeFieldType rangeFieldType @@ -210,8 +241,13 @@ public static QueryList dateTermQueryList(MappedFieldType field, SearchExecution /** * Returns a list of geo_shape queries for the given field and the input block. */ - public static QueryList geoShapeQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block) { - return new GeoShapeQueryList(field, searchExecutionContext, block, null); + public static QueryList geoShapeQueryList( + MappedFieldType field, + SearchExecutionContext searchExecutionContext, + AliasFilter aliasFilter, + Block block + ) { + return new GeoShapeQueryList(field, searchExecutionContext, aliasFilter, block, null); } private static class TermQueryList extends QueryList { @@ -220,11 +256,12 @@ private static class TermQueryList extends QueryList { private TermQueryList( MappedFieldType field, SearchExecutionContext searchExecutionContext, + AliasFilter aliasFilter, Block block, OnlySingleValueParams onlySingleValueParams, IntFunction blockValueReader ) { - super(field, searchExecutionContext, block, onlySingleValueParams); + super(field, searchExecutionContext, aliasFilter, block, onlySingleValueParams); this.blockValueReader = blockValueReader; } @@ -233,6 +270,7 @@ public TermQueryList onlySingleValues(Warnings warnings, String multiValueWarnin return new TermQueryList( field, searchExecutionContext, + aliasFilter, block, new OnlySingleValueParams(warnings, multiValueWarningMessage), blockValueReader @@ -264,10 +302,11 @@ private static class GeoShapeQueryList extends QueryList { private GeoShapeQueryList( MappedFieldType field, SearchExecutionContext searchExecutionContext, + AliasFilter aliasFilter, Block block, OnlySingleValueParams onlySingleValueParams ) { - super(field, searchExecutionContext, block, onlySingleValueParams); + super(field, searchExecutionContext, aliasFilter, block, onlySingleValueParams); this.blockValueReader = blockToGeometry(block); this.shapeQuery = shapeQuery(); @@ -278,6 +317,7 @@ public GeoShapeQueryList onlySingleValues(Warnings warnings, String multiValueWa return new GeoShapeQueryList( field, searchExecutionContext, + aliasFilter, block, new OnlySingleValueParams(warnings, multiValueWarningMessage) ); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/EnrichQuerySourceOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/EnrichQuerySourceOperatorTests.java index df0a31965055e..2aadb81a8b086 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/EnrichQuerySourceOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/lookup/EnrichQuerySourceOperatorTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; @@ -83,7 +84,7 @@ public void testQueries() throws Exception { var inputTerms = makeTermsBlock(List.of(List.of("b2"), List.of("c1", "a2"), List.of("z2"), List.of(), List.of("a3"), List.of())) ) { MappedFieldType uidField = new KeywordFieldMapper.KeywordFieldType("uid"); - QueryList queryList = QueryList.rawTermQueryList(uidField, directoryData.searchExecutionContext, inputTerms); + QueryList queryList = QueryList.rawTermQueryList(uidField, directoryData.searchExecutionContext, AliasFilter.EMPTY, inputTerms); assertThat(queryList.getPositionCount(), equalTo(6)); assertThat(queryList.getQuery(0), equalTo(new TermQuery(new Term("uid", new BytesRef("b2"))))); assertThat(queryList.getQuery(1), equalTo(new TermInSetQuery("uid", List.of(new BytesRef("c1"), new BytesRef("a2"))))); @@ -153,7 +154,12 @@ public void testRandomMatchQueries() throws Exception { }).toList(); try (var directoryData = makeDirectoryWith(directoryTermsList); var inputTerms = makeTermsBlock(inputTermsList)) { - var queryList = QueryList.rawTermQueryList(directoryData.field, directoryData.searchExecutionContext, inputTerms); + var queryList = QueryList.rawTermQueryList( + directoryData.field, + directoryData.searchExecutionContext, + AliasFilter.EMPTY, + inputTerms + ); int maxPageSize = between(1, 256); EnrichQuerySourceOperator queryOperator = new EnrichQuerySourceOperator( blockFactory, @@ -190,8 +196,12 @@ public void testQueries_OnlySingleValues() throws Exception { List.of(List.of("b2"), List.of("c1", "a2"), List.of("z2"), List.of(), List.of("a3"), List.of("a3", "a2", "z2", "xx")) ) ) { - QueryList queryList = QueryList.rawTermQueryList(directoryData.field, directoryData.searchExecutionContext, inputTerms) - .onlySingleValues(warnings(), "multi-value found"); + QueryList queryList = QueryList.rawTermQueryList( + directoryData.field, + directoryData.searchExecutionContext, + AliasFilter.EMPTY, + inputTerms + ).onlySingleValues(warnings(), "multi-value found"); // pos -> terms -> docs // ----------------------------- // 0 -> [b2] -> [] diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java index 0fa29cb596e35..759b65f7463a2 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java @@ -42,7 +42,9 @@ import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; public class EsqlSecurityIT extends ESRestTestCase { @ClassRule @@ -59,10 +61,14 @@ public class EsqlSecurityIT extends ESRestTestCase { .user("user5", "x-pack-test-password", "user5", false) .user("fls_user", "x-pack-test-password", "fls_user", false) .user("fls_user2", "x-pack-test-password", "fls_user2", false) + .user("fls_user2_alias", "x-pack-test-password", "fls_user2_alias", false) .user("fls_user3", "x-pack-test-password", "fls_user3", false) + .user("fls_user3_alias", "x-pack-test-password", "fls_user3_alias", false) .user("fls_user4_1", "x-pack-test-password", "fls_user4_1", false) + .user("fls_user4_1_alias", "x-pack-test-password", "fls_user4_1_alias", false) .user("dls_user", "x-pack-test-password", "dls_user", false) .user("metadata1_read2", "x-pack-test-password", "metadata1_read2", false) + .user("metadata1_alias_read2", "x-pack-test-password", "metadata1_alias_read2", false) .user("alias_user1", "x-pack-test-password", "alias_user1", false) .user("alias_user2", "x-pack-test-password", "alias_user2", false) .user("logs_foo_all", "x-pack-test-password", "logs_foo_all", false) @@ -159,6 +165,12 @@ public void indexDocuments() throws IOException { } } }, + { + "add": { + "alias": "lookup-second-alias", + "index": "lookup-user2" + } + }, { "add": { "alias": "second-alias", @@ -196,6 +208,17 @@ private void createMultiRoleUsers() throws IOException { } """); assertOK(client().performRequest(request)); + + request = new Request("POST", "_security/user/fls_user4_alias"); + request.setJsonEntity(""" + { + "password" : "x-pack-test-password", + "roles" : [ "fls_user4_1_alias", "fls_user4_2_alias" ], + "full_name" : "Test Role", + "email" : "test.role@example.com" + } + """); + assertOK(client().performRequest(request)); } protected MapMatcher responseMatcher(Map result) { @@ -586,22 +609,76 @@ public void testLookupJoinIndexAllowed() throws Exception { ); assertThat(respMap.get("values"), equalTo(List.of(List.of(40.0, "sales")))); - // Aliases are not allowed in LOOKUP JOIN - var resp2 = expectThrows( + // user is not allowed to use the alias (but is allowed to use the index) + expectThrows( ResponseException.class, - () -> runESQLCommand("alias_user1", "ROW x = 31.0 | EVAL value = x | LOOKUP JOIN lookup-first-alias ON value | KEEP x, org") + () -> runESQLCommand( + "metadata1_read2", + "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-second-alias ON value | KEEP x, org" + ) ); - assertThat(resp2.getMessage(), containsString("Aliases and index patterns are not allowed for LOOKUP JOIN [lookup-first-alias]")); - assertThat(resp2.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST)); - - // Aliases are not allowed in LOOKUP JOIN, regardless of alias filters - resp2 = expectThrows( + // user is not allowed to use the index (but is allowed to use the alias) + expectThrows( ResponseException.class, - () -> runESQLCommand("alias_user1", "ROW x = 123.0 | EVAL value = x | LOOKUP JOIN lookup-first-alias ON value | KEEP x, org") + () -> runESQLCommand("metadata1_alias_read2", "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-user2 ON value | KEEP x, org") + ); + + // user has permission on the alias, and can read the key + resp = runESQLCommand( + "metadata1_alias_read2", + "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-second-alias ON value | KEEP x, org" + ); + assertOK(resp); + respMap = entityAsMap(resp); + assertThat( + respMap.get("columns"), + equalTo(List.of(Map.of("name", "x", "type", "double"), Map.of("name", "org", "type", "keyword"))) ); - assertThat(resp2.getMessage(), containsString("Aliases and index patterns are not allowed for LOOKUP JOIN [lookup-first-alias]")); - assertThat(resp2.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST)); + assertThat(respMap.get("values"), equalTo(List.of(List.of(40.0, "sales")))); + + // user has permission on the alias, but can't read the key (doc level security at role level) + resp = runESQLCommand( + "metadata1_alias_read2", + "ROW x = 32.0 | EVAL value = x | LOOKUP JOIN lookup-second-alias ON value | KEEP x, org" + ); + assertOK(resp); + respMap = entityAsMap(resp); + assertThat( + respMap.get("columns"), + equalTo(List.of(Map.of("name", "x", "type", "double"), Map.of("name", "org", "type", "keyword"))) + ); + List values = (List) respMap.get("values"); + assertThat(values.size(), is(1)); + List row = (List) values.get(0); + assertThat(row.size(), is(2)); + assertThat(row.get(0), is(32.0)); + assertThat(row.get(1), is(nullValue())); + + // user has permission on the alias, the alias has a filter that doesn't allow to see the value + resp = runESQLCommand("alias_user1", "ROW x = 12.0 | EVAL value = x | LOOKUP JOIN lookup-first-alias ON value | KEEP x, org"); + assertOK(resp); + respMap = entityAsMap(resp); + assertThat( + respMap.get("columns"), + equalTo(List.of(Map.of("name", "x", "type", "double"), Map.of("name", "org", "type", "keyword"))) + ); + values = (List) respMap.get("values"); + assertThat(values.size(), is(1)); + row = (List) values.get(0); + assertThat(row.size(), is(2)); + assertThat(row.get(0), is(12.0)); + assertThat(row.get(1), is(nullValue())); + + // user has permission on the alias, the alias has a filter that allows to see the value + resp = runESQLCommand("alias_user1", "ROW x = 31.0 | EVAL value = x | LOOKUP JOIN lookup-first-alias ON value | KEEP x, org"); + assertOK(resp); + respMap = entityAsMap(resp); + assertThat( + respMap.get("columns"), + equalTo(List.of(Map.of("name", "x", "type", "double"), Map.of("name", "org", "type", "keyword"))) + ); + assertThat(respMap.get("values"), equalTo(List.of(List.of(31.0, "sales")))); } @SuppressWarnings("unchecked") @@ -712,6 +789,64 @@ public void testLookupJoinFieldLevelSecurity() throws Exception { assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST)); } + public void testLookupJoinFieldLevelSecurityOnAlias() throws Exception { + assumeTrue( + "Requires LOOKUP JOIN capability", + EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName())) + ); + + Response resp = runESQLCommand("fls_user2_alias", "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-second-alias ON value"); + assertOK(resp); + Map respMap = entityAsMap(resp); + assertThat( + respMap.get("columns"), + equalTo( + List.of( + Map.of("name", "x", "type", "double"), + Map.of("name", "value", "type", "double"), + Map.of("name", "org", "type", "keyword") + ) + ) + ); + + resp = runESQLCommand("fls_user3_alias", "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-second-alias ON value"); + assertOK(resp); + respMap = entityAsMap(resp); + assertThat( + respMap.get("columns"), + equalTo( + List.of( + Map.of("name", "x", "type", "double"), + Map.of("name", "value", "type", "double"), + Map.of("name", "org", "type", "keyword"), + Map.of("name", "other", "type", "keyword") + ) + ) + + ); + + resp = runESQLCommand("fls_user4_alias", "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-second-alias ON value"); + assertOK(resp); + respMap = entityAsMap(resp); + assertThat( + respMap.get("columns"), + equalTo( + List.of( + Map.of("name", "x", "type", "double"), + Map.of("name", "value", "type", "double"), + Map.of("name", "org", "type", "keyword") + ) + ) + ); + + ResponseException error = expectThrows( + ResponseException.class, + () -> runESQLCommand("fls_user4_1_alias", "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-second-alias ON value") + ); + assertThat(error.getMessage(), containsString("Unknown column [value] in right side of join")); + assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST)); + } + public void testLookupJoinIndexForbidden() throws Exception { assumeTrue( "Requires LOOKUP JOIN capability", diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml b/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml index f66d0b72962c1..f2a3d93cdc559 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml @@ -40,6 +40,17 @@ metadata1_read2: - names: [ 'index-user2', 'lookup-user2' ] privileges: [ 'read' ] +metadata1_alias_read2: + cluster: [] + indices: + - names: [ 'index-user1', 'lookup-first-alias' ] + privileges: [ 'view_index_metadata' ] + - names: [ 'index-user2' ] + privileges: [ 'read' ] + - names: [ 'lookup-second-alias' ] + privileges: [ 'read' ] + query: '{"match": {"org": "sales"}}' + alias_user1: cluster: [] indices: @@ -101,6 +112,14 @@ fls_user2: field_security: grant: [ "org", "value" ] +fls_user2_alias: + cluster: [] + indices: + - names: [ 'lookup-second-alias' ] + privileges: [ 'read' ] + field_security: + grant: [ "org", "value" ] + fls_user3: cluster: [] indices: @@ -109,6 +128,15 @@ fls_user3: field_security: grant: [ "org", "value", "other" ] +fls_user3_alias: + cluster: [] + indices: + - names: [ 'lookup-second-alias' ] + privileges: [ 'read' ] + field_security: + grant: [ "org", "value", "other" ] + + fls_user4_1: cluster: [] indices: @@ -117,6 +145,14 @@ fls_user4_1: field_security: grant: [ "org" ] +fls_user4_1_alias: + cluster: [] + indices: + - names: [ 'lookup-second-alias' ] + privileges: [ 'read' ] + field_security: + grant: [ "org" ] + fls_user4_2: cluster: [] indices: @@ -125,6 +161,14 @@ fls_user4_2: field_security: grant: [ "value" ] +fls_user4_2_alias: + cluster: [] + indices: + - names: [ 'lookup-second-alias' ] + privileges: [ 'read' ] + field_security: + grant: [ "value" ] + dls_user: cluster: [] indices: diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle index c0d1605af0753..321488c8c6c17 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle @@ -23,7 +23,7 @@ apply plugin: org.elasticsearch.gradle.internal.precommit.TestingConventionsPrec restResources { restApi { - include '_common', 'bulk', 'get', 'indices', 'esql', 'xpack', 'enrich', 'cluster', 'capabilities' + include '_common', 'bulk', 'get', 'indices', 'esql', 'xpack', 'enrich', 'cluster', 'capabilities', 'index' } restTests { includeXpack 'esql' diff --git a/x-pack/plugin/esql/qa/server/multi-node/build.gradle b/x-pack/plugin/esql/qa/server/multi-node/build.gradle index beb1efb6b890f..c0b91d1931db1 100644 --- a/x-pack/plugin/esql/qa/server/multi-node/build.gradle +++ b/x-pack/plugin/esql/qa/server/multi-node/build.gradle @@ -28,7 +28,7 @@ tasks.named('javaRestTest') { restResources { restApi { - include '_common', 'bulk', 'get', 'indices', 'esql', 'xpack', 'enrich', 'cluster', 'capabilities' + include '_common', 'bulk', 'get', 'indices', 'esql', 'xpack', 'enrich', 'cluster', 'capabilities', 'index' } restTests { includeXpack 'esql' diff --git a/x-pack/plugin/esql/qa/server/single-node/build.gradle b/x-pack/plugin/esql/qa/server/single-node/build.gradle index 03f2383ecfcd9..ff14956317a58 100644 --- a/x-pack/plugin/esql/qa/server/single-node/build.gradle +++ b/x-pack/plugin/esql/qa/server/single-node/build.gradle @@ -34,7 +34,7 @@ dependencies { restResources { restApi { - include '_common', 'bulk', 'get', 'indices', 'esql', 'xpack', 'enrich', 'cluster', 'capabilities' + include '_common', 'bulk', 'get', 'indices', 'esql', 'xpack', 'enrich', 'cluster', 'capabilities', 'index' } restTests { includeXpack 'esql' diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java index b3fcf74234aa2..e6b96e2e1267d 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java @@ -228,6 +228,7 @@ private void runLookup(DataType keyType, PopulateIndices populateIndices) throws ctx -> internalCluster().getInstance(TransportEsqlQueryAction.class, finalNodeWithShard).getLookupFromIndexService(), keyType, "lookup", + "lookup", "key", List.of(new Alias(Source.EMPTY, "l", new ReferenceAttribute(Source.EMPTY, "l", DataType.LONG))), Source.EMPTY diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 565a6519a831e..e1822e9ed51a1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1149,7 +1149,12 @@ public enum Cap { /** * Dense vector field type support */ - DENSE_VECTOR_FIELD_TYPE(EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG); + DENSE_VECTOR_FIELD_TYPE(EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG), + + /** + * Enable support for index aliases in lookup joins + */ + ENABLE_LOOKUP_JOIN_ON_ALIASES(JOIN_LOOKUP_V12.isEnabled()); private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java index 012e82ee10bd2..38ca44046b8a0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java @@ -14,6 +14,8 @@ import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; @@ -52,6 +54,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.SearchContext; @@ -122,8 +125,10 @@ public abstract class AbstractLookupService { private final String actionName; protected final ClusterService clusterService; + protected final IndicesService indicesService; private final LookupShardContextFactory lookupShardContextFactory; protected final TransportService transportService; + IndexNameExpressionResolver indexNameExpressionResolver; protected final Executor executor; private final BigArrays bigArrays; private final BlockFactory blockFactory; @@ -141,8 +146,10 @@ public abstract class AbstractLookupService QueryList.ipTermQueryList(field, searchExecutionContext, (BytesRefBlock) block); - case DATETIME -> QueryList.dateTermQueryList(field, searchExecutionContext, (LongBlock) block); - case null, default -> QueryList.rawTermQueryList(field, searchExecutionContext, block); + case IP -> QueryList.ipTermQueryList(field, searchExecutionContext, aliasFilter, (BytesRefBlock) block); + case DATETIME -> QueryList.dateTermQueryList(field, searchExecutionContext, aliasFilter, (LongBlock) block); + case null, default -> QueryList.rawTermQueryList(field, searchExecutionContext, aliasFilter, block); }; } @@ -265,6 +276,14 @@ private void doLookup(T request, CancellableTask task, ActionListener final List releasables = new ArrayList<>(6); boolean started = false; try { + + ProjectMetadata projMeta = clusterService.state().metadata().getProject(); + AliasFilter aliasFilter = indicesService.buildAliasFilter( + clusterService.state().projectState(), + request.shardId.getIndex().getName(), + indexNameExpressionResolver.resolveExpressions(projMeta, request.indexPattern) + ); + LookupShardContext shardContext = lookupShardContextFactory.create(request.shardId); releasables.add(shardContext.release); final LocalCircuitBreaker localBreaker = new LocalCircuitBreaker( @@ -310,7 +329,14 @@ private void doLookup(T request, CancellableTask task, ActionListener request.source.source().getColumnNumber(), request.source.text() ); - QueryList queryList = queryList(request, shardContext.executionContext, inputBlock, request.inputDataType, warnings); + QueryList queryList = queryList( + request, + shardContext.executionContext, + aliasFilter, + inputBlock, + request.inputDataType, + warnings + ); var queryOperator = new EnrichQuerySourceOperator( driverContext.blockFactory(), EnrichQuerySourceOperator.DEFAULT_MAX_PAGE_SIZE, @@ -464,6 +490,7 @@ public void messageReceived(T request, TransportChannel channel, Task task) { abstract static class Request { final String sessionId; final String index; + final String indexPattern; final DataType inputDataType; final Page inputPage; final List extractFields; @@ -472,6 +499,7 @@ abstract static class Request { Request( String sessionId, String index, + String indexPattern, DataType inputDataType, Page inputPage, List extractFields, @@ -479,6 +507,7 @@ abstract static class Request { ) { this.sessionId = sessionId; this.index = index; + this.indexPattern = indexPattern; this.inputDataType = inputDataType; this.inputPage = inputPage; this.extractFields = extractFields; @@ -489,6 +518,7 @@ abstract static class Request { abstract static class TransportRequest extends AbstractTransportRequest implements IndicesRequest { final String sessionId; final ShardId shardId; + final String indexPattern; /** * For mixed clusters with nodes <8.14, this will be null. */ @@ -504,6 +534,7 @@ abstract static class TransportRequest extends AbstractTransportRequest implemen TransportRequest( String sessionId, ShardId shardId, + String indexPattern, DataType inputDataType, Page inputPage, Page toRelease, @@ -512,6 +543,7 @@ abstract static class TransportRequest extends AbstractTransportRequest implemen ) { this.sessionId = sessionId; this.shardId = shardId; + this.indexPattern = indexPattern; this.inputDataType = inputDataType; this.inputPage = inputPage; this.toRelease = toRelease; @@ -521,7 +553,7 @@ abstract static class TransportRequest extends AbstractTransportRequest implemen @Override public final String[] indices() { - return new String[] { shardId.getIndexName() }; + return new String[] { indexPattern }; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 999bdef0427ea..00cdb7b753c69 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ContextPreservingActionListener; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; @@ -31,6 +32,8 @@ import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.TransportRequestOptions; @@ -67,16 +70,20 @@ public class EnrichLookupService extends AbstractLookupService termQueryList(fieldType, context, inputBlock, inputDataType); - case "geo_match" -> QueryList.geoShapeQueryList(fieldType, context, inputBlock); + case "match", "range" -> termQueryList(fieldType, context, aliasFilter, inputBlock, inputDataType); + case "geo_match" -> QueryList.geoShapeQueryList(fieldType, context, aliasFilter, inputBlock); default -> throw new EsqlIllegalArgumentException("illegal match type " + request.matchType); }; } @@ -170,7 +178,7 @@ public static class Request extends AbstractLookupService.Request { List extractFields, Source source ) { - super(sessionId, index, inputDataType, inputPage, extractFields, source); + super(sessionId, index, index, inputDataType, inputPage, extractFields, source); this.matchType = matchType; this.matchField = matchField; } @@ -191,7 +199,7 @@ protected static class TransportRequest extends AbstractLookupService.TransportR List extractFields, Source source ) { - super(sessionId, shardId, inputDataType, inputPage, toRelease, extractFields, source); + super(sessionId, shardId, shardId.getIndexName(), inputDataType, inputPage, toRelease, extractFields, source); this.matchType = matchType; this.matchField = matchField; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java index 1f1361876d645..b3a98c37ca37c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java @@ -43,6 +43,7 @@ public record Factory( int inputChannel, Function lookupService, DataType inputDataType, + String lookupIndexPattern, String lookupIndex, String matchField, List loadFields, @@ -73,6 +74,7 @@ public Operator get(DriverContext driverContext) { inputChannel, lookupService.apply(driverContext), inputDataType, + lookupIndexPattern, lookupIndex, matchField, loadFields, @@ -86,6 +88,7 @@ public Operator get(DriverContext driverContext) { private final CancellableTask parentTask; private final int inputChannel; private final DataType inputDataType; + private final String lookupIndexPattern; private final String lookupIndex; private final String matchField; private final List loadFields; @@ -108,6 +111,7 @@ public LookupFromIndexOperator( int inputChannel, LookupFromIndexService lookupService, DataType inputDataType, + String lookupIndexPattern, String lookupIndex, String matchField, List loadFields, @@ -119,6 +123,7 @@ public LookupFromIndexOperator( this.inputChannel = inputChannel; this.lookupService = lookupService; this.inputDataType = inputDataType; + this.lookupIndexPattern = lookupIndexPattern; this.lookupIndex = lookupIndex; this.matchField = matchField; this.loadFields = loadFields; @@ -132,6 +137,7 @@ protected void performAsync(Page inputPage, ActionListener listener LookupFromIndexService.Request request = new LookupFromIndexService.Request( sessionId, lookupIndex, + lookupIndexPattern, inputDataType, matchField, new Page(inputBlock), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java index a7d62ed5d5b65..1c7cfa846f1af 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.enrich; import org.elasticsearch.TransportVersions; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; @@ -23,8 +24,11 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.action.EsqlQueryAction; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -36,8 +40,6 @@ import java.util.List; import java.util.Objects; -import static java.lang.System.in; - /** * {@link LookupFromIndexService} performs lookup against a Lookup index for * a given input page. See {@link AbstractLookupService} for how it works @@ -48,16 +50,20 @@ public class LookupFromIndexService extends AbstractLookupService extractFields, Source source ) { - super(sessionId, index, inputDataType, inputPage, extractFields, source); + super(sessionId, index, indexPattern, inputDataType, inputPage, extractFields, source); this.matchField = matchField; } } @@ -126,6 +135,7 @@ protected static class TransportRequest extends AbstractLookupService.TransportR TransportRequest( String sessionId, ShardId shardId, + String indexPattern, DataType inputDataType, Page inputPage, Page toRelease, @@ -133,7 +143,7 @@ protected static class TransportRequest extends AbstractLookupService.TransportR String matchField, Source source ) { - super(sessionId, shardId, inputDataType, inputPage, toRelease, extractFields, source); + super(sessionId, shardId, indexPattern, inputDataType, inputPage, toRelease, extractFields, source); this.matchField = matchField; } @@ -141,6 +151,14 @@ static TransportRequest readFrom(StreamInput in, BlockFactory blockFactory) thro TaskId parentTaskId = TaskId.readFromStream(in); String sessionId = in.readString(); ShardId shardId = new ShardId(in); + + String indexPattern; + if (in.getTransportVersion().onOrAfter(TransportVersions.JOIN_ON_ALIASES)) { + indexPattern = in.readString(); + } else { + indexPattern = shardId.getIndexName(); + } + DataType inputDataType = DataType.fromTypeName(in.readString()); Page inputPage; try (BlockStreamInput bsi = new BlockStreamInput(in, blockFactory)) { @@ -162,6 +180,7 @@ static TransportRequest readFrom(StreamInput in, BlockFactory blockFactory) thro TransportRequest result = new TransportRequest( sessionId, shardId, + indexPattern, inputDataType, inputPage, inputPage, @@ -178,6 +197,13 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(sessionId); out.writeWriteable(shardId); + + if (out.getTransportVersion().onOrAfter(TransportVersions.JOIN_ON_ALIASES)) { + out.writeString(indexPattern); + } else if (indexPattern.equals(shardId.getIndexName()) == false) { + throw new EsqlIllegalArgumentException("Aliases and index patterns are not allowed for LOOKUP JOIN []", indexPattern); + } + out.writeString(inputDataType.typeName()); out.writeWriteable(inputPage); PlanStreamOutput planOut = new PlanStreamOutput(out, null); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java index 5f1f569e3671b..caa2491db8d38 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java @@ -102,11 +102,6 @@ public void postAnalysisVerification(Failures failures) { ) ); } - - // this check is crucial for security: ES|QL would use the concrete indices, so it would bypass the security on the alias - if (esr.concreteIndices().contains(esr.indexPattern()) == false) { - failures.add(fail(this, "Aliases and index patterns are not allowed for LOOKUP JOIN [{}]", esr.indexPattern())); - } }); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 3a91ac0513b2a..3e66a55b58d94 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -739,6 +739,7 @@ private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlan matchConfig.channel(), ctx -> lookupFromIndexService, matchConfig.type(), + localSourceExec.indexPattern(), indexName, matchConfig.fieldName(), join.addedFields().stream().map(f -> (NamedExpression) f).toList(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index ab96b0505477d..33146991609eb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -118,15 +118,19 @@ public TransportEsqlQueryAction( .fromSearchService(searchService); this.enrichLookupService = new EnrichLookupService( clusterService, + searchService.getIndicesService(), lookupLookupShardContextFactory, transportService, + indexNameExpressionResolver, bigArrays, blockFactoryProvider.blockFactory() ); this.lookupFromIndexService = new LookupFromIndexService( clusterService, + searchService.getIndicesService(), lookupLookupShardContextFactory, transportService, + indexNameExpressionResolver, bigArrays, blockFactoryProvider.blockFactory() ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java index 5a9ea4f09ea92..badc164f24722 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java @@ -15,6 +15,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; @@ -44,6 +45,8 @@ import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.CrankyCircuitBreakerService; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.TaskId; @@ -73,6 +76,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.matchesPattern; +import static org.mockito.Mockito.mock; public class LookupFromIndexOperatorTests extends OperatorTestCase { private static final int LOOKUP_SIZE = 1000; @@ -143,6 +147,7 @@ protected Operator.OperatorFactory simple(SimpleOptions options) { this::lookupService, inputDataType, lookupIndex, + lookupIndex, matchField, loadFields, Source.EMPTY @@ -174,6 +179,8 @@ private LookupFromIndexService lookupService(DriverContext mainContext) { .build(), ClusterSettings.createBuiltInClusterSettings() ); + IndicesService indicesService = mock(IndicesService.class); + IndexNameExpressionResolver indexNameExpressionResolver = TestIndexNameExpressionResolver.newInstance(); releasables.add(clusterService::stop); ClusterServiceUtils.setState(clusterService, ClusterStateCreationUtils.state("idx", 1, 1)); if (beCranky) { @@ -184,8 +191,10 @@ private LookupFromIndexService lookupService(DriverContext mainContext) { BlockFactory blockFactory = ctx.blockFactory(); return new LookupFromIndexService( clusterService, + indicesService, lookupShardContextFactory(), transportService(clusterService), + indexNameExpressionResolver, bigArrays, blockFactory ); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml index c735296ea6da3..fdd11840b7a27 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml @@ -6,7 +6,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [join_lookup_v12, join_lookup_skip_mv_warnings] + capabilities: [join_lookup_v12, enable_lookup_join_on_aliases] reason: "uses LOOKUP JOIN" - do: indices.create: @@ -187,8 +187,6 @@ non-lookup index: --- alias-repeated-alias: - - skip: - awaits_fix: "LOOKUP JOIN does not support index aliases for now" - do: esql.query: body: @@ -203,8 +201,6 @@ alias-repeated-alias: --- alias-repeated-index: - - skip: - awaits_fix: "LOOKUP JOIN does not support index aliases for now" - do: esql.query: body: @@ -219,8 +215,6 @@ alias-repeated-index: --- alias-pattern-multiple: - - skip: - awaits_fix: "LOOKUP JOIN does not support index aliases for now" - do: esql.query: body: @@ -228,12 +222,10 @@ alias-pattern-multiple: catch: "bad_request" - match: { error.type: "verification_exception" } - - contains: { error.reason: "Found 1 problem\nline 1:34: invalid [test-lookup-alias-pattern-multiple] resolution in lookup mode to [2] indices" } + - contains: { error.reason: "Found 1 problem\nline 1:34: invalid [test-lookup-alias-pattern-multiple] resolution in lookup mode to [4] indices" } --- alias-pattern-single: - - skip: - awaits_fix: "LOOKUP JOIN does not support index aliases for now" - do: esql.query: body: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/191_lookup_join_on_datastreams.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/191_lookup_join_on_datastreams.yml index 6f9b70b0d94f1..e98df81ba6f0a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/191_lookup_join_on_datastreams.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/191_lookup_join_on_datastreams.yml @@ -6,7 +6,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [lookup_join_no_aliases] + capabilities: [enable_lookup_join_on_aliases] reason: "uses LOOKUP JOIN" - do: @@ -30,6 +30,8 @@ setup: type: date x: type: keyword + y: + type: keyword - do: indices.put_index_template: @@ -40,29 +42,33 @@ setup: composed_of: [ "my_mappings", "my_settings" ] priority: 500 + - do: + indices.create_data_stream: + name: my_data_stream - do: - bulk: - index: "my_data_stream" - refresh: true + index: + index: my_data_stream body: - - { "index": { } } - - { "x": "foo", "y": "y1" } - - { "index": { } } - - { "x": "bar", "y": "y2" } - - + '@timestamp': '2020-12-12' + 'x': 'foo' + 'y': 'y1' + - do: + indices.refresh: + index: my_data_stream --- -"data streams not supported in LOOKUP JOIN": +"data streams supported in LOOKUP JOIN": - do: esql.query: body: - query: 'row x = "foo" | LOOKUP JOIN my_data_stream ON x' - catch: "bad_request" + query: 'ROW x = "foo" | LOOKUP JOIN my_data_stream ON x | KEEP x, y | LIMIT 1' - - match: { error.type: "verification_exception" } - - contains: { error.reason: "Found 1 problem\nline 1:17: Aliases and index patterns are not allowed for LOOKUP JOIN [my_data_stream]" } + - match: {columns.0.name: "x"} + - match: {columns.0.type: "keyword"} + - match: {columns.1.name: "y"} + - match: {columns.1.type: "keyword"} + - match: {values.0: ["foo", "y1"]}