From 1b99a4d8a5906be93505f4d5b356292f8bd1c154 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Thu, 16 Jan 2025 14:37:35 +0100 Subject: [PATCH 01/11] Disallow CCS with lookup join It is not possible to perform lookup joins with CC at the moment. --- .../xpack/esql/analysis/Analyzer.java | 23 +++++++++++++++++++ .../xpack/esql/analysis/AnalyzerTests.java | 18 +++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index a11b511cb83b7..a7115ac5412d5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -11,6 +11,7 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.index.IndexMode; import org.elasticsearch.logging.Logger; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.Column; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -653,6 +654,28 @@ private Join resolveLookupJoin(LookupJoin join) { return join; } + // joining with remote cluster is not supported yet + if (join.left() instanceof EsRelation esr) { + for (var index : esr.index().concreteIndices()) { + if (index.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR) != -1) { + return join.withConfig( + new JoinConfig( + type, + List.of( + new UnresolvedAttribute( + join.source(), + "unsupported", + "LOOKUP JOIN does not support joining with remote cluster indices [" + esr.index().name() + "]" + ) + ), + List.of(), + List.of() + ) + ); + } + } + } + JoinType coreJoin = using.coreJoin(); // verify the join type if (coreJoin != JoinTypes.LEFT) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 84af46a8cbbf0..fd2c701e9a029 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2250,6 +2250,24 @@ public void testLookupJoinIndexMode() { assertThat(e.getMessage(), containsString("1:25: invalid [test] resolution in lookup mode to an index in [standard] mode")); } + public void testLookupJoinRemoteClusterUnsupported() { + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + + var remoteIndexResolution = loadMapping("mapping-default.json", "remote:my-index"); + var lookupResolution = AnalyzerTestUtils.defaultLookupResolution(); + VerificationException e = expectThrows( + VerificationException.class, + () -> analyze( + "FROM remote:my-index | LOOKUP JOIN languages_lookup ON language_code", + AnalyzerTestUtils.analyzer(remoteIndexResolution, lookupResolution) + ) + ); + assertThat( + e.getMessage(), + containsString("1:24: LOOKUP JOIN does not support joining with remote cluster indices [remote:my-index]") + ); + } + public void testImplicitCasting() { var e = expectThrows(VerificationException.class, () -> analyze(""" from test | eval x = concat("2024", "-04", "-01") + 1 day From b8cf1b323bf1e64cec80e994ed5b1e9d94428d61 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Thu, 16 Jan 2025 14:45:16 +0100 Subject: [PATCH 02/11] ensure join target does not accept remote --- .../esql/parser/StatementParserTests.java | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index a6243d25ba579..ef61e680ce360 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -298,18 +298,18 @@ public void testStatsWithoutGroups() { ); } - public void testStatsWithoutAggs() throws Exception { + public void testStatsWithoutAggs() { assertEquals( new Aggregate(EMPTY, PROCESSING_CMD_INPUT, Aggregate.AggregateType.STANDARD, List.of(attribute("a")), List.of(attribute("a"))), processingCommand("stats by a") ); } - public void testStatsWithoutAggsOrGroup() throws Exception { + public void testStatsWithoutAggsOrGroup() { expectError("from text | stats", "At least one aggregation or grouping expression required in [stats]"); } - public void testAggsWithGroupKeyAsAgg() throws Exception { + public void testAggsWithGroupKeyAsAgg() { var queries = new String[] { """ row a = 1, b = 2 | stats a by a @@ -330,7 +330,7 @@ public void testAggsWithGroupKeyAsAgg() throws Exception { } } - public void testStatsWithGroupKeyAndAggFilter() throws Exception { + public void testStatsWithGroupKeyAndAggFilter() { var a = attribute("a"); var f = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a)); var filter = new Alias(EMPTY, "min(a) where a > 1", new FilteredExpression(EMPTY, f, new GreaterThan(EMPTY, a, integer(1)))); @@ -340,7 +340,7 @@ public void testStatsWithGroupKeyAndAggFilter() throws Exception { ); } - public void testStatsWithGroupKeyAndMixedAggAndFilter() throws Exception { + public void testStatsWithGroupKeyAndMixedAggAndFilter() { var a = attribute("a"); var min = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a)); var max = new UnresolvedFunction(EMPTY, "max", DEFAULT, List.of(a)); @@ -375,7 +375,7 @@ public void testStatsWithGroupKeyAndMixedAggAndFilter() throws Exception { ); } - public void testStatsWithoutGroupKeyMixedAggAndFilter() throws Exception { + public void testStatsWithoutGroupKeyMixedAggAndFilter() { var a = attribute("a"); var f = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a)); var filter = new Alias(EMPTY, "min(a) where a > 1", new FilteredExpression(EMPTY, f, new GreaterThan(EMPTY, a, integer(1)))); @@ -2058,41 +2058,41 @@ private void assertStringAsLookupIndexPattern(String string, String statement) { assertThat(tableName.fold(FoldContext.small()), equalTo(string)); } - public void testIdPatternUnquoted() throws Exception { + public void testIdPatternUnquoted() { var string = "regularString"; assertThat(breakIntoFragments(string), contains(string)); } - public void testIdPatternQuoted() throws Exception { + public void testIdPatternQuoted() { var string = "`escaped string`"; assertThat(breakIntoFragments(string), contains(string)); } - public void testIdPatternQuotedWithDoubleBackticks() throws Exception { + public void testIdPatternQuotedWithDoubleBackticks() { var string = "`escaped``string`"; assertThat(breakIntoFragments(string), contains(string)); } - public void testIdPatternUnquotedAndQuoted() throws Exception { + public void testIdPatternUnquotedAndQuoted() { var string = "this`is`a`mix`of`ids`"; assertThat(breakIntoFragments(string), contains("this", "`is`", "a", "`mix`", "of", "`ids`")); } - public void testIdPatternQuotedTraling() throws Exception { + public void testIdPatternQuotedTraling() { var string = "`foo`*"; assertThat(breakIntoFragments(string), contains("`foo`", "*")); } - public void testIdPatternWithDoubleQuotedStrings() throws Exception { + public void testIdPatternWithDoubleQuotedStrings() { var string = "`this``is`a`quoted `` string``with`backticks"; assertThat(breakIntoFragments(string), contains("`this``is`", "a", "`quoted `` string``with`", "backticks")); } - public void testSpaceNotAllowedInIdPattern() throws Exception { + public void testSpaceNotAllowedInIdPattern() { expectError("ROW a = 1| RENAME a AS this is `not okay`", "mismatched input 'is' expecting {, '|', ',', '.'}"); } - public void testSpaceNotAllowedInIdPatternKeep() throws Exception { + public void testSpaceNotAllowedInIdPatternKeep() { expectError("ROW a = 1, b = 1| KEEP a b", "extraneous input 'b'"); } @@ -2358,4 +2358,12 @@ public void testFailingMetadataWithSquareBrackets() { "line 1:11: mismatched input '[' expecting {, '|', ',', 'metadata'}" ); } + + public void testInvalidRemoteLookupJoin() { + // TODO ES-10559 this should be replaced with a proper error message once grammar allows indexPattern as joinTarget + expectError( + "FROM my-index | LOOKUP JOIN remote:languages_lookup ON language_code", + "line 1:35: token recognition error at: ':'" + ); + } } From b1a0280098e5d2b7d0801e45b07131925889d447 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 16 Jan 2025 13:53:46 +0000 Subject: [PATCH 03/11] [CI] Auto commit changes from spotless --- .../xpack/esql/parser/StatementParserTests.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index ef61e680ce360..b087713c5081c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -2361,9 +2361,6 @@ public void testFailingMetadataWithSquareBrackets() { public void testInvalidRemoteLookupJoin() { // TODO ES-10559 this should be replaced with a proper error message once grammar allows indexPattern as joinTarget - expectError( - "FROM my-index | LOOKUP JOIN remote:languages_lookup ON language_code", - "line 1:35: token recognition error at: ':'" - ); + expectError("FROM my-index | LOOKUP JOIN remote:languages_lookup ON language_code", "line 1:35: token recognition error at: ':'"); } } From 5f55608f33a0797b469e6d9ab7de00582c9d9912 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Tue, 21 Jan 2025 14:09:13 +0100 Subject: [PATCH 04/11] inc lookup version --- .../xpack/esql/EsqlSecurityIT.java | 4 +- .../esql/qa/mixed/MixedClusterEsqlSpecIT.java | 4 +- .../xpack/esql/ccq/MultiClusterSpecIT.java | 4 +- .../rest/RequestIndexFilteringTestCase.java | 2 +- .../src/main/resources/lookup-join.csv-spec | 142 +++++++++--------- .../xpack/esql/action/EsqlCapabilities.java | 2 +- .../elasticsearch/xpack/esql/CsvTests.java | 2 +- .../xpack/esql/analysis/AnalyzerTests.java | 10 +- .../xpack/esql/analysis/ParsingTests.java | 8 +- .../xpack/esql/analysis/VerifierTests.java | 2 +- .../optimizer/LogicalPlanOptimizerTests.java | 12 +- .../optimizer/PhysicalPlanOptimizerTests.java | 10 +- .../session/IndexResolverFieldNamesTests.java | 24 +-- .../test/esql/190_lookup_join.yml | 2 +- 14 files changed, 114 insertions(+), 114 deletions(-) diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java index 98e5799c8d3f2..f57e10dbcac3c 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java @@ -548,7 +548,7 @@ record Listen(long timestamp, String songId, double duration) { public void testLookupJoinIndexAllowed() throws Exception { assumeTrue( "Requires LOOKUP JOIN capability", - EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V11.capabilityName())) + EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName())) ); Response resp = runESQLCommand( @@ -587,7 +587,7 @@ public void testLookupJoinIndexAllowed() throws Exception { public void testLookupJoinIndexForbidden() throws Exception { assumeTrue( "Requires LOOKUP JOIN capability", - EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V11.capabilityName())) + EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName())) ); var resp = expectThrows( diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java index 3b5377c2768fb..790b12346bb14 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java @@ -20,7 +20,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V11; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12; public class MixedClusterEsqlSpecIT extends EsqlSpecTestCase { @ClassRule @@ -82,7 +82,7 @@ protected boolean supportsInferenceTestService() { @Override protected boolean supportsIndexModeLookup() throws IOException { - return hasCapabilities(List.of(JOIN_LOOKUP_V11.capabilityName())); + return hasCapabilities(List.of(JOIN_LOOKUP_V12.capabilityName())); } @Override diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index f8b921f239923..4d06db94801bf 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -48,7 +48,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V11; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST; import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC; @@ -124,7 +124,7 @@ protected void shouldSkipTest(String testName) throws IOException { assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); - assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V11.capabilityName())); + assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName())); } private TestFeatureService remoteFeaturesService() throws IOException { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java index 5e0aeb5b3535d..7b8eb435a8c1b 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java @@ -227,7 +227,7 @@ public void testIndicesDontExist() throws IOException { assertThat(e.getMessage(), containsString("index_not_found_exception")); assertThat(e.getMessage(), anyOf(containsString("no such index [foo]"), containsString("no such index [remote_cluster:foo]"))); - if (EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()) { + if (EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()) { e = expectThrows( ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test1") + " | LOOKUP JOIN foo ON id1")) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 8d24ddb45602b..3d483d3f10157 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -8,7 +8,7 @@ ############################################### basicOnTheDataNode -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | EVAL language_code = languages @@ -25,7 +25,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; basicRow -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW language_code = 1 | LOOKUP JOIN languages_lookup ON language_code @@ -36,7 +36,7 @@ language_code:integer | language_name:keyword ; basicOnTheCoordinator -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | SORT emp_no @@ -53,7 +53,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; subsequentEvalOnTheDataNode -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | EVAL language_code = languages @@ -71,7 +71,7 @@ emp_no:integer | language_code:integer | language_name:keyword | language_code_x ; subsequentEvalOnTheCoordinator -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | SORT emp_no @@ -89,7 +89,7 @@ emp_no:integer | language_code:integer | language_name:keyword | language_code_x ; sortEvalBeforeLookup -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | SORT emp_no @@ -106,7 +106,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; nonUniqueLeftKeyOnTheDataNode -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | WHERE emp_no <= 10030 @@ -130,7 +130,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; nonUniqueRightKeyOnTheDataNode -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | EVAL language_code = emp_no % 10 @@ -154,7 +154,7 @@ emp_no:integer | language_code:integer | language_name:keyword | country:text ; nonUniqueRightKeyOnTheCoordinator -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | SORT emp_no @@ -178,7 +178,7 @@ emp_no:integer | language_code:integer | language_name:keyword | country:text ; nonUniqueRightKeyFromRow -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW language_code = 2 | LOOKUP JOIN languages_lookup_non_unique_key ON language_code @@ -192,7 +192,7 @@ language_code:integer | country:text | language_name:keyword ; repeatedIndexOnFrom -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM languages_lookup | LOOKUP JOIN languages_lookup ON language_code @@ -210,7 +210,7 @@ dropAllLookedUpFieldsOnTheDataNode-Ignore // Depends on // https://github.com/elastic/elasticsearch/issues/118778 // https://github.com/elastic/elasticsearch/issues/118781 -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | EVAL language_code = emp_no % 10 @@ -231,7 +231,7 @@ dropAllLookedUpFieldsOnTheCoordinator-Ignore // Depends on // https://github.com/elastic/elasticsearch/issues/118778 // https://github.com/elastic/elasticsearch/issues/118781 -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | SORT emp_no @@ -256,7 +256,7 @@ emp_no:integer ############################################### filterOnLeftSide -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | EVAL language_code = languages @@ -273,7 +273,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnRightSide -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -289,7 +289,7 @@ FROM sample_data ; filterOnRightSideAfterStats -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -302,7 +302,7 @@ count:long | type:keyword ; filterOnJoinKey -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | EVAL language_code = languages @@ -317,7 +317,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnJoinKeyAndRightSide -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | WHERE emp_no < 10006 @@ -334,7 +334,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnRightSideOnTheCoordinator -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | SORT emp_no @@ -350,7 +350,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnJoinKeyOnTheCoordinator -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | SORT emp_no @@ -366,7 +366,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnJoinKeyAndRightSideOnTheCoordinator -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | SORT emp_no @@ -383,7 +383,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnTheDataNodeThenFilterOnTheCoordinator -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | EVAL language_code = languages @@ -404,7 +404,7 @@ emp_no:integer | language_code:integer | language_name:keyword ########################################################################### nullJoinKeyOnTheDataNode -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | WHERE emp_no < 10004 @@ -426,7 +426,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; mvJoinKeyOnTheDataNode -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | WHERE 10003 < emp_no AND emp_no < 10008 @@ -445,7 +445,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; mvJoinKeyFromRow -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW language_code = [4, 5, 6, 7] | LOOKUP JOIN languages_lookup_non_unique_key ON language_code @@ -461,7 +461,7 @@ language_code:integer | language_name:keyword | country:text ; mvJoinKeyFromRowExpanded -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW language_code = [4, 5, 6, 7, 8] | MV_EXPAND language_code @@ -484,7 +484,7 @@ language_code:integer | language_name:keyword | country:text ########################################################################### joinOnNestedField -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM employees | WHERE 10000 < emp_no AND emp_no < 10006 @@ -504,7 +504,7 @@ emp_no:integer | language.id:integer | language.name:text joinOnNestedFieldRow -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW language.code = "EN" | LOOKUP JOIN languages_nested_fields ON language.code @@ -517,7 +517,7 @@ language.id:integer | language.code:keyword | language.name.keyword:keyword joinOnNestedNestedFieldRow -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW language.name.keyword = "English" | LOOKUP JOIN languages_nested_fields ON language.name.keyword @@ -533,7 +533,7 @@ language.id:integer | language.name:text | language.name.keyword:keyword ############################################### lookupIPFromRow -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -544,7 +544,7 @@ left | 172.21.0.5 | right | Development ; lookupIPFromKeepRow -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", right = "right" | KEEP left, client_ip, right @@ -556,7 +556,7 @@ left | 172.21.0.5 | right | Development ; lookupIPFromRowWithShadowing -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -567,7 +567,7 @@ left | 172.21.0.5 | right | Development ; lookupIPFromRowWithShadowingKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -580,7 +580,7 @@ left | 172.21.0.5 | right | Development ; lookupIPFromRowWithShadowingKeepReordered -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -593,7 +593,7 @@ right | Development | 172.21.0.5 ; lookupIPFromIndex -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -612,7 +612,7 @@ ignoreOrder:true ; lookupIPFromIndexKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -632,7 +632,7 @@ ignoreOrder:true ; lookupIPFromIndexKeepKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | KEEP client_ip, event_duration, @timestamp, message @@ -654,7 +654,7 @@ timestamp:date | client_ip:keyword | event_duration:long | msg:keyword ; lookupIPFromIndexStats -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -670,7 +670,7 @@ count:long | env:keyword ; lookupIPFromIndexStatsKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -687,7 +687,7 @@ count:long | env:keyword ; statsAndLookupIPFromIndex -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -708,7 +708,7 @@ count:long | client_ip:keyword | env:keyword ############################################### lookupMessageFromRow -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", message = "Connected to 10.1.0.1", right = "right" | LOOKUP JOIN message_types_lookup ON message @@ -719,7 +719,7 @@ left | Connected to 10.1.0.1 | right | Success ; lookupMessageFromKeepRow -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", message = "Connected to 10.1.0.1", right = "right" | KEEP left, message, right @@ -731,7 +731,7 @@ left | Connected to 10.1.0.1 | right | Success ; lookupMessageFromRowWithShadowing -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", message = "Connected to 10.1.0.1", type = "unknown", right = "right" | LOOKUP JOIN message_types_lookup ON message @@ -742,7 +742,7 @@ left | Connected to 10.1.0.1 | right | Success ; lookupMessageFromRowWithShadowingKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", message = "Connected to 10.1.0.1", type = "unknown", right = "right" | LOOKUP JOIN message_types_lookup ON message @@ -754,7 +754,7 @@ left | Connected to 10.1.0.1 | right | Success ; lookupMessageFromIndex -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -772,7 +772,7 @@ ignoreOrder:true ; lookupMessageFromIndexKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -791,7 +791,7 @@ ignoreOrder:true ; lookupMessageFromIndexKeepKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | KEEP client_ip, event_duration, @timestamp, message @@ -811,7 +811,7 @@ ignoreOrder:true ; lookupMessageFromIndexKeepReordered -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -830,7 +830,7 @@ Success | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 ; lookupMessageFromIndexStats -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -845,7 +845,7 @@ count:long | type:keyword ; lookupMessageFromIndexStatsKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -861,7 +861,7 @@ count:long | type:keyword ; statsAndLookupMessageFromIndex -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | STATS count = count(message) BY message @@ -879,7 +879,7 @@ count:long | type:keyword | message:keyword ; lookupMessageFromIndexTwice -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -901,7 +901,7 @@ ignoreOrder:true ; lookupMessageFromIndexTwiceKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -924,7 +924,7 @@ ignoreOrder:true ; lookupMessageFromIndexTwiceFullyShadowing -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -948,7 +948,7 @@ ignoreOrder:true ############################################### lookupIPAndMessageFromRow -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -960,7 +960,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowKeepBefore -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" | KEEP left, client_ip, message, right @@ -973,7 +973,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowKeepBetween -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -986,7 +986,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowKeepAfter -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -999,7 +999,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowWithShadowing -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", type = "type", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -1011,7 +1011,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowWithShadowingKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -1025,7 +1025,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowWithShadowingKeepKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -1040,7 +1040,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowWithShadowingKeepKeepKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -1056,7 +1056,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowWithShadowingKeepReordered -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -1070,7 +1070,7 @@ right | Development | Success | 172.21.0.5 ; lookupIPAndMessageFromIndex -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1090,7 +1090,7 @@ ignoreOrder:true ; lookupIPAndMessageFromIndexKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1111,7 +1111,7 @@ ignoreOrder:true ; lookupIPAndMessageFromIndexStats -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1129,7 +1129,7 @@ count:long | env:keyword | type:keyword ; lookupIPAndMessageFromIndexStatsKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1148,7 +1148,7 @@ count:long | env:keyword | type:keyword ; statsAndLookupIPAndMessageFromIndex -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1167,7 +1167,7 @@ count:long | client_ip:keyword | message:keyword | env:keyword | type:keyw ; lookupIPAndMessageFromIndexChainedEvalKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1189,7 +1189,7 @@ ignoreOrder:true ; lookupIPAndMessageFromIndexChainedRenameKeep -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1211,7 +1211,7 @@ ignoreOrder:true ; lookupIndexInFromRepeatedRowBug -required_capability: join_lookup_v11 +required_capability: join_lookup_v12 FROM languages_lookup_non_unique_key | WHERE language_code == 1 | LOOKUP JOIN languages_lookup ON language_code diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 5468d57392c2e..84820531a4208 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -680,7 +680,7 @@ public enum Cap { /** * LOOKUP JOIN */ - JOIN_LOOKUP_V11(Build.current().isSnapshot()), + JOIN_LOOKUP_V12(Build.current().isSnapshot()), /** * Fix for https://github.com/elastic/elasticsearch/issues/117054 diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index ed1ee71ff1968..89150d6a52534 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -268,7 +268,7 @@ public final void test() throws Throwable { ); assumeFalse( "lookup join disabled for csv tests", - testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.JOIN_LOOKUP_V11.capabilityName()) + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()) ); assumeFalse( "can't use TERM function in csv tests", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index fd2c701e9a029..4eb6888990c9c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2140,7 +2140,7 @@ public void testLookupMatchTypeWrong() { } public void testLookupJoinUnknownIndex() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); String errorMessage = "Unknown index [foobar]"; IndexResolution missingLookupIndex = IndexResolution.invalid(errorMessage); @@ -2169,7 +2169,7 @@ public void testLookupJoinUnknownIndex() { } public void testLookupJoinUnknownField() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); String query = "FROM test | LOOKUP JOIN languages_lookup ON last_name"; String errorMessage = "1:45: Unknown column [last_name] in right side of join"; @@ -2192,7 +2192,7 @@ public void testLookupJoinUnknownField() { } public void testMultipleLookupJoinsGiveDifferentAttributes() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); // The field attributes that get contributed by different LOOKUP JOIN commands must have different name ids, // even if they have the same names. Otherwise, things like dependency analysis - like in PruneColumns - cannot work based on @@ -2222,7 +2222,7 @@ public void testMultipleLookupJoinsGiveDifferentAttributes() { } public void testLookupJoinIndexMode() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); var indexResolution = AnalyzerTestUtils.expandedDefaultIndexResolution(); var lookupResolution = AnalyzerTestUtils.defaultLookupResolution(); @@ -2251,7 +2251,7 @@ public void testLookupJoinIndexMode() { } public void testLookupJoinRemoteClusterUnsupported() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); var remoteIndexResolution = loadMapping("mapping-default.json", "remote:my-index"); var lookupResolution = AnalyzerTestUtils.defaultLookupResolution(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java index 2ee6cf6136114..859e1d788ff06 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java @@ -113,7 +113,7 @@ public void testTooBigQuery() { } public void testJoinOnConstant() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertEquals( "1:55: JOIN ON clause only supports fields at the moment, found [123]", error("row languages = 1, gender = \"f\" | lookup join test on 123") @@ -129,7 +129,7 @@ public void testJoinOnConstant() { } public void testJoinOnMultipleFields() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertEquals( "1:35: JOIN ON clause only supports one field at the moment, found [2]", error("row languages = 1, gender = \"f\" | lookup join test on gender, languages") @@ -137,7 +137,7 @@ public void testJoinOnMultipleFields() { } public void testJoinTwiceOnTheSameField() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertEquals( "1:35: JOIN ON clause only supports one field at the moment, found [2]", error("row languages = 1, gender = \"f\" | lookup join test on languages, languages") @@ -145,7 +145,7 @@ public void testJoinTwiceOnTheSameField() { } public void testJoinTwiceOnTheSameField_TwoLookups() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertEquals( "1:80: JOIN ON clause only supports one field at the moment, found [2]", error("row languages = 1, gender = \"f\" | lookup join test on languages | eval x = 1 | lookup join test on gender, gender") diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index f932992e81557..0d81efd7a1ade 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1984,7 +1984,7 @@ public void testSortByAggregate() { } public void testLookupJoinDataTypeMismatch() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); query("FROM test | EVAL language_code = languages | LOOKUP JOIN languages_lookup ON language_code"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index a8f8054fbc6b1..404bb041e7c61 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -4928,7 +4928,7 @@ public void testPlanSanityCheck() throws Exception { } public void testPlanSanityCheckWithBinaryPlans() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); var plan = optimizedPlan(""" FROM test @@ -6003,7 +6003,7 @@ public void testLookupStats() { * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19] */ public void testLookupJoinPushDownFilterOnJoinKeyWithRename() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); String query = """ FROM test @@ -6045,7 +6045,7 @@ public void testLookupJoinPushDownFilterOnJoinKeyWithRename() { * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19] */ public void testLookupJoinPushDownFilterOnLeftSideField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); String query = """ FROM test @@ -6088,7 +6088,7 @@ public void testLookupJoinPushDownFilterOnLeftSideField() { * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19] */ public void testLookupJoinPushDownDisabledForLookupField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); String query = """ FROM test @@ -6132,7 +6132,7 @@ public void testLookupJoinPushDownDisabledForLookupField() { * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#19, language_name{f}#20] */ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); String query = """ FROM test @@ -6183,7 +6183,7 @@ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightFiel * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#19, language_name{f}#20] */ public void testLookupJoinPushDownDisabledForDisjunctionBetweenLeftAndRightField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); String query = """ FROM test diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 75825f4e8f480..17b18bd861bb1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -2618,7 +2618,7 @@ public void testVerifierOnMissingReferences() { } public void testVerifierOnMissingReferencesWithBinaryPlans() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); // Do not assert serialization: // This will have a LookupJoinExec, which is not serializable because it doesn't leave the coordinator. @@ -7301,7 +7301,7 @@ public void testLookupThenTopN() { } public void testLookupJoinFieldLoading() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); TestDataSource data = dataSetWithLookupIndices(Map.of("lookup_index", List.of("first_name", "foo", "bar", "baz"))); @@ -7378,7 +7378,7 @@ public void testLookupJoinFieldLoading() throws Exception { } public void testLookupJoinFieldLoadingTwoLookups() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); TestDataSource data = dataSetWithLookupIndices( Map.of( @@ -7432,7 +7432,7 @@ public void testLookupJoinFieldLoadingTwoLookups() throws Exception { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/119082") public void testLookupJoinFieldLoadingTwoLookupsProjectInBetween() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); TestDataSource data = dataSetWithLookupIndices( Map.of( @@ -7473,7 +7473,7 @@ public void testLookupJoinFieldLoadingTwoLookupsProjectInBetween() throws Except @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/118778") public void testLookupJoinFieldLoadingDropAllFields() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); TestDataSource data = dataSetWithLookupIndices(Map.of("lookup_index", List.of("first_name", "foo", "bar", "baz"))); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index b1c9030db7a43..e7ea479d199d8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1365,7 +1365,7 @@ public void testMetrics() { } public void testLookupJoin() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( "FROM employees | KEEP languages | RENAME languages AS language_code | LOOKUP JOIN languages_lookup ON language_code", Set.of("languages", "languages.*", "language_code", "language_code.*"), @@ -1374,7 +1374,7 @@ public void testLookupJoin() { } public void testLookupJoinKeep() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM employees @@ -1388,7 +1388,7 @@ public void testLookupJoinKeep() { } public void testLookupJoinKeepWildcard() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM employees @@ -1402,7 +1402,7 @@ public void testLookupJoinKeepWildcard() { } public void testMultiLookupJoin() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1415,7 +1415,7 @@ public void testMultiLookupJoin() { } public void testMultiLookupJoinKeepBefore() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1429,7 +1429,7 @@ public void testMultiLookupJoinKeepBefore() { } public void testMultiLookupJoinKeepBetween() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1454,7 +1454,7 @@ public void testMultiLookupJoinKeepBetween() { } public void testMultiLookupJoinKeepAfter() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1481,7 +1481,7 @@ public void testMultiLookupJoinKeepAfter() { } public void testMultiLookupJoinKeepAfterWildcard() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1495,7 +1495,7 @@ public void testMultiLookupJoinKeepAfterWildcard() { } public void testMultiLookupJoinSameIndex() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1509,7 +1509,7 @@ public void testMultiLookupJoinSameIndex() { } public void testMultiLookupJoinSameIndexKeepBefore() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1524,7 +1524,7 @@ public void testMultiLookupJoinSameIndexKeepBefore() { } public void testMultiLookupJoinSameIndexKeepBetween() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1550,7 +1550,7 @@ public void testMultiLookupJoinSameIndexKeepBetween() { } public void testMultiLookupJoinSameIndexKeepAfter() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assertFieldNames( """ FROM sample_data diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml index e7cda33896149..b31b39fc4b8f8 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml @@ -6,7 +6,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [join_lookup_v11] + capabilities: [join_lookup_v12] reason: "uses LOOKUP JOIN" - do: indices.create: From 57a0df6cb0b2e54b8d34d732af3935f242f1cede Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Tue, 21 Jan 2025 14:10:24 +0100 Subject: [PATCH 05/11] Revert "inc lookup version" This reverts commit 5f55608f33a0797b469e6d9ab7de00582c9d9912. --- .../xpack/esql/EsqlSecurityIT.java | 4 +- .../esql/qa/mixed/MixedClusterEsqlSpecIT.java | 4 +- .../xpack/esql/ccq/MultiClusterSpecIT.java | 4 +- .../rest/RequestIndexFilteringTestCase.java | 2 +- .../src/main/resources/lookup-join.csv-spec | 142 +++++++++--------- .../xpack/esql/action/EsqlCapabilities.java | 2 +- .../elasticsearch/xpack/esql/CsvTests.java | 2 +- .../xpack/esql/analysis/AnalyzerTests.java | 10 +- .../xpack/esql/analysis/ParsingTests.java | 8 +- .../xpack/esql/analysis/VerifierTests.java | 2 +- .../optimizer/LogicalPlanOptimizerTests.java | 12 +- .../optimizer/PhysicalPlanOptimizerTests.java | 10 +- .../session/IndexResolverFieldNamesTests.java | 24 +-- .../test/esql/190_lookup_join.yml | 2 +- 14 files changed, 114 insertions(+), 114 deletions(-) diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java index f57e10dbcac3c..98e5799c8d3f2 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java @@ -548,7 +548,7 @@ record Listen(long timestamp, String songId, double duration) { public void testLookupJoinIndexAllowed() throws Exception { assumeTrue( "Requires LOOKUP JOIN capability", - EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName())) + EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V11.capabilityName())) ); Response resp = runESQLCommand( @@ -587,7 +587,7 @@ public void testLookupJoinIndexAllowed() throws Exception { public void testLookupJoinIndexForbidden() throws Exception { assumeTrue( "Requires LOOKUP JOIN capability", - EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName())) + EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V11.capabilityName())) ); var resp = expectThrows( diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java index 790b12346bb14..3b5377c2768fb 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java @@ -20,7 +20,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V11; public class MixedClusterEsqlSpecIT extends EsqlSpecTestCase { @ClassRule @@ -82,7 +82,7 @@ protected boolean supportsInferenceTestService() { @Override protected boolean supportsIndexModeLookup() throws IOException { - return hasCapabilities(List.of(JOIN_LOOKUP_V12.capabilityName())); + return hasCapabilities(List.of(JOIN_LOOKUP_V11.capabilityName())); } @Override diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index 4d06db94801bf..f8b921f239923 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -48,7 +48,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V11; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST; import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC; @@ -124,7 +124,7 @@ protected void shouldSkipTest(String testName) throws IOException { assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); - assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName())); + assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V11.capabilityName())); } private TestFeatureService remoteFeaturesService() throws IOException { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java index 7b8eb435a8c1b..5e0aeb5b3535d 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java @@ -227,7 +227,7 @@ public void testIndicesDontExist() throws IOException { assertThat(e.getMessage(), containsString("index_not_found_exception")); assertThat(e.getMessage(), anyOf(containsString("no such index [foo]"), containsString("no such index [remote_cluster:foo]"))); - if (EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()) { + if (EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()) { e = expectThrows( ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test1") + " | LOOKUP JOIN foo ON id1")) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 3d483d3f10157..8d24ddb45602b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -8,7 +8,7 @@ ############################################### basicOnTheDataNode -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | EVAL language_code = languages @@ -25,7 +25,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; basicRow -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW language_code = 1 | LOOKUP JOIN languages_lookup ON language_code @@ -36,7 +36,7 @@ language_code:integer | language_name:keyword ; basicOnTheCoordinator -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | SORT emp_no @@ -53,7 +53,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; subsequentEvalOnTheDataNode -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | EVAL language_code = languages @@ -71,7 +71,7 @@ emp_no:integer | language_code:integer | language_name:keyword | language_code_x ; subsequentEvalOnTheCoordinator -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | SORT emp_no @@ -89,7 +89,7 @@ emp_no:integer | language_code:integer | language_name:keyword | language_code_x ; sortEvalBeforeLookup -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | SORT emp_no @@ -106,7 +106,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; nonUniqueLeftKeyOnTheDataNode -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | WHERE emp_no <= 10030 @@ -130,7 +130,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; nonUniqueRightKeyOnTheDataNode -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | EVAL language_code = emp_no % 10 @@ -154,7 +154,7 @@ emp_no:integer | language_code:integer | language_name:keyword | country:text ; nonUniqueRightKeyOnTheCoordinator -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | SORT emp_no @@ -178,7 +178,7 @@ emp_no:integer | language_code:integer | language_name:keyword | country:text ; nonUniqueRightKeyFromRow -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW language_code = 2 | LOOKUP JOIN languages_lookup_non_unique_key ON language_code @@ -192,7 +192,7 @@ language_code:integer | country:text | language_name:keyword ; repeatedIndexOnFrom -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM languages_lookup | LOOKUP JOIN languages_lookup ON language_code @@ -210,7 +210,7 @@ dropAllLookedUpFieldsOnTheDataNode-Ignore // Depends on // https://github.com/elastic/elasticsearch/issues/118778 // https://github.com/elastic/elasticsearch/issues/118781 -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | EVAL language_code = emp_no % 10 @@ -231,7 +231,7 @@ dropAllLookedUpFieldsOnTheCoordinator-Ignore // Depends on // https://github.com/elastic/elasticsearch/issues/118778 // https://github.com/elastic/elasticsearch/issues/118781 -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | SORT emp_no @@ -256,7 +256,7 @@ emp_no:integer ############################################### filterOnLeftSide -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | EVAL language_code = languages @@ -273,7 +273,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnRightSide -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -289,7 +289,7 @@ FROM sample_data ; filterOnRightSideAfterStats -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -302,7 +302,7 @@ count:long | type:keyword ; filterOnJoinKey -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | EVAL language_code = languages @@ -317,7 +317,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnJoinKeyAndRightSide -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | WHERE emp_no < 10006 @@ -334,7 +334,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnRightSideOnTheCoordinator -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | SORT emp_no @@ -350,7 +350,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnJoinKeyOnTheCoordinator -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | SORT emp_no @@ -366,7 +366,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnJoinKeyAndRightSideOnTheCoordinator -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | SORT emp_no @@ -383,7 +383,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; filterOnTheDataNodeThenFilterOnTheCoordinator -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | EVAL language_code = languages @@ -404,7 +404,7 @@ emp_no:integer | language_code:integer | language_name:keyword ########################################################################### nullJoinKeyOnTheDataNode -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | WHERE emp_no < 10004 @@ -426,7 +426,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; mvJoinKeyOnTheDataNode -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | WHERE 10003 < emp_no AND emp_no < 10008 @@ -445,7 +445,7 @@ emp_no:integer | language_code:integer | language_name:keyword ; mvJoinKeyFromRow -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW language_code = [4, 5, 6, 7] | LOOKUP JOIN languages_lookup_non_unique_key ON language_code @@ -461,7 +461,7 @@ language_code:integer | language_name:keyword | country:text ; mvJoinKeyFromRowExpanded -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW language_code = [4, 5, 6, 7, 8] | MV_EXPAND language_code @@ -484,7 +484,7 @@ language_code:integer | language_name:keyword | country:text ########################################################################### joinOnNestedField -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM employees | WHERE 10000 < emp_no AND emp_no < 10006 @@ -504,7 +504,7 @@ emp_no:integer | language.id:integer | language.name:text joinOnNestedFieldRow -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW language.code = "EN" | LOOKUP JOIN languages_nested_fields ON language.code @@ -517,7 +517,7 @@ language.id:integer | language.code:keyword | language.name.keyword:keyword joinOnNestedNestedFieldRow -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW language.name.keyword = "English" | LOOKUP JOIN languages_nested_fields ON language.name.keyword @@ -533,7 +533,7 @@ language.id:integer | language.name:text | language.name.keyword:keyword ############################################### lookupIPFromRow -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -544,7 +544,7 @@ left | 172.21.0.5 | right | Development ; lookupIPFromKeepRow -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", right = "right" | KEEP left, client_ip, right @@ -556,7 +556,7 @@ left | 172.21.0.5 | right | Development ; lookupIPFromRowWithShadowing -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -567,7 +567,7 @@ left | 172.21.0.5 | right | Development ; lookupIPFromRowWithShadowingKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -580,7 +580,7 @@ left | 172.21.0.5 | right | Development ; lookupIPFromRowWithShadowingKeepReordered -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -593,7 +593,7 @@ right | Development | 172.21.0.5 ; lookupIPFromIndex -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -612,7 +612,7 @@ ignoreOrder:true ; lookupIPFromIndexKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -632,7 +632,7 @@ ignoreOrder:true ; lookupIPFromIndexKeepKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | KEEP client_ip, event_duration, @timestamp, message @@ -654,7 +654,7 @@ timestamp:date | client_ip:keyword | event_duration:long | msg:keyword ; lookupIPFromIndexStats -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -670,7 +670,7 @@ count:long | env:keyword ; lookupIPFromIndexStatsKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -687,7 +687,7 @@ count:long | env:keyword ; statsAndLookupIPFromIndex -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -708,7 +708,7 @@ count:long | client_ip:keyword | env:keyword ############################################### lookupMessageFromRow -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", message = "Connected to 10.1.0.1", right = "right" | LOOKUP JOIN message_types_lookup ON message @@ -719,7 +719,7 @@ left | Connected to 10.1.0.1 | right | Success ; lookupMessageFromKeepRow -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", message = "Connected to 10.1.0.1", right = "right" | KEEP left, message, right @@ -731,7 +731,7 @@ left | Connected to 10.1.0.1 | right | Success ; lookupMessageFromRowWithShadowing -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", message = "Connected to 10.1.0.1", type = "unknown", right = "right" | LOOKUP JOIN message_types_lookup ON message @@ -742,7 +742,7 @@ left | Connected to 10.1.0.1 | right | Success ; lookupMessageFromRowWithShadowingKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", message = "Connected to 10.1.0.1", type = "unknown", right = "right" | LOOKUP JOIN message_types_lookup ON message @@ -754,7 +754,7 @@ left | Connected to 10.1.0.1 | right | Success ; lookupMessageFromIndex -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -772,7 +772,7 @@ ignoreOrder:true ; lookupMessageFromIndexKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -791,7 +791,7 @@ ignoreOrder:true ; lookupMessageFromIndexKeepKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | KEEP client_ip, event_duration, @timestamp, message @@ -811,7 +811,7 @@ ignoreOrder:true ; lookupMessageFromIndexKeepReordered -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -830,7 +830,7 @@ Success | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 ; lookupMessageFromIndexStats -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -845,7 +845,7 @@ count:long | type:keyword ; lookupMessageFromIndexStatsKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -861,7 +861,7 @@ count:long | type:keyword ; statsAndLookupMessageFromIndex -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | STATS count = count(message) BY message @@ -879,7 +879,7 @@ count:long | type:keyword | message:keyword ; lookupMessageFromIndexTwice -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -901,7 +901,7 @@ ignoreOrder:true ; lookupMessageFromIndexTwiceKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -924,7 +924,7 @@ ignoreOrder:true ; lookupMessageFromIndexTwiceFullyShadowing -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | LOOKUP JOIN message_types_lookup ON message @@ -948,7 +948,7 @@ ignoreOrder:true ############################################### lookupIPAndMessageFromRow -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -960,7 +960,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowKeepBefore -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" | KEEP left, client_ip, message, right @@ -973,7 +973,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowKeepBetween -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -986,7 +986,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowKeepAfter -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -999,7 +999,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowWithShadowing -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", type = "type", right = "right" | LOOKUP JOIN clientips_lookup ON client_ip @@ -1011,7 +1011,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowWithShadowingKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -1025,7 +1025,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowWithShadowingKeepKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -1040,7 +1040,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowWithShadowingKeepKeepKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -1056,7 +1056,7 @@ left | 172.21.0.5 | Connected to 10.1.0.1 | right | Devel ; lookupIPAndMessageFromRowWithShadowingKeepReordered -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 ROW left = "left", client_ip = "172.21.0.5", message = "Connected to 10.1.0.1", env = "env", right = "right" | EVAL client_ip = client_ip::keyword @@ -1070,7 +1070,7 @@ right | Development | Success | 172.21.0.5 ; lookupIPAndMessageFromIndex -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1090,7 +1090,7 @@ ignoreOrder:true ; lookupIPAndMessageFromIndexKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1111,7 +1111,7 @@ ignoreOrder:true ; lookupIPAndMessageFromIndexStats -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1129,7 +1129,7 @@ count:long | env:keyword | type:keyword ; lookupIPAndMessageFromIndexStatsKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1148,7 +1148,7 @@ count:long | env:keyword | type:keyword ; statsAndLookupIPAndMessageFromIndex -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1167,7 +1167,7 @@ count:long | client_ip:keyword | message:keyword | env:keyword | type:keyw ; lookupIPAndMessageFromIndexChainedEvalKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1189,7 +1189,7 @@ ignoreOrder:true ; lookupIPAndMessageFromIndexChainedRenameKeep -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM sample_data | EVAL client_ip = client_ip::keyword @@ -1211,7 +1211,7 @@ ignoreOrder:true ; lookupIndexInFromRepeatedRowBug -required_capability: join_lookup_v12 +required_capability: join_lookup_v11 FROM languages_lookup_non_unique_key | WHERE language_code == 1 | LOOKUP JOIN languages_lookup ON language_code diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 84820531a4208..5468d57392c2e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -680,7 +680,7 @@ public enum Cap { /** * LOOKUP JOIN */ - JOIN_LOOKUP_V12(Build.current().isSnapshot()), + JOIN_LOOKUP_V11(Build.current().isSnapshot()), /** * Fix for https://github.com/elastic/elasticsearch/issues/117054 diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 89150d6a52534..ed1ee71ff1968 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -268,7 +268,7 @@ public final void test() throws Throwable { ); assumeFalse( "lookup join disabled for csv tests", - testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()) + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.JOIN_LOOKUP_V11.capabilityName()) ); assumeFalse( "can't use TERM function in csv tests", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 4eb6888990c9c..fd2c701e9a029 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2140,7 +2140,7 @@ public void testLookupMatchTypeWrong() { } public void testLookupJoinUnknownIndex() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); String errorMessage = "Unknown index [foobar]"; IndexResolution missingLookupIndex = IndexResolution.invalid(errorMessage); @@ -2169,7 +2169,7 @@ public void testLookupJoinUnknownIndex() { } public void testLookupJoinUnknownField() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); String query = "FROM test | LOOKUP JOIN languages_lookup ON last_name"; String errorMessage = "1:45: Unknown column [last_name] in right side of join"; @@ -2192,7 +2192,7 @@ public void testLookupJoinUnknownField() { } public void testMultipleLookupJoinsGiveDifferentAttributes() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); // The field attributes that get contributed by different LOOKUP JOIN commands must have different name ids, // even if they have the same names. Otherwise, things like dependency analysis - like in PruneColumns - cannot work based on @@ -2222,7 +2222,7 @@ public void testMultipleLookupJoinsGiveDifferentAttributes() { } public void testLookupJoinIndexMode() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); var indexResolution = AnalyzerTestUtils.expandedDefaultIndexResolution(); var lookupResolution = AnalyzerTestUtils.defaultLookupResolution(); @@ -2251,7 +2251,7 @@ public void testLookupJoinIndexMode() { } public void testLookupJoinRemoteClusterUnsupported() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); var remoteIndexResolution = loadMapping("mapping-default.json", "remote:my-index"); var lookupResolution = AnalyzerTestUtils.defaultLookupResolution(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java index 859e1d788ff06..2ee6cf6136114 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java @@ -113,7 +113,7 @@ public void testTooBigQuery() { } public void testJoinOnConstant() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertEquals( "1:55: JOIN ON clause only supports fields at the moment, found [123]", error("row languages = 1, gender = \"f\" | lookup join test on 123") @@ -129,7 +129,7 @@ public void testJoinOnConstant() { } public void testJoinOnMultipleFields() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertEquals( "1:35: JOIN ON clause only supports one field at the moment, found [2]", error("row languages = 1, gender = \"f\" | lookup join test on gender, languages") @@ -137,7 +137,7 @@ public void testJoinOnMultipleFields() { } public void testJoinTwiceOnTheSameField() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertEquals( "1:35: JOIN ON clause only supports one field at the moment, found [2]", error("row languages = 1, gender = \"f\" | lookup join test on languages, languages") @@ -145,7 +145,7 @@ public void testJoinTwiceOnTheSameField() { } public void testJoinTwiceOnTheSameField_TwoLookups() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertEquals( "1:80: JOIN ON clause only supports one field at the moment, found [2]", error("row languages = 1, gender = \"f\" | lookup join test on languages | eval x = 1 | lookup join test on gender, gender") diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 0d81efd7a1ade..f932992e81557 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1984,7 +1984,7 @@ public void testSortByAggregate() { } public void testLookupJoinDataTypeMismatch() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); query("FROM test | EVAL language_code = languages | LOOKUP JOIN languages_lookup ON language_code"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 404bb041e7c61..a8f8054fbc6b1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -4928,7 +4928,7 @@ public void testPlanSanityCheck() throws Exception { } public void testPlanSanityCheckWithBinaryPlans() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); var plan = optimizedPlan(""" FROM test @@ -6003,7 +6003,7 @@ public void testLookupStats() { * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19] */ public void testLookupJoinPushDownFilterOnJoinKeyWithRename() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); String query = """ FROM test @@ -6045,7 +6045,7 @@ public void testLookupJoinPushDownFilterOnJoinKeyWithRename() { * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19] */ public void testLookupJoinPushDownFilterOnLeftSideField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); String query = """ FROM test @@ -6088,7 +6088,7 @@ public void testLookupJoinPushDownFilterOnLeftSideField() { * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19] */ public void testLookupJoinPushDownDisabledForLookupField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); String query = """ FROM test @@ -6132,7 +6132,7 @@ public void testLookupJoinPushDownDisabledForLookupField() { * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#19, language_name{f}#20] */ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); String query = """ FROM test @@ -6183,7 +6183,7 @@ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightFiel * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#19, language_name{f}#20] */ public void testLookupJoinPushDownDisabledForDisjunctionBetweenLeftAndRightField() { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); String query = """ FROM test diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 17b18bd861bb1..75825f4e8f480 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -2618,7 +2618,7 @@ public void testVerifierOnMissingReferences() { } public void testVerifierOnMissingReferencesWithBinaryPlans() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); // Do not assert serialization: // This will have a LookupJoinExec, which is not serializable because it doesn't leave the coordinator. @@ -7301,7 +7301,7 @@ public void testLookupThenTopN() { } public void testLookupJoinFieldLoading() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); TestDataSource data = dataSetWithLookupIndices(Map.of("lookup_index", List.of("first_name", "foo", "bar", "baz"))); @@ -7378,7 +7378,7 @@ public void testLookupJoinFieldLoading() throws Exception { } public void testLookupJoinFieldLoadingTwoLookups() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); TestDataSource data = dataSetWithLookupIndices( Map.of( @@ -7432,7 +7432,7 @@ public void testLookupJoinFieldLoadingTwoLookups() throws Exception { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/119082") public void testLookupJoinFieldLoadingTwoLookupsProjectInBetween() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); TestDataSource data = dataSetWithLookupIndices( Map.of( @@ -7473,7 +7473,7 @@ public void testLookupJoinFieldLoadingTwoLookupsProjectInBetween() throws Except @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/118778") public void testLookupJoinFieldLoadingDropAllFields() throws Exception { - assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("Requires LOOKUP JOIN", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); TestDataSource data = dataSetWithLookupIndices(Map.of("lookup_index", List.of("first_name", "foo", "bar", "baz"))); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index e7ea479d199d8..b1c9030db7a43 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -1365,7 +1365,7 @@ public void testMetrics() { } public void testLookupJoin() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( "FROM employees | KEEP languages | RENAME languages AS language_code | LOOKUP JOIN languages_lookup ON language_code", Set.of("languages", "languages.*", "language_code", "language_code.*"), @@ -1374,7 +1374,7 @@ public void testLookupJoin() { } public void testLookupJoinKeep() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( """ FROM employees @@ -1388,7 +1388,7 @@ public void testLookupJoinKeep() { } public void testLookupJoinKeepWildcard() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( """ FROM employees @@ -1402,7 +1402,7 @@ public void testLookupJoinKeepWildcard() { } public void testMultiLookupJoin() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1415,7 +1415,7 @@ public void testMultiLookupJoin() { } public void testMultiLookupJoinKeepBefore() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1429,7 +1429,7 @@ public void testMultiLookupJoinKeepBefore() { } public void testMultiLookupJoinKeepBetween() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1454,7 +1454,7 @@ public void testMultiLookupJoinKeepBetween() { } public void testMultiLookupJoinKeepAfter() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1481,7 +1481,7 @@ public void testMultiLookupJoinKeepAfter() { } public void testMultiLookupJoinKeepAfterWildcard() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1495,7 +1495,7 @@ public void testMultiLookupJoinKeepAfterWildcard() { } public void testMultiLookupJoinSameIndex() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1509,7 +1509,7 @@ public void testMultiLookupJoinSameIndex() { } public void testMultiLookupJoinSameIndexKeepBefore() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1524,7 +1524,7 @@ public void testMultiLookupJoinSameIndexKeepBefore() { } public void testMultiLookupJoinSameIndexKeepBetween() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( """ FROM sample_data @@ -1550,7 +1550,7 @@ public void testMultiLookupJoinSameIndexKeepBetween() { } public void testMultiLookupJoinSameIndexKeepAfter() { - assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue("LOOKUP JOIN available as snapshot only", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); assertFieldNames( """ FROM sample_data diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml index b31b39fc4b8f8..e7cda33896149 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/190_lookup_join.yml @@ -6,7 +6,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [join_lookup_v12] + capabilities: [join_lookup_v11] reason: "uses LOOKUP JOIN" - do: indices.create: From c6be044fed74ec334ab248ba224625309d8f9a21 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 22 Jan 2025 13:19:32 +0100 Subject: [PATCH 06/11] move check before index resolution --- .../xpack/esql/analysis/Analyzer.java | 23 ------------------- .../xpack/esql/parser/LogicalPlanBuilder.java | 23 +++++++++++++++++-- .../xpack/esql/analysis/AnalyzerTests.java | 18 --------------- .../esql/parser/StatementParserTests.java | 7 +++++- 4 files changed, 27 insertions(+), 44 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index acd261d255b31..fc1b7f6329ab3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -11,7 +11,6 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.index.IndexMode; import org.elasticsearch.logging.Logger; -import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.Column; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -660,28 +659,6 @@ private Join resolveLookupJoin(LookupJoin join) { return join; } - // joining with remote cluster is not supported yet - if (join.left() instanceof EsRelation esr) { - for (var index : esr.index().concreteIndices()) { - if (index.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR) != -1) { - return join.withConfig( - new JoinConfig( - type, - List.of( - new UnresolvedAttribute( - join.source(), - "unsupported", - "LOOKUP JOIN does not support joining with remote cluster indices [" + esr.index().name() + "]" - ) - ), - List.of(), - List.of() - ) - ); - } - } - } - JoinType coreJoin = using.coreJoin(); // verify the join type if (coreJoin != JoinTypes.LEFT) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index cc1ac6ac31385..301268e219147 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -15,6 +15,7 @@ import org.elasticsearch.dissect.DissectException; import org.elasticsearch.dissect.DissectParser; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.core.expression.Alias; @@ -523,9 +524,15 @@ public PlanFactory visitJoinCommand(EsqlBaseParser.JoinCommandContext ctx) { } var target = ctx.joinTarget(); + var rightPattern = visitIdentifier(target.index); + + if (RemoteClusterAware.isRemoteIndexName(rightPattern)) { + throw new ParsingException(source(target), "LOOKUP JOIN does not support remote cluster indices [{}]", rightPattern); + } + UnresolvedRelation right = new UnresolvedRelation( source(target), - new TableIdentifier(source(target.index), null, visitIdentifier(target.index)), + new TableIdentifier(source(target.index), null, rightPattern), false, emptyList(), IndexMode.LOOKUP, @@ -552,6 +559,18 @@ public PlanFactory visitJoinCommand(EsqlBaseParser.JoinCommandContext ctx) { throw new ParsingException(source, "JOIN ON clause only supports one field at the moment, found [{}]", matchFieldsCount); } - return p -> new LookupJoin(source, p, right, joinFields); + return p -> { + p.forEachUp(UnresolvedRelation.class, r -> { + if (RemoteClusterAware.isRemoteIndexName(r.table().index())) { + throw new ParsingException( + source(target), + "LOOKUP JOIN does not support remote cluster indices [{}]", + r.table().index() + ); + } + }); + + return new LookupJoin(source, p, right, joinFields); + }; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 1bdd0e87fb002..1c3b3a5c463e7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2253,24 +2253,6 @@ public void testLookupJoinIndexMode() { assertThat(e.getMessage(), containsString("1:25: invalid [test] resolution in lookup mode to an index in [standard] mode")); } - public void testLookupJoinRemoteClusterUnsupported() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()); - - var remoteIndexResolution = loadMapping("mapping-default.json", "remote:my-index"); - var lookupResolution = AnalyzerTestUtils.defaultLookupResolution(); - VerificationException e = expectThrows( - VerificationException.class, - () -> analyze( - "FROM remote:my-index | LOOKUP JOIN languages_lookup ON language_code", - AnalyzerTestUtils.analyzer(remoteIndexResolution, lookupResolution) - ) - ); - assertThat( - e.getMessage(), - containsString("1:24: LOOKUP JOIN does not support joining with remote cluster indices [remote:my-index]") - ); - } - public void testImplicitCasting() { var e = expectThrows(VerificationException.class, () -> analyze(""" from test | eval x = concat("2024", "-04", "-01") + 1 day diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 585f0681321e3..f2057d9bc5c4c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -2940,7 +2940,12 @@ public void testNamedFunctionArgumentWithUnsupportedNamedParameterTypes() { } } - public void testInvalidRemoteLookupJoin() { + public void testRemoteLookupJoin() { + expectError("FROM remote:left | LOOKUP JOIN right ON field", "LOOKUP JOIN does not support remote cluster indices [remote:left]"); + expectError( + "FROM left | LOOKUP JOIN `remote:right` ON field", + "LOOKUP JOIN does not support remote cluster indices [remote:right]" + ); // TODO ES-10559 this should be replaced with a proper error message once grammar allows indexPattern as joinTarget expectError("FROM my-index | LOOKUP JOIN remote:languages_lookup ON language_code", "line 1:35: token recognition error at: ':'"); } From 6a94eaf708566522fd3e512484605503e9ddd712 Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Wed, 22 Jan 2025 13:49:43 +0100 Subject: [PATCH 07/11] Update docs/changelog/120277.yaml --- docs/changelog/120277.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/120277.yaml diff --git a/docs/changelog/120277.yaml b/docs/changelog/120277.yaml new file mode 100644 index 0000000000000..82dd5bc07bf82 --- /dev/null +++ b/docs/changelog/120277.yaml @@ -0,0 +1,5 @@ +pr: 120277 +summary: Disallow CCS with lookup join +area: ES|QL +type: enhancement +issues: [] From d283968495cabc8a1b40637077d834da2149a87a Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Fri, 24 Jan 2025 14:02:42 +0100 Subject: [PATCH 08/11] Delete docs/changelog/120277.yaml --- docs/changelog/120277.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/120277.yaml diff --git a/docs/changelog/120277.yaml b/docs/changelog/120277.yaml deleted file mode 100644 index 82dd5bc07bf82..0000000000000 --- a/docs/changelog/120277.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 120277 -summary: Disallow CCS with lookup join -area: ES|QL -type: enhancement -issues: [] From 343d1041f98d815c35b974a92922cd16b0ab6a40 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Fri, 24 Jan 2025 14:36:53 +0100 Subject: [PATCH 09/11] fix merge and resolve some comments --- .../xpack/esql/parser/LogicalPlanBuilder.java | 21 ++++--- .../esql/parser/StatementParserTests.java | 56 ++++++++++++------- 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index dec12f44a93f2..50bbf3490e72c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -11,6 +11,7 @@ import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTree; import org.elasticsearch.Build; +import org.elasticsearch.common.Strings; import org.elasticsearch.core.Tuple; import org.elasticsearch.dissect.DissectException; import org.elasticsearch.dissect.DissectParser; @@ -529,7 +530,11 @@ public PlanFactory visitJoinCommand(EsqlBaseParser.JoinCommandContext ctx) { throw new ParsingException(source(target), "invalid index pattern [{}], * is not allowed in LOOKUP JOIN", rightPattern); } if (RemoteClusterAware.isRemoteIndexName(rightPattern)) { - throw new ParsingException(source(target), "LOOKUP JOIN does not support remote cluster indices [{}]", rightPattern); + throw new ParsingException( + source(target), + "invalid index pattern [{}], remote clusters are not supported in LOOKUP JOIN", + rightPattern + ); } UnresolvedRelation right = new UnresolvedRelation( @@ -563,12 +568,14 @@ public PlanFactory visitJoinCommand(EsqlBaseParser.JoinCommandContext ctx) { return p -> { p.forEachUp(UnresolvedRelation.class, r -> { - if (RemoteClusterAware.isRemoteIndexName(r.table().index())) { - throw new ParsingException( - source(target), - "LOOKUP JOIN does not support remote cluster indices [{}]", - r.table().index() - ); + for (var leftPattern : Strings.splitStringByCommaToArray(r.table().index())) { + if (RemoteClusterAware.isRemoteIndexName(leftPattern)) { + throw new ParsingException( + source(target), + "invalid index pattern [{}], remote clusters are not supported in LOOKUP JOIN", + r.table().index() + ); + } } }); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 535d5f8bf8b9b..4e1acff3cd310 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -78,6 +78,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.paramAsIdentifier; import static org.elasticsearch.xpack.esql.EsqlTestUtils.paramAsPattern; import static org.elasticsearch.xpack.esql.EsqlTestUtils.referenceAttribute; +import static org.elasticsearch.xpack.esql.IdentifierGenerator.Features.CROSS_CLUSTER; import static org.elasticsearch.xpack.esql.IdentifierGenerator.Features.WILDCARD_PATTERN; import static org.elasticsearch.xpack.esql.IdentifierGenerator.randomIndexPattern; import static org.elasticsearch.xpack.esql.IdentifierGenerator.randomIndexPatterns; @@ -2947,13 +2948,20 @@ public void testNamedFunctionArgumentWithUnsupportedNamedParameterTypes() { } } - public void testValidJoinPattern() { + public void testValidFromPattern() { var basePattern = randomIndexPatterns(); - var joinPattern = randomIndexPattern(without(WILDCARD_PATTERN)); + + var plan = statement("FROM " + basePattern); + + assertThat(as(plan, UnresolvedRelation.class).table().index(), equalTo(unquoteIndexPattern(basePattern))); + } + + public void testValidJoinPattern() { + var basePattern = randomIndexPatterns(without(CROSS_CLUSTER)); + var joinPattern = randomIndexPattern(without(WILDCARD_PATTERN), without(CROSS_CLUSTER)); var onField = randomIdentifier(); - var type = randomFrom("", "LOOKUP "); - var plan = statement("FROM " + basePattern + " | " + type + " JOIN " + joinPattern + " ON " + onField); + var plan = statement("FROM " + basePattern + " | LOOKUP JOIN " + joinPattern + " ON " + onField); var join = as(plan, LookupJoin.class); assertThat(as(join.left(), UnresolvedRelation.class).table().index(), equalTo(unquoteIndexPattern(basePattern))); @@ -2966,20 +2974,30 @@ public void testValidJoinPattern() { } public void testInvalidJoinPatterns() { - var joinPattern = randomIndexPattern(WILDCARD_PATTERN); - expectError( - "FROM " + randomIndexPatterns() + " | JOIN " + joinPattern + " ON " + randomIdentifier(), - "invalid index pattern [" + unquoteIndexPattern(joinPattern) + "], * is not allowed in LOOKUP JOIN" - ); - } - - public void testRemoteLookupJoin() { - expectError("FROM remote:left | LOOKUP JOIN right ON field", "LOOKUP JOIN does not support remote cluster indices [remote:left]"); - expectError( - "FROM left | LOOKUP JOIN `remote:right` ON field", - "LOOKUP JOIN does not support remote cluster indices [remote:right]" - ); - // TODO ES-10559 this should be replaced with a proper error message once grammar allows indexPattern as joinTarget - expectError("FROM my-index | LOOKUP JOIN remote:languages_lookup ON language_code", "line 1:35: token recognition error at: ':'"); + { + // wildcard + var joinPattern = randomIndexPattern(WILDCARD_PATTERN, without(CROSS_CLUSTER)); + expectError( + "FROM " + randomIndexPatterns() + " | LOOKUP JOIN " + joinPattern + " ON " + randomIdentifier(), + "invalid index pattern [" + unquoteIndexPattern(joinPattern) + "], * is not allowed in LOOKUP JOIN" + ); + } + { + // remote cluster on the right + var joinPattern = randomIndexPattern(CROSS_CLUSTER, without(WILDCARD_PATTERN)); + expectError( + "FROM " + randomIndexPatterns() + " | LOOKUP JOIN " + joinPattern + " ON " + randomIdentifier(), + "invalid index pattern [" + unquoteIndexPattern(joinPattern) + "], remote clusters are not supported in LOOKUP JOIN" + ); + } + { + // remote cluster on the left + var fromPatterns = randomIndexPatterns(CROSS_CLUSTER, without(WILDCARD_PATTERN)); + var joinPattern = randomIndexPattern(without(CROSS_CLUSTER), without(WILDCARD_PATTERN)); + expectError( + "FROM " + fromPatterns + " | LOOKUP JOIN " + joinPattern + " ON " + randomIdentifier(), + "invalid index pattern [" + unquoteIndexPattern(fromPatterns) + "], remote clusters are not supported in LOOKUP JOIN" + ); + } } } From 416a8fcfa553b23a66938a53ae5695c392686b49 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 27 Jan 2025 09:30:15 +0100 Subject: [PATCH 10/11] upd --- .../xpack/esql/parser/StatementParserTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 4e1acff3cd310..07f37c47d4a05 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -2984,15 +2984,16 @@ public void testInvalidJoinPatterns() { } { // remote cluster on the right + var fromPatterns = randomIndexPatterns(without(CROSS_CLUSTER)); var joinPattern = randomIndexPattern(CROSS_CLUSTER, without(WILDCARD_PATTERN)); expectError( - "FROM " + randomIndexPatterns() + " | LOOKUP JOIN " + joinPattern + " ON " + randomIdentifier(), + "FROM " + fromPatterns + " | LOOKUP JOIN " + joinPattern + " ON " + randomIdentifier(), "invalid index pattern [" + unquoteIndexPattern(joinPattern) + "], remote clusters are not supported in LOOKUP JOIN" ); } { // remote cluster on the left - var fromPatterns = randomIndexPatterns(CROSS_CLUSTER, without(WILDCARD_PATTERN)); + var fromPatterns = randomIndexPatterns(CROSS_CLUSTER); var joinPattern = randomIndexPattern(without(CROSS_CLUSTER), without(WILDCARD_PATTERN)); expectError( "FROM " + fromPatterns + " | LOOKUP JOIN " + joinPattern + " ON " + randomIdentifier(), From 59b78b72678b9577a2d81321dce395660ecf26a7 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 27 Jan 2025 09:34:07 +0100 Subject: [PATCH 11/11] fix merge --- .../elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java | 4 ++-- .../elasticsearch/xpack/esql/parser/StatementParserTests.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 5717c016dca72..7ddd3dafd2784 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -568,12 +568,12 @@ public PlanFactory visitJoinCommand(EsqlBaseParser.JoinCommandContext ctx) { return p -> { p.forEachUp(UnresolvedRelation.class, r -> { - for (var leftPattern : Strings.splitStringByCommaToArray(r.table().index())) { + for (var leftPattern : Strings.splitStringByCommaToArray(r.indexPattern().indexPattern())) { if (RemoteClusterAware.isRemoteIndexName(leftPattern)) { throw new ParsingException( source(target), "invalid index pattern [{}], remote clusters are not supported in LOOKUP JOIN", - r.table().index() + r.indexPattern().indexPattern() ); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 164e9e60baa0c..cb2df8dec9a6f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -2945,7 +2945,7 @@ public void testValidFromPattern() { var plan = statement("FROM " + basePattern); - assertThat(as(plan, UnresolvedRelation.class).table().index(), equalTo(unquoteIndexPattern(basePattern))); + assertThat(as(plan, UnresolvedRelation.class).indexPattern().indexPattern(), equalTo(unquoteIndexPattern(basePattern))); } public void testValidJoinPattern() {