Skip to content

Commit af92abf

Browse files
smalyshevsmalyshev
committed
Some test refactoring & unsupported syntax tests
1 parent 5ffc7d2 commit af92abf

File tree

8 files changed

+152
-28
lines changed

8 files changed

+152
-28
lines changed

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterLookupJoinIT.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,19 @@
77

88
package org.elasticsearch.xpack.esql.action;
99

10+
import org.elasticsearch.client.internal.Client;
1011
import org.elasticsearch.test.junit.annotations.TestLogging;
12+
import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
13+
import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction;
14+
import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction;
1115
import org.elasticsearch.xpack.esql.VerificationException;
1216
import org.elasticsearch.xpack.esql.core.type.DataType;
1317

1418
import java.io.IOException;
1519
import java.util.List;
1620
import java.util.Map;
1721

22+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
1823
import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
1924
import static org.hamcrest.Matchers.containsString;
2025
import static org.hamcrest.Matchers.empty;
@@ -346,7 +351,6 @@ public void testLookupJoinFieldTypes() throws IOException {
346351
assertThat(keyColumn.get().type(), equalTo(DataType.UNSUPPORTED));
347352
assertThat(keyColumn.get().originalTypes(), hasItems("keyword", "long"));
348353
}
349-
350354
}
351355

352356
protected Map<String, Object> setupClustersAndLookups() throws IOException {
@@ -356,6 +360,23 @@ protected Map<String, Object> setupClustersAndLookups() throws IOException {
356360
return setupData;
357361
}
358362

363+
public void setupHostsEnrich() {
364+
// the hosts policy are identical on every node
365+
Map<String, String> allHosts = Map.of("192.168.1.2", "Windows");
366+
Client client = client(LOCAL_CLUSTER);
367+
client.admin().indices().prepareCreate("hosts").setMapping("ip", "type=ip", "os", "type=keyword").get();
368+
for (Map.Entry<String, String> h : allHosts.entrySet()) {
369+
client.prepareIndex("hosts").setSource("ip", h.getKey(), "os", h.getValue()).get();
370+
}
371+
client.admin().indices().prepareRefresh("hosts").get();
372+
EnrichPolicy hostPolicy = new EnrichPolicy("match", null, List.of("hosts"), "ip", List.of("ip", "os"));
373+
client.execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "hosts", hostPolicy))
374+
.actionGet();
375+
client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "hosts"))
376+
.actionGet();
377+
assertAcked(client.admin().indices().prepareDelete("hosts"));
378+
}
379+
359380
private static void assertCCSExecutionInfoDetails(EsqlExecutionInfo executionInfo) {
360381
assertNotNull(executionInfo);
361382
assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterLookupJoinUnsupportedIT.java‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.esql.action;
9+
10+
import org.elasticsearch.client.internal.Client;
11+
import org.elasticsearch.common.settings.Settings;
12+
import org.elasticsearch.ingest.common.IngestCommonPlugin;
13+
import org.elasticsearch.plugins.Plugin;
14+
import org.elasticsearch.reindex.ReindexPlugin;
15+
import org.elasticsearch.xpack.core.XPackSettings;
16+
import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
17+
import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction;
18+
import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction;
19+
import org.elasticsearch.xpack.esql.VerificationException;
20+
21+
import java.io.IOException;
22+
import java.util.ArrayList;
23+
import java.util.Collection;
24+
import java.util.List;
25+
import java.util.Map;
26+
27+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
28+
import static org.hamcrest.Matchers.containsString;
29+
30+
public class CrossClusterLookupJoinUnsupportedIT extends AbstractCrossClusterTestCase {
31+
32+
@Override
33+
protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
34+
List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins(clusterAlias));
35+
plugins.add(CrossClusterEnrichIT.LocalStateEnrich.class);
36+
plugins.remove(EsqlAsyncActionIT.LocalStateEsqlAsync.class);
37+
plugins.add(IngestCommonPlugin.class);
38+
plugins.add(ReindexPlugin.class);
39+
return plugins;
40+
}
41+
42+
@Override
43+
protected Settings nodeSettings() {
44+
return Settings.builder().put(super.nodeSettings()).put(XPackSettings.SECURITY_ENABLED.getKey(), false).build();
45+
}
46+
47+
public void testLookupJoinUnsupported() throws IOException {
48+
setupClusters(2);
49+
populateLookupIndex(LOCAL_CLUSTER, "values_lookup", 10);
50+
populateLookupIndex(REMOTE_CLUSTER_1, "values_lookup", 25);
51+
52+
var ex = expectThrows(
53+
VerificationException.class,
54+
() -> runQuery(
55+
"FROM logs-*,c*:logs-* | STATS c = COUNT(*) by v | EVAL lookup_key = v | LOOKUP JOIN values_lookup ON lookup_key",
56+
randomBoolean()
57+
)
58+
);
59+
assertThat(ex.getMessage(), containsString("LOOKUP JOIN with remote indices can't be executed after STATS"));
60+
61+
ex = expectThrows(
62+
VerificationException.class,
63+
() -> runQuery(
64+
"FROM logs-*,c*:logs-* | SORT v | EVAL lookup_key = v | LOOKUP JOIN values_lookup ON lookup_key",
65+
randomBoolean()
66+
)
67+
);
68+
assertThat(ex.getMessage(), containsString("LOOKUP JOIN with remote indices can't be executed after SORT"));
69+
70+
setupHostsEnrich();
71+
ex = expectThrows(
72+
VerificationException.class,
73+
() -> runQuery(
74+
"FROM logs-*,c*:logs-* | EVAL ip = id | ENRICH _coordinator:hosts | EVAL lookup_key = v "
75+
+ "| LOOKUP JOIN values_lookup ON lookup_key",
76+
randomBoolean()
77+
)
78+
);
79+
assertThat(
80+
ex.getMessage(),
81+
containsString("LOOKUP JOIN with remote indices can't be executed after ENRICH with coordinator policy")
82+
);
83+
}
84+
85+
public void setupHostsEnrich() {
86+
// the hosts policy are identical on every node
87+
Map<String, String> allHosts = Map.of("192.168.1.2", "Windows");
88+
Client client = client(LOCAL_CLUSTER);
89+
client.admin().indices().prepareCreate("hosts").setMapping("ip", "type=ip", "os", "type=keyword").get();
90+
for (Map.Entry<String, String> h : allHosts.entrySet()) {
91+
client.prepareIndex("hosts").setSource("ip", h.getKey(), "os", h.getValue()).get();
92+
}
93+
client.admin().indices().prepareRefresh("hosts").get();
94+
EnrichPolicy hostPolicy = new EnrichPolicy("match", null, List.of("hosts"), "ip", List.of("ip", "os"));
95+
client.execute(PutEnrichPolicyAction.INSTANCE, new PutEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "hosts", hostPolicy))
96+
.actionGet();
97+
client.execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(TEST_REQUEST_TIMEOUT, "hosts"))
98+
.actionGet();
99+
assertAcked(client.admin().indices().prepareDelete("hosts"));
100+
}
101+
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -742,7 +742,7 @@ private Join resolveLookupJoin(LookupJoin join) {
742742
List<Attribute> rightKeys = resolveUsingColumns(cols, join.right().output(), "right");
743743

744744
config = new JoinConfig(coreJoin, leftKeys, leftKeys, rightKeys);
745-
join = new LookupJoin(join.source(), join.left(), join.right(), config);
745+
join = new LookupJoin(join.source(), join.left(), join.right(), config, join.isRemote());
746746
} else if (type != JoinTypes.LEFT) {
747747
// everything else is unsupported for now
748748
// LEFT can only happen by being mapped from a USING above. So we need to exclude this as well because this rule can be run

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -627,15 +627,15 @@ public PlanFactory visitJoinCommand(EsqlBaseParser.JoinCommandContext ctx) {
627627
}
628628

629629
return p -> {
630-
boolean[] hasRemotes = { false };
631-
p.forEachUp(UnresolvedRelation.class, r -> {
632-
for (var leftPattern : Strings.splitStringByCommaToArray(r.indexPattern().indexPattern())) {
633-
if (RemoteClusterAware.isRemoteIndexName(leftPattern)) {
634-
hasRemotes[0] = true;
635-
}
630+
boolean hasRemotes = p.anyMatch(node -> {
631+
if (node instanceof UnresolvedRelation r) {
632+
return Arrays.stream(Strings.splitStringByCommaToArray(r.indexPattern().indexPattern()))
633+
.anyMatch(RemoteClusterAware::isRemoteIndexName);
634+
} else {
635+
return false;
636636
}
637637
});
638-
return new LookupJoin(source, p, right, joinFields).setRemote(hasRemotes[0]);
638+
return new LookupJoin(source, p, right, joinFields, hasRemotes);
639639
};
640640
}
641641

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,13 @@ public class Join extends BinaryPlan implements PostAnalysisVerificationAware, S
8686
private transient boolean isRemote = false;
8787

8888
public Join(Source source, LogicalPlan left, LogicalPlan right, JoinConfig config) {
89+
this(source, left, right, config, false);
90+
}
91+
92+
public Join(Source source, LogicalPlan left, LogicalPlan right, JoinConfig config, boolean isRemote) {
8993
super(source, left, right);
9094
this.config = config;
95+
this.isRemote = isRemote;
9196
}
9297

9398
public Join(
@@ -235,17 +240,17 @@ public boolean resolved() {
235240
}
236241

237242
public Join withConfig(JoinConfig config) {
238-
return new Join(source(), left(), right(), config);
243+
return new Join(source(), left(), right(), config, isRemote);
239244
}
240245

241246
@Override
242247
public Join replaceChildren(LogicalPlan left, LogicalPlan right) {
243-
return new Join(source(), left, right, config);
248+
return new Join(source(), left, right, config, isRemote);
244249
}
245250

246251
@Override
247252
public int hashCode() {
248-
return Objects.hash(config, left(), right());
253+
return Objects.hash(config, left(), right(), isRemote);
249254
}
250255

251256
@Override
@@ -303,9 +308,4 @@ private static boolean comparableTypes(Attribute left, Attribute right) {
303308
public boolean isRemote() {
304309
return isRemote;
305310
}
306-
307-
public Join setRemote(boolean remote) {
308-
isRemote = remote;
309-
return this;
310-
}
311311
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/LookupJoin.java‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@
3232
*/
3333
public class LookupJoin extends Join implements SurrogateLogicalPlan, PostAnalysisVerificationAware, TelemetryAware {
3434

35-
public LookupJoin(Source source, LogicalPlan left, LogicalPlan right, List<Attribute> joinFields) {
36-
this(source, left, right, new UsingJoinType(LEFT, joinFields), emptyList(), emptyList(), emptyList());
35+
public LookupJoin(Source source, LogicalPlan left, LogicalPlan right, List<Attribute> joinFields, boolean isRemote) {
36+
this(source, left, right, new UsingJoinType(LEFT, joinFields), emptyList(), emptyList(), emptyList(), isRemote);
3737
}
3838

3939
public LookupJoin(
@@ -43,13 +43,14 @@ public LookupJoin(
4343
JoinType type,
4444
List<Attribute> joinFields,
4545
List<Attribute> leftFields,
46-
List<Attribute> rightFields
46+
List<Attribute> rightFields,
47+
boolean isRemote
4748
) {
48-
this(source, left, right, new JoinConfig(type, joinFields, leftFields, rightFields));
49+
this(source, left, right, new JoinConfig(type, joinFields, leftFields, rightFields), isRemote);
4950
}
5051

51-
public LookupJoin(Source source, LogicalPlan left, LogicalPlan right, JoinConfig joinConfig) {
52-
super(source, left, right, joinConfig);
52+
public LookupJoin(Source source, LogicalPlan left, LogicalPlan right, JoinConfig joinConfig, boolean isRemote) {
53+
super(source, left, right, joinConfig, isRemote);
5354
}
5455

5556
/**
@@ -58,12 +59,12 @@ public LookupJoin(Source source, LogicalPlan left, LogicalPlan right, JoinConfig
5859
@Override
5960
public LogicalPlan surrogate() {
6061
// TODO: decide whether to introduce USING or just basic ON semantics - keep the ordering out for now
61-
return new Join(source(), left(), right(), config()).setRemote(isRemote());
62+
return new Join(source(), left(), right(), config(), isRemote());
6263
}
6364

6465
@Override
6566
public Join replaceChildren(LogicalPlan left, LogicalPlan right) {
66-
return new LookupJoin(source(), left, right, config());
67+
return new LookupJoin(source(), left, right, config(), isRemote());
6768
}
6869

6970
@Override
@@ -76,7 +77,8 @@ protected NodeInfo<Join> info() {
7677
config().type(),
7778
config().matchFields(),
7879
config().leftFields(),
79-
config().rightFields()
80+
config().rightFields(),
81+
isRemote()
8082
);
8183
}
8284

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1244,7 +1244,7 @@ public void testPushdownLimitsPastLeftJoin() {
12441244
var joinConfig = new JoinConfig(JoinTypes.LEFT, List.of(), List.of(), List.of());
12451245
var join = switch (randomIntBetween(0, 2)) {
12461246
case 0 -> new Join(EMPTY, leftChild, rightChild, joinConfig);
1247-
case 1 -> new LookupJoin(EMPTY, leftChild, rightChild, joinConfig);
1247+
case 1 -> new LookupJoin(EMPTY, leftChild, rightChild, joinConfig, false);
12481248
case 2 -> new InlineJoin(EMPTY, leftChild, rightChild, joinConfig);
12491249
default -> throw new IllegalArgumentException();
12501250
};

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/CommandLicenseTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ private static LogicalPlan createInstance(Class<? extends LogicalPlan> clazz, Lo
164164
return new Sample(source, null, child);
165165
}
166166
case "LookupJoin" -> {
167-
return new LookupJoin(source, child, child, List.of());
167+
return new LookupJoin(source, child, child, List.of(), false);
168168
}
169169
case "Limit" -> {
170170
return new Limit(source, null, child);

0 commit comments

Comments
 (0)