Skip to content

Commit 9d8504f

Browse files
smalyshevsmalyshev
authored
[8.x] Add CCS tests for index filtering (#119619) (#119716)
* Add CCS tests for index filtering (#119619) * Add CCS tests for index filtering See also: #116755 * Don't run the test on pre-8.18
1 parent d521232 commit 9d8504f

File tree

2 files changed

+129
-26
lines changed

2 files changed

+129
-26
lines changed

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.esql.ccq;
9+
10+
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
11+
12+
import org.apache.http.HttpHost;
13+
import org.elasticsearch.Version;
14+
import org.elasticsearch.client.Request;
15+
import org.elasticsearch.client.ResponseException;
16+
import org.elasticsearch.client.RestClient;
17+
import org.elasticsearch.core.IOUtils;
18+
import org.elasticsearch.test.TestClustersThreadFilter;
19+
import org.elasticsearch.test.cluster.ElasticsearchCluster;
20+
import org.elasticsearch.xpack.esql.qa.rest.RequestIndexFilteringTestCase;
21+
import org.junit.After;
22+
import org.junit.AfterClass;
23+
import org.junit.Before;
24+
import org.junit.BeforeClass;
25+
import org.junit.ClassRule;
26+
import org.junit.rules.RuleChain;
27+
import org.junit.rules.TestRule;
28+
29+
import java.io.IOException;
30+
31+
@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
32+
public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase {
33+
34+
static ElasticsearchCluster remoteCluster = Clusters.remoteCluster();
35+
static ElasticsearchCluster localCluster = Clusters.localCluster(remoteCluster);
36+
37+
@ClassRule
38+
public static TestRule clusterRule = RuleChain.outerRule(remoteCluster).around(localCluster);
39+
private static RestClient remoteClient;
40+
41+
@Override
42+
protected String getTestRestCluster() {
43+
return localCluster.getHttpAddresses();
44+
}
45+
46+
@Before
47+
public void setRemoteClient() throws IOException {
48+
if (remoteClient == null) {
49+
var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses());
50+
remoteClient = buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0]));
51+
}
52+
}
53+
54+
@BeforeClass
55+
public static void checkVersion() {
56+
assumeTrue("skip if version before 8.18", Clusters.localClusterVersion().onOrAfter(Version.V_8_18_0));
57+
}
58+
59+
@AfterClass
60+
public static void closeRemoteClients() throws IOException {
61+
try {
62+
IOUtils.close(remoteClient);
63+
} finally {
64+
remoteClient = null;
65+
}
66+
}
67+
68+
@Override
69+
protected void indexTimestampData(int docs, String indexName, String date, String differentiatorFieldName) throws IOException {
70+
indexTimestampDataForClient(client(), docs, indexName, date, differentiatorFieldName);
71+
indexTimestampDataForClient(remoteClient, docs, indexName, date, differentiatorFieldName);
72+
}
73+
74+
@Override
75+
protected String from(String... indexName) {
76+
if (randomBoolean()) {
77+
return "FROM *:" + String.join(",*:", indexName);
78+
} else {
79+
return "FROM " + String.join(",", indexName);
80+
}
81+
}
82+
83+
@After
84+
public void wipeRemoteTestData() throws IOException {
85+
try {
86+
var response = remoteClient.performRequest(new Request("DELETE", "/test*"));
87+
assertEquals(200, response.getStatusLine().getStatusCode());
88+
} catch (ResponseException re) {
89+
assertEquals(404, re.getResponse().getStatusLine().getStatusCode());
90+
}
91+
}
92+
}

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java‎

Lines changed: 37 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.client.Request;
1212
import org.elasticsearch.client.Response;
1313
import org.elasticsearch.client.ResponseException;
14+
import org.elasticsearch.client.RestClient;
1415
import org.elasticsearch.test.rest.ESRestTestCase;
1516
import org.elasticsearch.xcontent.XContentType;
1617
import org.elasticsearch.xpack.esql.AssertWarnings;
@@ -30,12 +31,13 @@
3031
import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.entityToMap;
3132
import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.requestObjectBuilder;
3233
import static org.hamcrest.Matchers.allOf;
34+
import static org.hamcrest.Matchers.anyOf;
3335
import static org.hamcrest.Matchers.containsString;
34-
import static org.hamcrest.Matchers.equalTo;
3536
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
3637
import static org.hamcrest.Matchers.hasSize;
3738
import static org.hamcrest.Matchers.instanceOf;
3839
import static org.hamcrest.Matchers.nullValue;
40+
import static org.hamcrest.Matchers.oneOf;
3941

4042
public abstract class RequestIndexFilteringTestCase extends ESRestTestCase {
4143

@@ -49,14 +51,18 @@ public void wipeTestData() throws IOException {
4951
}
5052
}
5153

54+
protected String from(String... indexName) {
55+
return "FROM " + String.join(",", indexName);
56+
}
57+
5258
public void testTimestampFilterFromQuery() throws IOException {
5359
int docsTest1 = 50;
5460
int docsTest2 = 30;
5561
indexTimestampData(docsTest1, "test1", "2024-11-26", "id1");
5662
indexTimestampData(docsTest2, "test2", "2023-11-26", "id2");
5763

5864
// filter includes both indices in the result (all columns, all rows)
59-
RestEsqlTestCase.RequestObjectBuilder builder = timestampFilter("gte", "2023-01-01").query("FROM test*");
65+
RestEsqlTestCase.RequestObjectBuilder builder = timestampFilter("gte", "2023-01-01").query(from("test*"));
6066
Map<String, Object> result = runEsql(builder);
6167
assertMap(
6268
result,
@@ -70,7 +76,7 @@ public void testTimestampFilterFromQuery() throws IOException {
7076
);
7177

7278
// filter includes only test1. Columns from test2 are filtered out, as well (not only rows)!
73-
builder = timestampFilter("gte", "2024-01-01").query("FROM test*");
79+
builder = timestampFilter("gte", "2024-01-01").query(from("test*"));
7480
assertMap(
7581
runEsql(builder),
7682
matchesMap().entry(
@@ -83,7 +89,7 @@ public void testTimestampFilterFromQuery() throws IOException {
8389

8490
// filter excludes both indices (no rows); the first analysis step fails because there are no columns, a second attempt succeeds
8591
// after eliminating the index filter. All columns are returned.
86-
builder = timestampFilter("gte", "2025-01-01").query("FROM test*");
92+
builder = timestampFilter("gte", "2025-01-01").query(from("test*"));
8793
assertMap(
8894
runEsql(builder),
8995
matchesMap().entry(
@@ -103,7 +109,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException {
103109
indexTimestampData(docsTest2, "test2", "2023-11-26", "id2");
104110

105111
// filter includes only test1. Columns and rows of test2 are filtered out
106-
RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query("FROM test*");
112+
RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query(from("test*"));
107113
Map<String, Object> result = runEsql(builder);
108114
assertMap(
109115
result,
@@ -116,7 +122,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException {
116122
);
117123

118124
// filter includes only test1. Columns from test2 are filtered out, as well (not only rows)!
119-
builder = existsFilter("id1").query("FROM test* METADATA _index | KEEP _index, id*");
125+
builder = existsFilter("id1").query(from("test*") + " METADATA _index | KEEP _index, id*");
120126
result = runEsql(builder);
121127
assertMap(
122128
result,
@@ -129,7 +135,7 @@ public void testFieldExistsFilter_KeepWildcard() throws IOException {
129135
@SuppressWarnings("unchecked")
130136
var values = (List<List<Object>>) result.get("values");
131137
for (List<Object> row : values) {
132-
assertThat(row.get(0), equalTo("test1"));
138+
assertThat(row.get(0), oneOf("test1", "remote_cluster:test1"));
133139
assertThat(row.get(1), instanceOf(Integer.class));
134140
}
135141
}
@@ -142,7 +148,7 @@ public void testFieldExistsFilter_With_ExplicitUseOfDiscardedIndexFields() throw
142148

143149
// test2 is explicitly used in a query with "SORT id2" even if the index filter should discard test2
144150
RestEsqlTestCase.RequestObjectBuilder builder = existsFilter("id1").query(
145-
"FROM test* METADATA _index | SORT id2 | KEEP _index, id*"
151+
from("test*") + " METADATA _index | SORT id2 | KEEP _index, id*"
146152
);
147153
Map<String, Object> result = runEsql(builder);
148154
assertMap(
@@ -157,7 +163,7 @@ public void testFieldExistsFilter_With_ExplicitUseOfDiscardedIndexFields() throw
157163
@SuppressWarnings("unchecked")
158164
var values = (List<List<Object>>) result.get("values");
159165
for (List<Object> row : values) {
160-
assertThat(row.get(0), equalTo("test1"));
166+
assertThat(row.get(0), oneOf("test1", "remote_cluster:test1"));
161167
assertThat(row.get(1), instanceOf(Integer.class));
162168
assertThat(row.get(2), nullValue());
163169
}
@@ -172,59 +178,59 @@ public void testFieldNameTypo() throws IOException {
172178
// idx field name is explicitly used, though it doesn't exist in any of the indices. First test - without filter
173179
ResponseException e = expectThrows(
174180
ResponseException.class,
175-
() -> runEsql(requestObjectBuilder().query("FROM test* | WHERE idx == 123"))
181+
() -> runEsql(requestObjectBuilder().query(from("test*") + " | WHERE idx == 123"))
176182
);
177183
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
178184
assertThat(e.getMessage(), containsString("verification_exception"));
179185
assertThat(e.getMessage(), containsString("Found 1 problem"));
180-
assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]"));
186+
assertThat(e.getMessage(), containsString("Unknown column [idx]"));
181187

182-
e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query("FROM test1 | WHERE idx == 123")));
188+
e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query(from("test1") + " | WHERE idx == 123")));
183189
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
184190
assertThat(e.getMessage(), containsString("verification_exception"));
185191
assertThat(e.getMessage(), containsString("Found 1 problem"));
186-
assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]"));
192+
assertThat(e.getMessage(), containsString("Unknown column [idx]"));
187193

188194
e = expectThrows(
189195
ResponseException.class,
190-
() -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test* | WHERE idx == 123"))
196+
() -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test*") + " | WHERE idx == 123"))
191197
);
192198
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
193199
assertThat(e.getMessage(), containsString("Found 1 problem"));
194-
assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]"));
200+
assertThat(e.getMessage(), containsString("Unknown column [idx]"));
195201

196202
e = expectThrows(
197203
ResponseException.class,
198-
() -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test2 | WHERE idx == 123"))
204+
() -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test2") + " | WHERE idx == 123"))
199205
);
200206
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
201207
assertThat(e.getMessage(), containsString("Found 1 problem"));
202-
assertThat(e.getMessage(), containsString("line 1:20: Unknown column [idx]"));
208+
assertThat(e.getMessage(), containsString("Unknown column [idx]"));
203209
}
204210

205211
public void testIndicesDontExist() throws IOException {
206212
int docsTest1 = 0; // we are interested only in the created index, not necessarily that it has data
207213
indexTimestampData(docsTest1, "test1", "2024-11-26", "id1");
208214

209-
ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo")));
215+
ResponseException e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo"))));
210216
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
211217
assertThat(e.getMessage(), containsString("verification_exception"));
212-
assertThat(e.getMessage(), containsString("Unknown index [foo]"));
218+
assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo]"), containsString("Unknown index [remote_cluster:foo]")));
213219

214-
e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo*")));
220+
e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*"))));
215221
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
216222
assertThat(e.getMessage(), containsString("verification_exception"));
217-
assertThat(e.getMessage(), containsString("Unknown index [foo*]"));
223+
assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo*]"), containsString("Unknown index [remote_cluster:foo*]")));
218224

219-
e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo,test1")));
225+
e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo", "test1"))));
220226
assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
221227
assertThat(e.getMessage(), containsString("index_not_found_exception"));
222-
assertThat(e.getMessage(), containsString("no such index [foo]"));
228+
assertThat(e.getMessage(), anyOf(containsString("no such index [foo]"), containsString("no such index [remote_cluster:foo]")));
223229

224230
if (EsqlCapabilities.Cap.JOIN_LOOKUP_V10.isEnabled()) {
225231
e = expectThrows(
226232
ResponseException.class,
227-
() -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM test1 | LOOKUP JOIN foo ON id1"))
233+
() -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test1") + " | LOOKUP JOIN foo ON id1"))
228234
);
229235
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
230236
assertThat(e.getMessage(), containsString("verification_exception"));
@@ -251,6 +257,11 @@ public Map<String, Object> runEsql(RestEsqlTestCase.RequestObjectBuilder request
251257
}
252258

253259
protected void indexTimestampData(int docs, String indexName, String date, String differentiatorFieldName) throws IOException {
260+
indexTimestampDataForClient(client(), docs, indexName, date, differentiatorFieldName);
261+
}
262+
263+
protected void indexTimestampDataForClient(RestClient client, int docs, String indexName, String date, String differentiatorFieldName)
264+
throws IOException {
254265
Request createIndex = new Request("PUT", indexName);
255266
createIndex.setJsonEntity("""
256267
{
@@ -273,7 +284,7 @@ protected void indexTimestampData(int docs, String indexName, String date, Strin
273284
}
274285
}
275286
}""".replace("%differentiator_field_name%", differentiatorFieldName));
276-
Response response = client().performRequest(createIndex);
287+
Response response = client.performRequest(createIndex);
277288
assertThat(
278289
entityToMap(response.getEntity(), XContentType.JSON),
279290
matchesMap().entry("shards_acknowledged", true).entry("index", indexName).entry("acknowledged", true)
@@ -291,7 +302,7 @@ protected void indexTimestampData(int docs, String indexName, String date, Strin
291302
bulk.addParameter("refresh", "true");
292303
bulk.addParameter("filter_path", "errors");
293304
bulk.setJsonEntity(b.toString());
294-
response = client().performRequest(bulk);
305+
response = client.performRequest(bulk);
295306
Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8));
296307
}
297308
}

0 commit comments

Comments
 (0)