Skip to content

Commit d7d55f5

Browse files
smalyshevsmalyshev
committed
Enable testing remote metadata for ES|QL CCS (#116767)
* Enable testing remote metadata for CCS
1 parent 2e5be96 commit d7d55f5

File tree

4 files changed

+180
-4
lines changed

4 files changed

+180
-4
lines changed

‎x-pack/plugin/esql/qa/server/multi-clusters/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ apply plugin: 'elasticsearch.bwc-test'
1515
dependencies {
1616
javaRestTestImplementation project(xpackModule('esql:qa:testFixtures'))
1717
javaRestTestImplementation project(xpackModule('esql:qa:server'))
18+
javaRestTestImplementation project(xpackModule('esql'))
1819
}
1920

2021
def supportedVersion = bwcVersion -> {

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@
4545
import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled;
4646
import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.ENRICH_SOURCE_INDICES;
4747
import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
48+
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS;
49+
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2;
50+
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1;
51+
import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST;
4852
import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC;
4953
import static org.mockito.ArgumentMatchers.any;
5054
import static org.mockito.Mockito.doAnswer;
@@ -101,16 +105,25 @@ public MultiClusterSpecIT(
101105

102106
@Override
103107
protected void shouldSkipTest(String testName) throws IOException {
108+
boolean remoteMetadata = testCase.requiredCapabilities.contains(METADATA_FIELDS_REMOTE_TEST.capabilityName());
109+
if (remoteMetadata) {
110+
// remove the capability from the test to enable it
111+
testCase.requiredCapabilities = testCase.requiredCapabilities.stream()
112+
.filter(c -> c.equals("metadata_fields_remote_test") == false)
113+
.toList();
114+
}
104115
super.shouldSkipTest(testName);
105116
checkCapabilities(remoteClusterClient(), remoteFeaturesService(), testName, testCase);
106-
assumeFalse("can't test with _index metadata", hasIndexMetadata(testCase.query));
117+
// Do not run tests including "METADATA _index" unless marked with metadata_fields_remote_test,
118+
// because they may produce inconsistent results with multiple clusters.
119+
assumeFalse("can't test with _index metadata", (remoteMetadata == false) && hasIndexMetadata(testCase.query));
107120
assumeTrue(
108121
"Test " + testName + " is skipped on " + Clusters.oldVersion(),
109122
isEnabled(testName, instructions, Clusters.oldVersion())
110123
);
111-
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains("inlinestats"));
112-
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains("inlinestats_v2"));
113-
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains("join_planning_v1"));
124+
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName()));
125+
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName()));
126+
assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName()));
114127
}
115128

116129
private TestFeatureService remoteFeaturesService() throws IOException {
@@ -151,6 +164,9 @@ protected RestClient buildClient(Settings settings, HttpHost[] localHosts) throw
151164
return twoClients(localClient, remoteClient);
152165
}
153166

167+
// These indices are used in metadata tests so we want them on remote only for consistency
168+
public static final List<String> METADATA_INDICES = List.of("employees", "apps", "ul_logs");
169+
154170
/**
155171
* Creates a new mock client that dispatches every request to both the local and remote clusters, excluding _bulk and _query requests.
156172
* - '_bulk' requests are randomly sent to either the local or remote cluster to populate data. Some spec tests, such as AVG,
@@ -166,6 +182,8 @@ static RestClient twoClients(RestClient localClient, RestClient remoteClient) th
166182
String endpoint = request.getEndpoint();
167183
if (endpoint.startsWith("/_query")) {
168184
return localClient.performRequest(request);
185+
} else if (endpoint.endsWith("/_bulk") && METADATA_INDICES.stream().anyMatch(i -> endpoint.equals("/" + i + "/_bulk"))) {
186+
return remoteClient.performRequest(request);
169187
} else if (endpoint.endsWith("/_bulk") && ENRICH_SOURCE_INDICES.stream().noneMatch(i -> endpoint.equals("/" + i + "/_bulk"))) {
170188
return bulkClient.performRequest(request);
171189
} else {
@@ -203,6 +221,9 @@ static Request[] cloneRequests(Request orig, int numClones) throws IOException {
203221
return clones;
204222
}
205223

224+
/**
225+
* Convert FROM employees ... => FROM *:employees,employees
226+
*/
206227
static CsvSpecReader.CsvTestCase convertToRemoteIndices(CsvSpecReader.CsvTestCase testCase) {
207228
String query = testCase.query;
208229
String[] commands = query.split("\\|");

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata-remote.csv-spec‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
simpleKeep
2+
required_capability: metadata_fields
3+
required_capability: metadata_fields_remote_test
4+
from employees metadata _index, _version | sort _index desc, emp_no | limit 2 | keep emp_no, _index, _version;
5+
6+
emp_no:integer |_index:keyword |_version:long
7+
10001 |remote_cluster:employees |1
8+
10002 |remote_cluster:employees |1
9+
;
10+
11+
aliasWithSameName
12+
required_capability: metadata_fields
13+
required_capability: metadata_fields_remote_test
14+
from employees metadata _index, _version | sort _index desc, emp_no | limit 2 | eval _index = _index, _version = _version | keep emp_no, _index, _version;
15+
16+
emp_no:integer |_index:keyword |_version:long
17+
10001 |remote_cluster:employees |1
18+
10002 |remote_cluster:employees |1
19+
;
20+
21+
inComparison
22+
required_capability: metadata_fields
23+
required_capability: metadata_fields_remote_test
24+
from employees metadata _index, _version | sort emp_no | where _index == "remote_cluster:employees" | where _version == 1 | keep emp_no | limit 2;
25+
26+
emp_no:integer
27+
10001
28+
10002
29+
;
30+
31+
metaIndexInAggs
32+
required_capability: metadata_fields
33+
required_capability: metadata_fields_remote_test
34+
FROM employees METADATA _index, _id
35+
| STATS max = MAX(emp_no) BY _index | SORT _index;
36+
37+
max:integer |_index:keyword
38+
10100 |remote_cluster:employees
39+
;
40+
41+
metaIndexAliasedInAggs
42+
required_capability: metadata_fields
43+
required_capability: metadata_fields_remote_test
44+
from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i | SORT _i;
45+
46+
max:integer |_i:keyword
47+
10100 |remote_cluster:employees
48+
;
49+
50+
metaVersionInAggs
51+
required_capability: metadata_fields
52+
required_capability: metadata_fields_remote_test
53+
from employees metadata _version | stats min = min(emp_no) by _version;
54+
55+
min:integer |_version:long
56+
10001 |1
57+
;
58+
59+
metaVersionAliasedInAggs
60+
required_capability: metadata_fields
61+
required_capability: metadata_fields_remote_test
62+
from employees metadata _version | eval _v = _version | stats min = min(emp_no) by _v;
63+
64+
min:integer |_v:long
65+
10001 |1
66+
;
67+
68+
inAggsAndAsGroups
69+
required_capability: metadata_fields
70+
required_capability: metadata_fields_remote_test
71+
from employees metadata _index, _version | stats max = max(_version) by _index | SORT _index;
72+
73+
max:long |_index:keyword
74+
1 |remote_cluster:employees
75+
;
76+
77+
inAggsAndAsGroupsAliased
78+
required_capability: metadata_fields
79+
required_capability: metadata_fields_remote_test
80+
from employees metadata _index, _version | eval _i = _index, _v = _version | stats max = max(_v) by _i | SORT _i;
81+
82+
max:long |_i:keyword
83+
1 |remote_cluster:employees
84+
;
85+
86+
inFunction
87+
required_capability: metadata_fields
88+
required_capability: metadata_fields_remote_test
89+
from employees metadata _index, _version | sort emp_no | where length(_index) == length("remote_cluster:employees") | where abs(_version) == 1 | keep emp_no | limit 2;
90+
91+
emp_no:integer
92+
10001
93+
10002
94+
;
95+
96+
inArithmetics
97+
required_capability: metadata_fields
98+
required_capability: metadata_fields_remote_test
99+
from employees metadata _index, _version | eval i = _version + 2 | stats min = min(emp_no) by i;
100+
101+
min:integer |i:long
102+
10001 |3
103+
;
104+
105+
inSort
106+
required_capability: metadata_fields
107+
required_capability: metadata_fields_remote_test
108+
from employees metadata _index, _version | sort _version, _index desc, emp_no | keep emp_no, _version, _index | limit 2;
109+
110+
emp_no:integer |_version:long |_index:keyword
111+
10001 |1 |remote_cluster:employees
112+
10002 |1 |remote_cluster:employees
113+
;
114+
115+
withMvFunction
116+
required_capability: metadata_fields
117+
required_capability: metadata_fields_remote_test
118+
from employees metadata _version | eval i = mv_avg(_version) + 2 | stats min = min(emp_no) by i;
119+
120+
min:integer |i:double
121+
10001 |3.0
122+
;
123+
124+
overwritten
125+
required_capability: metadata_fields
126+
required_capability: metadata_fields_remote_test
127+
from employees metadata _index, _version | sort emp_no | eval _index = 3, _version = "version" | keep emp_no, _index, _version | limit 3;
128+
129+
emp_no:integer |_index:integer |_version:keyword
130+
10001 |3 |version
131+
10002 |3 |version
132+
10003 |3 |version
133+
;
134+
135+
multipleIndices
136+
required_capability: metadata_fields
137+
required_capability: metadata_fields_remote_test
138+
FROM ul_logs, apps METADATA _index, _version
139+
| WHERE id IN (13, 14) AND _version == 1
140+
| EVAL key = CONCAT(_index, "_", TO_STR(id))
141+
| SORT id, _index
142+
| KEEP id, _index, _version, key
143+
;
144+
145+
id:long |_index:keyword |_version:long |key:keyword
146+
13 |remote_cluster:apps |1 |remote_cluster:apps_13
147+
13 |remote_cluster:ul_logs |1 |remote_cluster:ul_logs_13
148+
14 |remote_cluster:apps |1 |remote_cluster:apps_14
149+
14 |remote_cluster:ul_logs |1 |remote_cluster:ul_logs_14
150+
151+
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -476,6 +476,9 @@ public enum Cap {
476476
ADD_LIMIT_INSIDE_MV_EXPAND,
477477

478478
DELAY_DEBUG_FN(Build.current().isSnapshot()),
479+
480+
/** Capability for remote metadata test */
481+
METADATA_FIELDS_REMOTE_TEST(false),
479482
/**
480483
* WIP on Join planning
481484
* - Introduce BinaryPlan and co

0 commit comments

Comments
 (0)