Skip to content

Commit 5e1bae4

Browse files
luigidellaquilaluigidellaquila
authored
ES|QL: add support for include_execution_metadata parameter (#134446)
1 parent 45a8766 commit 5e1bae4

File tree

9 files changed

+129
-22
lines changed

9 files changed

+129
-22
lines changed

‎docs/changelog/134446.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 134446
2+
summary: Add support for `include_execution_metadata` parameter
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ public boolean isRemoteClusterServerEnabled() {
8282
private final Map<ProjectId, Map<String, RemoteClusterConnection>> remoteClusters;
8383
private final RemoteClusterCredentialsManager remoteClusterCredentialsManager;
8484
private final ProjectResolver projectResolver;
85-
private final boolean canUseSkipUnavailable;
85+
private final boolean crossProjectEnabled;
8686

8787
RemoteClusterService(Settings settings, TransportService transportService, ProjectResolver projectResolver) {
8888
super(settings);
@@ -103,7 +103,7 @@ public boolean isRemoteClusterServerEnabled() {
103103
* TODO: This is not the right way to check if we're in CPS context and is more of a temporary measure since
104104
* the functionality to do it the right way is not yet ready -- replace this code when it's ready.
105105
*/
106-
this.canUseSkipUnavailable = settings.getAsBoolean("serverless.cross_project.enabled", false) == false;
106+
this.crossProjectEnabled = settings.getAsBoolean("serverless.cross_project.enabled", false);
107107
}
108108

109109
/**
@@ -220,13 +220,17 @@ void ensureConnected(String clusterAlias, ActionListener<Void> listener) {
220220
* it returns an empty value where we default/fall back to true.
221221
*/
222222
public Optional<Boolean> isSkipUnavailable(String clusterAlias) {
223-
if (canUseSkipUnavailable == false) {
223+
if (crossProjectEnabled) {
224224
return Optional.empty();
225225
} else {
226226
return Optional.of(getRemoteClusterConnection(clusterAlias).isSkipUnavailable());
227227
}
228228
}
229229

230+
public boolean crossProjectEnabled() {
231+
return crossProjectEnabled;
232+
}
233+
230234
/**
231235
* Signifies if an error can be skipped for the specified cluster based on skip_unavailable, or,
232236
* allow_partial_search_results if in CPS-like environment.

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractCrossClusterTestCase.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -350,7 +350,11 @@ protected EsqlQueryResponse runQuery(String query, Boolean ccsMetadataInResponse
350350
request.profile(randomInt(5) == 2);
351351
request.columnar(randomBoolean());
352352
if (ccsMetadataInResponse != null) {
353-
request.includeCCSMetadata(ccsMetadataInResponse);
353+
if (randomBoolean()) {
354+
request.includeExecutionMetadata(ccsMetadataInResponse);
355+
} else {
356+
request.includeCCSMetadata(ccsMetadataInResponse);
357+
}
354358
}
355359
return runQuery(request);
356360
}

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryIT.java‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,8 @@
5555
import static org.hamcrest.Matchers.lessThanOrEqualTo;
5656

5757
public class CrossClusterQueryIT extends AbstractCrossClusterTestCase {
58-
private static final String IDX_ALIAS = "alias1";
59-
private static final String FILTERED_IDX_ALIAS = "alias-filtered-1";
58+
protected static final String IDX_ALIAS = "alias1";
59+
protected static final String FILTERED_IDX_ALIAS = "alias-filtered-1";
6060

6161
@Override
6262
protected Map<String, Boolean> skipUnavailableForRemoteClusters() {
@@ -234,6 +234,10 @@ public void testSearchesAgainstIndicesWithNoMappingsSkipUnavailableTrue() throws
234234
}
235235

236236
public void testSearchesAgainstNonMatchingIndices() throws Exception {
237+
testSearchesAgainstNonMatchingIndices(true);
238+
}
239+
240+
protected void testSearchesAgainstNonMatchingIndices(boolean exceptionWithSkipUnavailableFalse) throws Exception {
237241
int numClusters = 3;
238242
Map<String, Object> testClusterInfo = setupClusters(numClusters);
239243
int localNumShards = (Integer) testClusterInfo.get("local.num_shards");
@@ -260,9 +264,11 @@ public void testSearchesAgainstNonMatchingIndices() throws Exception {
260264
{
261265
String q = "FROM logs*,cluster-a:nomatch";
262266
String expectedError = "Unknown index [cluster-a:nomatch]";
263-
setSkipUnavailable(REMOTE_CLUSTER_1, false);
264-
expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta);
265-
setSkipUnavailable(REMOTE_CLUSTER_1, true);
267+
if (exceptionWithSkipUnavailableFalse) {
268+
setSkipUnavailable(REMOTE_CLUSTER_1, false);
269+
expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta);
270+
setSkipUnavailable(REMOTE_CLUSTER_1, true);
271+
}
266272
try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) {
267273
assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1));
268274
EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
@@ -406,9 +412,11 @@ public void testSearchesAgainstNonMatchingIndices() throws Exception {
406412
String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS);
407413
String q = Strings.format("FROM %s*,cluster-a:nomatch,%s:%s*", localIndexName, REMOTE_CLUSTER_2, remote2IndexName);
408414
String expectedError = "Unknown index [cluster-a:nomatch]";
409-
setSkipUnavailable(REMOTE_CLUSTER_1, false);
410-
expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta);
411-
setSkipUnavailable(REMOTE_CLUSTER_1, true);
415+
if (exceptionWithSkipUnavailableFalse) {
416+
setSkipUnavailable(REMOTE_CLUSTER_1, false);
417+
expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta);
418+
setSkipUnavailable(REMOTE_CLUSTER_1, true);
419+
}
412420
try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) {
413421
assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1));
414422
EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
@@ -443,7 +451,7 @@ record ExpectedCluster(String clusterAlias, String indexExpression, EsqlExecutio
443451
* Runs the provided query, expecting a VerificationError. It then runs the same query with a "| LIMIT 0"
444452
* extra processing step to ensure that ESQL coordinator-only operations throw the same VerificationError.
445453
*/
446-
private void expectVerificationExceptionForQuery(String query, String error, Boolean requestIncludeMeta) {
454+
protected void expectVerificationExceptionForQuery(String query, String error, Boolean requestIncludeMeta) {
447455
VerificationException e = expectThrows(VerificationException.class, () -> runQuery(query, requestIncludeMeta));
448456
assertThat(e.getDetailedMessage(), containsString(error));
449457

@@ -1017,4 +1025,24 @@ public void testMultiTypes() throws Exception {
10171025
}
10181026
}
10191027
}
1028+
1029+
public void testNoBothIncludeCcsMetadataAndIncludeExecutionMetadata() throws Exception {
1030+
setupTwoClusters();
1031+
var query = "from logs-*,c*:logs-* | stats sum (v)";
1032+
EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest();
1033+
request.query(query);
1034+
request.pragmas(AbstractEsqlIntegTestCase.randomPragmas());
1035+
request.profile(randomInt(5) == 2);
1036+
request.columnar(randomBoolean());
1037+
request.includeCCSMetadata(randomBoolean());
1038+
request.includeExecutionMetadata(randomBoolean());
1039+
1040+
assertThat(
1041+
expectThrows(VerificationException.class, () -> runQuery(request)).getMessage(),
1042+
containsString(
1043+
"Both [include_execution_metadata] and [include_ccs_metadata] query parameters are set. "
1044+
+ "Use only [include_execution_metadata]"
1045+
)
1046+
);
1047+
}
10201048
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ public class EsqlQueryRequest extends org.elasticsearch.xpack.core.esql.action.E
4545
private String query;
4646
private boolean columnar;
4747
private boolean profile;
48-
private boolean includeCCSMetadata;
48+
private Boolean includeCCSMetadata;
49+
private Boolean includeExecutionMetadata;
4950
private Locale locale;
5051
private QueryBuilder filter;
5152
private QueryPragmas pragmas = new QueryPragmas(Settings.EMPTY);
@@ -134,14 +135,22 @@ public void profile(boolean profile) {
134135
this.profile = profile;
135136
}
136137

137-
public void includeCCSMetadata(boolean include) {
138+
public void includeCCSMetadata(Boolean include) {
138139
this.includeCCSMetadata = include;
139140
}
140141

141-
public boolean includeCCSMetadata() {
142+
public Boolean includeCCSMetadata() {
142143
return includeCCSMetadata;
143144
}
144145

146+
public void includeExecutionMetadata(Boolean include) {
147+
this.includeExecutionMetadata = include;
148+
}
149+
150+
public Boolean includeExecutionMetadata() {
151+
return includeExecutionMetadata;
152+
}
153+
145154
/**
146155
* Is profiling enabled?
147156
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ String fields() {
8080
private static final ParseField PROFILE_FIELD = new ParseField("profile");
8181
private static final ParseField ACCEPT_PRAGMA_RISKS = new ParseField("accept_pragma_risks");
8282
private static final ParseField INCLUDE_CCS_METADATA_FIELD = new ParseField("include_ccs_metadata");
83+
private static final ParseField INCLUDE_EXECUTION_METADATA_FIELD = new ParseField("include_execution_metadata");
8384
static final ParseField TABLES_FIELD = new ParseField("tables");
8485

8586
static final ParseField WAIT_FOR_COMPLETION_TIMEOUT = new ParseField("wait_for_completion_timeout");
@@ -105,6 +106,7 @@ private static void objectParserCommon(ObjectParser<EsqlQueryRequest, ?> parser)
105106
parser.declareObject(EsqlQueryRequest::filter, (p, c) -> AbstractQueryBuilder.parseTopLevelQuery(p), FILTER_FIELD);
106107
parser.declareBoolean(EsqlQueryRequest::acceptedPragmaRisks, ACCEPT_PRAGMA_RISKS);
107108
parser.declareBoolean(EsqlQueryRequest::includeCCSMetadata, INCLUDE_CCS_METADATA_FIELD);
109+
parser.declareBoolean(EsqlQueryRequest::includeExecutionMetadata, INCLUDE_EXECUTION_METADATA_FIELD);
108110
parser.declareObject(
109111
EsqlQueryRequest::pragmas,
110112
(p, c) -> new QueryPragmas(Settings.builder().loadFromMap(p.map()).build()),

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParser.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public static MediaType getResponseMediaType(RestRequest request, EsqlQueryReque
4545
var mediaType = getResponseMediaType(request, (MediaType) null);
4646
validateColumnarRequest(esqlRequest.columnar(), mediaType);
4747
validateIncludeCCSMetadata(esqlRequest.includeCCSMetadata(), mediaType);
48+
validateIncludeExecutionMetadata(esqlRequest.includeExecutionMetadata(), mediaType);
4849
validateProfile(esqlRequest.profile(), mediaType);
4950
return checkNonNullMediaType(mediaType, request);
5051
}
@@ -72,12 +73,18 @@ private static void validateColumnarRequest(boolean requestIsColumnar, MediaType
7273
}
7374
}
7475

75-
private static void validateIncludeCCSMetadata(boolean includeCCSMetadata, MediaType fromMediaType) {
76-
if (includeCCSMetadata && fromMediaType instanceof TextFormat) {
76+
private static void validateIncludeCCSMetadata(Boolean includeCCSMetadata, MediaType fromMediaType) {
77+
if (Boolean.TRUE.equals(includeCCSMetadata) && fromMediaType instanceof TextFormat) {
7778
throw invalid("include_ccs_metadata");
7879
}
7980
}
8081

82+
private static void validateIncludeExecutionMetadata(Boolean includeExecutionMetadata, MediaType fromMediaType) {
83+
if (Boolean.TRUE.equals(includeExecutionMetadata) && fromMediaType instanceof TextFormat) {
84+
throw invalid("include_execution_metadata");
85+
}
86+
}
87+
8188
private static void validateProfile(boolean profile, MediaType fromMediaType) {
8289
if (profile && fromMediaType instanceof TextFormat) {
8390
throw invalid("profile");

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -356,10 +356,19 @@ private EsqlExecutionInfo getOrCreateExecutionInfo(Task task, EsqlQueryRequest r
356356
}
357357

358358
private EsqlExecutionInfo createEsqlExecutionInfo(EsqlQueryRequest request) {
359-
return new EsqlExecutionInfo(
360-
clusterAlias -> remoteClusterService.isSkipUnavailable(clusterAlias).orElse(true),
361-
request.includeCCSMetadata()
362-
);
359+
if (request.includeCCSMetadata() != null && request.includeExecutionMetadata() != null) {
360+
throw new VerificationException(
361+
"Both [include_execution_metadata] and [include_ccs_metadata] query parameters are set. "
362+
+ "Use only [include_execution_metadata]"
363+
);
364+
}
365+
366+
Boolean includeCcsMetadata = request.includeExecutionMetadata();
367+
if (includeCcsMetadata == null) {
368+
// include_ccs_metadata is considered only if include_execution_metadata is not set
369+
includeCcsMetadata = Boolean.TRUE.equals(request.includeCCSMetadata());
370+
}
371+
return new EsqlExecutionInfo(clusterAlias -> remoteClusterService.isSkipUnavailable(clusterAlias).orElse(true), includeCcsMetadata);
363372
}
364373

365374
private EsqlQueryResponse toResponse(Task task, EsqlQueryRequest request, Configuration configuration, Result result) {

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/EsqlMediaTypeParserTests.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,18 @@ public void testIncludeCCSMetadataWithAcceptText() {
9393
);
9494
}
9595

96+
public void testIncludeExecutionMetadataWithAcceptText() {
97+
var accept = randomFrom("text/plain", "text/csv", "text/tab-separated-values");
98+
IllegalArgumentException e = expectThrows(
99+
IllegalArgumentException.class,
100+
() -> getResponseMediaType(reqWithAccept(accept), createCpsTestInstance(false, true, false))
101+
);
102+
assertEquals(
103+
"Invalid use of [include_execution_metadata] argument: cannot be used in combination with [txt, csv, tsv] formats",
104+
e.getMessage()
105+
);
106+
}
107+
96108
public void testColumnarWithParamText() {
97109
IllegalArgumentException e = expectThrows(
98110
IllegalArgumentException.class,
@@ -121,6 +133,26 @@ public void testIncludeCCSMetadataWithNonJSONMediaTypesInParams() {
121133
}
122134
}
123135

136+
public void testIncludeExecutionMetadataWithNonJSONMediaTypesInParams() {
137+
{
138+
RestRequest restRequest = reqWithParams(Map.of("format", randomFrom("txt", "csv", "tsv")));
139+
IllegalArgumentException e = expectThrows(
140+
IllegalArgumentException.class,
141+
() -> getResponseMediaType(restRequest, createCpsTestInstance(false, true, false))
142+
);
143+
assertEquals(
144+
"Invalid use of [include_execution_metadata] argument: cannot be used in combination with [txt, csv, tsv] formats",
145+
e.getMessage()
146+
);
147+
}
148+
{
149+
// check that no exception is thrown for the XContent types
150+
RestRequest restRequest = reqWithParams(Map.of("format", randomFrom("SMILE", "YAML", "CBOR", "JSON")));
151+
MediaType responseMediaType = getResponseMediaType(restRequest, createCpsTestInstance(true, true, false));
152+
assertNotNull(responseMediaType);
153+
}
154+
}
155+
124156
public void testProfileWithNonJSONMediaTypesInParams() {
125157
{
126158
RestRequest restRequest = reqWithParams(Map.of("format", randomFrom("txt", "csv", "tsv")));
@@ -180,4 +212,11 @@ protected EsqlQueryRequest createTestInstance(boolean columnar, boolean includeC
180212
request.profile(profile);
181213
return request;
182214
}
215+
216+
protected EsqlQueryRequest createCpsTestInstance(boolean columnar, boolean includeExecutionMetadata, boolean profile) {
217+
var request = createTestInstance(columnar);
218+
request.includeExecutionMetadata(includeExecutionMetadata);
219+
request.profile(profile);
220+
return request;
221+
}
183222
}

0 commit comments

Comments
 (0)