diff --git a/muted-tests.yml b/muted-tests.yml index 26c9afc5a3ebc..daceba4dfddc9 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -309,9 +309,6 @@ tests: - class: org.elasticsearch.xpack.search.CrossClusterAsyncSearchIT method: testCCSClusterDetailsWhereAllShardsSkippedInCanMatch issue: https://github.com/elastic/elasticsearch/issues/128418 -- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithFiltersIT - method: testTimestampFilterFromQuery - issue: https://github.com/elastic/elasticsearch/issues/127332 - class: org.elasticsearch.xpack.esql.plugin.DataNodeRequestSenderIT method: testSearchWhileRelocating issue: https://github.com/elastic/elasticsearch/issues/128500 diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java index d9f54a8603735..af6915a717d3a 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java @@ -81,6 +81,7 @@ record Doc(int id, String color, long data) { final String lookupIndexLocal = "test-lookup-index-local"; final String lookupIndexRemote = "test-lookup-index-remote"; final String lookupAlias = "test-lookup-index"; + private Boolean shouldCheckShardCounts = null; @Before public void setUpIndices() throws Exception { @@ -211,6 +212,17 @@ private Map runEsql(RestEsqlTestCase.RequestObjectBuilder reques } } + private boolean checkShardCounts() { + if (shouldCheckShardCounts == null) { + try { + shouldCheckShardCounts = capabilitiesSupportedNewAndOld(List.of("correct_skipped_shard_count")); + } catch (IOException e) { + shouldCheckShardCounts = false; + } + } + return shouldCheckShardCounts; + } + private void assertResultMapWithCapabilities( boolean includeCCSMetadata, Map result, @@ -341,11 +353,16 @@ private void assertClusterDetailsMap(Map result, boolean remoteO assertThat( remoteClusterShards, matchesMap().entry("total", greaterThanOrEqualTo(0)) - .entry("successful", remoteClusterShards.get("total")) + .entry("successful", greaterThanOrEqualTo(0)) .entry("skipped", greaterThanOrEqualTo(0)) .entry("failed", 0) ); - + if (checkShardCounts()) { + assertThat( + (int) remoteClusterShards.get("successful") + (int) remoteClusterShards.get("skipped"), + equalTo(remoteClusterShards.get("total")) + ); + } if (remoteOnly == false) { @SuppressWarnings("unchecked") Map localCluster = (Map) details.get("(local)"); @@ -359,10 +376,16 @@ private void assertClusterDetailsMap(Map result, boolean remoteO assertThat( localClusterShards, matchesMap().entry("total", greaterThanOrEqualTo(0)) - .entry("successful", localClusterShards.get("total")) + .entry("successful", greaterThanOrEqualTo(0)) .entry("skipped", greaterThanOrEqualTo(0)) .entry("failed", 0) ); + if (checkShardCounts()) { + assertThat( + (int) localClusterShards.get("successful") + (int) localClusterShards.get("skipped"), + equalTo(localClusterShards.get("total")) + ); + } } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryWithFiltersIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryWithFiltersIT.java index ccea99b798d14..9f057621aabf7 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryWithFiltersIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryWithFiltersIT.java @@ -62,8 +62,10 @@ protected void assertClusterMetadata(EsqlExecutionInfo.Cluster clusterMetatata, protected void assertClusterMetadataSuccess(EsqlExecutionInfo.Cluster clusterMetatata, int shards, long took, String indexExpression) { assertClusterMetadata(clusterMetatata, took, indexExpression, Status.SUCCESSFUL); assertThat(clusterMetatata.getTotalShards(), equalTo(shards)); - assertThat(clusterMetatata.getSuccessfulShards(), equalTo(shards)); - assertThat(clusterMetatata.getSkippedShards(), equalTo(0)); + // We should have at least one successful shard for data + assertThat(clusterMetatata.getSuccessfulShards(), greaterThanOrEqualTo(1)); + // Some shards may be skipped, but total sum of the shards should match up + assertThat(clusterMetatata.getSkippedShards() + clusterMetatata.getSuccessfulShards(), equalTo(shards)); } protected void assertClusterMetadataNoShards(EsqlExecutionInfo.Cluster clusterMetatata, long took, String indexExpression) { @@ -81,7 +83,7 @@ protected void assertClusterMetadataSkippedShards( ) { assertClusterMetadata(clusterMetatata, took, indexExpression, Status.SUCCESSFUL); assertThat(clusterMetatata.getTotalShards(), equalTo(shards)); - assertThat(clusterMetatata.getSuccessfulShards(), equalTo(shards)); + assertThat(clusterMetatata.getSuccessfulShards(), equalTo(0)); assertThat(clusterMetatata.getSkippedShards(), equalTo(shards)); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 35f6d1af2e76f..24491f365890f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1294,7 +1294,12 @@ public enum Cap { /** * Support for the options field of CATEGORIZE. */ - CATEGORIZE_OPTIONS; + CATEGORIZE_OPTIONS, + + /** + * Support correct counting of skipped shards. + */ + CORRECT_SKIPPED_SHARDS_COUNT; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSender.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSender.java index 4409201606d0a..469a3fd0816c1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSender.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSender.java @@ -134,17 +134,20 @@ final void startComputeOnDataNodes(Set concreteIndices, Runnable runOnTa var computeListener = new ComputeListener( transportService.getThreadPool(), runOnTaskFailure, - listener.map( - completionInfo -> new ComputeResponse( + listener.map(completionInfo -> { + final int totalSkipShards = targetShards.skippedShards() + skippedShards.get(); + final int failedShards = shardFailures.size(); + final int successfulShards = targetShards.totalShards() - totalSkipShards - failedShards; + return new ComputeResponse( completionInfo, timeValueNanos(System.nanoTime() - startTimeInNanos), targetShards.totalShards(), - targetShards.totalShards() - shardFailures.size() - skippedShards.get(), - targetShards.skippedShards() + skippedShards.get(), - shardFailures.size(), + successfulShards, + totalSkipShards, + failedShards, selectFailures() - ) - ) + ); + }) ) ) { pendingShardIds.addAll(order(targetShards));