diff --git a/docs/changelog/130344.yaml b/docs/changelog/130344.yaml new file mode 100644 index 0000000000000..7584b9cd4f21f --- /dev/null +++ b/docs/changelog/130344.yaml @@ -0,0 +1,5 @@ +pr: 130344 +summary: "Fix queries with missing index, `skip_unavailable` and filters" +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryWithFiltersIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryWithFiltersIT.java index 91f1cc12851dc..ccea99b798d14 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryWithFiltersIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryWithFiltersIT.java @@ -313,13 +313,13 @@ public void testFilterWithMissingRemoteIndex() { new RangeQueryBuilder("@timestamp").from("2025-01-01").to("now") )) { count++; - // Local index missing + // Remote index missing VerificationException e = expectThrows( VerificationException.class, () -> runQuery("from cluster-a:missing", randomBoolean(), filter).close() ); assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:missing]")); - // Local index missing + wildcards + // Remote index missing + wildcards // FIXME: planner does not catch this now, it should be VerificationException but for now it's runtime RemoteException var ie = expectThrows( RemoteException.class, @@ -352,6 +352,66 @@ public void testFilterWithMissingRemoteIndex() { } } + public void testFilterWithMissingRemoteIndexSkipUnavailable() { + int docsTest1 = 50; + int docsTest2 = 30; + int localShards = randomIntBetween(1, 5); + int remoteShards = randomIntBetween(1, 5); + populateDateIndex(LOCAL_CLUSTER, LOCAL_INDEX, localShards, docsTest1, "2024-11-26"); + populateDateIndex(REMOTE_CLUSTER_1, REMOTE_INDEX, remoteShards, docsTest2, "2023-11-26"); + setSkipUnavailable(REMOTE_CLUSTER_1, true); + + int count = 0; + for (var filter : List.of( + new RangeQueryBuilder("@timestamp").from("2023-01-01").to("now"), + new RangeQueryBuilder("@timestamp").from("2024-01-01").to("now"), + new RangeQueryBuilder("@timestamp").from("2025-01-01").to("now") + )) { + count++; + try (EsqlQueryResponse resp = runQuery("from cluster-a:missing,logs-1", randomBoolean(), filter)) { + List> values = getValuesList(resp); + assertThat(values, hasSize(count > 2 ? 0 : docsTest1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertClusterMetadataSkipped(remoteCluster, overallTookMillis, "missing"); + assertThat(remoteCluster.getFailures(), hasSize(1)); + var fail = remoteCluster.getFailures().get(0); + assertThat(fail.getCause().getMessage(), containsString("Unknown index [cluster-a:missing]")); + } + + try (EsqlQueryResponse resp = runQuery("from cluster-a:missing,cluster-a:logs-*", randomBoolean(), filter)) { + List> values = getValuesList(resp); + assertThat(values, hasSize(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); + assertClusterMetadataSkipped(remoteCluster, overallTookMillis, "missing,logs-*"); + assertThat(remoteCluster.getFailures(), hasSize(1)); + var fail = remoteCluster.getFailures().get(0); + assertThat(fail.getCause().getMessage(), containsString("no such index [missing]")); + } + // TODO: for now, these fail, but in the future may be skipped instead + // Remote index missing + VerificationException e = expectThrows( + VerificationException.class, + () -> runQuery("from cluster-a:missing", randomBoolean(), filter).close() + ); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:missing]")); + // Wildcard index missing + e = expectThrows(VerificationException.class, () -> runQuery("from cluster-a:missing*", randomBoolean(), filter).close()); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:missing*]")); + } + } + private void checkRemoteFailures() { for (var filter : List.of( new RangeQueryBuilder("@timestamp").from("2024-01-01").to("now"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtils.java index 51cb9b35becb1..d507b8275178d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtils.java @@ -221,7 +221,7 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices( "Unknown index [%s]", (c.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) ? indexExpression : c + ":" + indexExpression) ); - if (executionInfo.isSkipUnavailable(c) == false) { + if (executionInfo.isSkipUnavailable(c) == false || filter != null) { if (fatalErrorMessage == null) { fatalErrorMessage = error; } else { @@ -229,7 +229,7 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices( } } if (filter == null) { - // We check for filter since the filter may be the reason why the index is missing, and then it's ok + // We check for filter since the filter may be the reason why the index is missing, and then we don't want to mark yet markClusterWithFinalStateAndNoShards( executionInfo, c,