Fix queries with missing index, skip_unavailable and filters (elastic#130344) (elastic#130369)

* Fix queries with missing index, skip_unavailable and filters

Author: smalyshev

docs/changelog/130344.yaml

@@ -0,0 +1,5 @@
+pr: 130344
+summary: "Fix queries with missing index, `skip_unavailable` and filters"
+area: ES|QL
+type: bug
+issues: []

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterQueryWithFiltersIT.java

@@ -313,13 +313,13 @@ public void testFilterWithMissingRemoteIndex() {
             new RangeQueryBuilder("@timestamp").from("2025-01-01").to("now")
         )) {
             count++;
-            // Local index missing
+            // Remote index missing
             VerificationException e = expectThrows(
                 VerificationException.class,
                 () -> runQuery("from cluster-a:missing", randomBoolean(), filter).close()
             );
             assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:missing]"));
-            // Local index missing + wildcards
+            // Remote index missing + wildcards
             // FIXME: planner does not catch this now, it should be VerificationException but for now it's runtime RemoteException
             var ie = expectThrows(
                 RemoteException.class,
@@ -352,6 +352,66 @@ public void testFilterWithMissingRemoteIndex() {
         }
     }
 
+    public void testFilterWithMissingRemoteIndexSkipUnavailable() {
+        int docsTest1 = 50;
+        int docsTest2 = 30;
+        int localShards = randomIntBetween(1, 5);
+        int remoteShards = randomIntBetween(1, 5);
+        populateDateIndex(LOCAL_CLUSTER, LOCAL_INDEX, localShards, docsTest1, "2024-11-26");
+        populateDateIndex(REMOTE_CLUSTER_1, REMOTE_INDEX, remoteShards, docsTest2, "2023-11-26");
+        setSkipUnavailable(REMOTE_CLUSTER_1, true);
+
+        int count = 0;
+        for (var filter : List.of(
+            new RangeQueryBuilder("@timestamp").from("2023-01-01").to("now"),
+            new RangeQueryBuilder("@timestamp").from("2024-01-01").to("now"),
+            new RangeQueryBuilder("@timestamp").from("2025-01-01").to("now")
+        )) {
+            count++;
+            try (EsqlQueryResponse resp = runQuery("from cluster-a:missing,logs-1", randomBoolean(), filter)) {
+                List<List<Object>> values = getValuesList(resp);
+                assertThat(values, hasSize(count > 2 ? 0 : docsTest1));
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertNotNull(executionInfo);
+                assertThat(executionInfo.isCrossClusterSearch(), is(true));
+                long overallTookMillis = executionInfo.overallTook().millis();
+                assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+
+                EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                assertClusterMetadataSkipped(remoteCluster, overallTookMillis, "missing");
+                assertThat(remoteCluster.getFailures(), hasSize(1));
+                var fail = remoteCluster.getFailures().get(0);
+                assertThat(fail.getCause().getMessage(), containsString("Unknown index [cluster-a:missing]"));
+            }
+
+            try (EsqlQueryResponse resp = runQuery("from cluster-a:missing,cluster-a:logs-*", randomBoolean(), filter)) {
+                List<List<Object>> values = getValuesList(resp);
+                assertThat(values, hasSize(0));
+                EsqlExecutionInfo executionInfo = resp.getExecutionInfo();
+                assertNotNull(executionInfo);
+                assertThat(executionInfo.isCrossClusterSearch(), is(true));
+                long overallTookMillis = executionInfo.overallTook().millis();
+                assertThat(overallTookMillis, greaterThanOrEqualTo(0L));
+
+                EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1);
+                assertClusterMetadataSkipped(remoteCluster, overallTookMillis, "missing,logs-*");
+                assertThat(remoteCluster.getFailures(), hasSize(1));
+                var fail = remoteCluster.getFailures().get(0);
+                assertThat(fail.getCause().getMessage(), containsString("no such index [missing]"));
+            }
+            // TODO: for now, these fail, but in the future may be skipped instead
+            // Remote index missing
+            VerificationException e = expectThrows(
+                VerificationException.class,
+                () -> runQuery("from cluster-a:missing", randomBoolean(), filter).close()
+            );
+            assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:missing]"));
+            // Wildcard index missing
+            e = expectThrows(VerificationException.class, () -> runQuery("from cluster-a:missing*", randomBoolean(), filter).close());
+            assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:missing*]"));
+        }
+    }
+
     private void checkRemoteFailures() {
         for (var filter : List.of(
             new RangeQueryBuilder("@timestamp").from("2024-01-01").to("now"),

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlCCSUtils.java

@@ -221,15 +221,15 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(
                     "Unknown index [%s]",
                     (c.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) ? indexExpression : c + ":" + indexExpression)
                 );
-                if (executionInfo.isSkipUnavailable(c) == false) {
+                if (executionInfo.isSkipUnavailable(c) == false || filter != null) {
                     if (fatalErrorMessage == null) {
                         fatalErrorMessage = error;
                     } else {
                         fatalErrorMessage += "; " + error;
                     }
                 }
                 if (filter == null) {
-                    // We check for filter since the filter may be the reason why the index is missing, and then it's ok
+                    // We check for filter since the filter may be the reason why the index is missing, and then we don't want to mark yet
                     markClusterWithFinalStateAndNoShards(
                         executionInfo,
                         c,