Fix ESQL index resolution

x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java

@@ -323,8 +323,13 @@ public void testInsufficientPrivilege() {
             () -> runESQLCommand("metadata1_read2", "FROM index-user1 | STATS sum=sum(value)")
         );
         logger.info("error", error);
-        assertThat(error.getMessage(), containsString("Unknown index [index-user1]"));
-        assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
+        assertThat(
+            error.getMessage(),
+            containsString(
+                "unauthorized for user [test-admin] run as [metadata1_read2] with effective roles [metadata1_read2] on indices [index-user1]"
+            )
+        );
+        assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN));
     }
 
     public void testIndexPatternErrorMessageComparison_ESQL_SearchDSL() throws Exception {

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlRestValidationTestCase.java

@@ -38,7 +38,6 @@ public abstract class EsqlRestValidationTestCase extends ESRestTestCase {
         aliasName + "*,inexistent*",
         "inexistent*," + aliasName };
     private static final String[] existentAliasWithoutWildcard = new String[] { aliasName + ",inexistent", "inexistent," + aliasName };
-    private static final String[] inexistentIndexNameWithWildcard = new String[] { "inexistent*", "inexistent1*,inexistent2*" };
     private static final String[] inexistentIndexNameWithoutWildcard = new String[] { "inexistent", "inexistent1,inexistent2" };
     private static final String createAlias = "{\"actions\":[{\"add\":{\"index\":\"" + indexName + "\",\"alias\":\"" + aliasName + "\"}}]}";
     private static final String removeAlias = "{\"actions\":[{\"remove\":{\"index\":\""
@@ -73,21 +72,20 @@ public void wipeTestData() throws IOException {
         }
     }
 
-    private String getInexistentIndexErrorMessage() {
-        return "\"reason\" : \"Found 1 problem\\nline 1:1: Unknown index ";
-    }
-
-    public void testInexistentIndexNameWithWildcard() throws IOException {
-        assertErrorMessages(inexistentIndexNameWithWildcard, getInexistentIndexErrorMessage(), 400);
-    }
-
     public void testInexistentIndexNameWithoutWildcard() throws IOException {
-        assertErrorMessages(inexistentIndexNameWithoutWildcard, getInexistentIndexErrorMessage(), 400);
+        for (String indexName : inexistentIndexNameWithoutWildcard) {
+            // TODO ES-13095 all from indexName should be reported as missing.
+            assertErrorMessage(
+                indexName,
+                "\"reason\" : \"Found 1 problem\\nline 1:1: Unknown index [" + clusterSpecificIndexName("inexistent"),
+                400
+            );
+        }
     }
 
     public void testExistentIndexWithoutWildcard() throws IOException {
         for (String indexName : existentIndexWithoutWildcard) {
-            assertErrorMessage(indexName, "\"reason\" : \"no such index [inexistent]\"", 404);
+            assertErrorMessage(indexName, "\"reason\" : \"Found 1 problem\\nline 1:1: Unknown index [inexistent]\"", 400);
         }
     }
 
@@ -99,19 +97,13 @@ public void testAlias() throws IOException {
         createAlias();
 
         for (String indexName : existentAliasWithoutWildcard) {
-            assertErrorMessage(indexName, "\"reason\" : \"no such index [inexistent]\"", 404);
+            assertErrorMessage(indexName, "\"reason\" : \"Found 1 problem\\nline 1:1: Unknown index [inexistent]\"", 400);
         }
         assertValidRequestOnIndices(existentAliasWithWildcard);
 
         deleteAlias();
     }
 
-    private void assertErrorMessages(String[] indices, String errorMessage, int statusCode) throws IOException {
-        for (String indexName : indices) {
-            assertErrorMessage(indexName, errorMessage + "[" + clusterSpecificIndexName(indexName) + "]", statusCode);
-        }
-    }
-
     protected String clusterSpecificIndexName(String indexName) {
         return indexName;
     }

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java

@@ -85,15 +85,11 @@ public void testTimestampFilterFromQuery() throws IOException {
             allOf(instanceOf(List.class), hasSize(docsTest1))
         );
 
-        // filter excludes both indices (no rows); the first analysis step fails because there are no columns, a second attempt succeeds
-        // after eliminating the index filter. All columns are returned.
+        // filter excludes both indices (no rows); Empty result set is derived.
         builder = timestampFilter("gte", "2025-01-01").query(from("test*"));
         assertQueryResult(
             runEsql(builder),
-            matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date"))
-                .item(matchesMap().entry("name", "id1").entry("type", "integer"))
-                .item(matchesMap().entry("name", "id2").entry("type", "integer"))
-                .item(matchesMap().entry("name", "value").entry("type", "long")),
+            matchesList().item(matchesMap().entry("name", "<no-fields>").entry("type", "null")),
             allOf(instanceOf(List.class), hasSize(0))
         );
     }
@@ -206,15 +202,10 @@ public void testIndicesDontExist() throws IOException {
         assertThat(e.getMessage(), containsString("verification_exception"));
         assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo]"), containsString("Unknown index [remote_cluster:foo]")));
 
-        e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("foo*"))));
+        e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo,test1")));
         assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
         assertThat(e.getMessage(), containsString("verification_exception"));
-        assertThat(e.getMessage(), anyOf(containsString("Unknown index [foo*]"), containsString("Unknown index [remote_cluster:foo*]")));
-
-        e = expectThrows(ResponseException.class, () -> runEsql(timestampFilter("gte", "2020-01-01").query("FROM foo, test1")));
-        assertEquals(404, e.getResponse().getStatusLine().getStatusCode());
-        assertThat(e.getMessage(), containsString("index_not_found_exception"));
-        assertThat(e.getMessage(), containsString("no such index [foo]"));
+        assertThat(e.getMessage(), containsString("Unknown index [foo,test1]"));// TODO why test1 is reported?
 
         // Don't test remote patterns here, we'll test them in the multi-cluster tests
         if (EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()) {

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/IndexResolutionIT.java

@@ -17,7 +17,6 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
-import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.plugins.Plugin;
@@ -112,23 +111,20 @@ public void testResolvesExclusionPattern() {
             assertOk(response);
             assertResultConcreteIndices(response, "index-1");// excludes pattern from pattern
         }
-        expectThrows(
-            VerificationException.class,
-            containsString("Unknown index [index-*,-*]"),
-            () -> run(syncEsqlQueryRequest().query("FROM index-*,-* METADATA _index")) // exclude all resolves to empty
-        );
+        try (var response = run(syncEsqlQueryRequest().query("FROM index-*,-* METADATA _index"))) {
+            assertOk(response);
+            assertResultConcreteIndices(response);
+        }
     }
 
     public void testDoesNotResolveEmptyPattern() {
         assertAcked(client().admin().indices().prepareCreate("data"));
         indexRandom(true, "data", 1);
 
-        expectThrows(
-            VerificationException.class,
-            containsString("Unknown index [index-*]"),
-            () -> run(syncEsqlQueryRequest().query("FROM index-* METADATA _index"))
-        );
-
+        try (var response = run(syncEsqlQueryRequest().query("FROM index-* METADATA _index"))) {
+            assertOk(response);
+            assertResultConcreteIndices(response);
+        }
         try (var response = run(syncEsqlQueryRequest().query("FROM data,index-* METADATA _index"))) {
             assertOk(response);
             assertResultConcreteIndices(response, "data");
@@ -224,21 +220,20 @@ public void testUnavailableIndex() {
 
     public void testPartialResolution() {
         assertAcked(client().admin().indices().prepareCreate("index-1"));
-        assertAcked(client().admin().indices().prepareCreate("index-2"));
-        indexRandom(true, "index-2", 10);
-
-        try (var response = run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1"))) {
-            assertOk(response); // okay when present index is empty
+        if (randomBoolean()) {
+            indexRandom(true, "index-1", 1);
         }
+
         expectThrows(
-            IndexNotFoundException.class,
-            equalTo("no such index [nonexisting-1]"), // fails when present index is non-empty
-            () -> run(syncEsqlQueryRequest().query("FROM index-2,nonexisting-1"))
+            VerificationException.class,
+            containsString("Unknown index [nonexisting-1]"),
+            () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1"))
         );
         expectThrows(
-            IndexNotFoundException.class,
-            equalTo("no such index [nonexisting-1]"), // only the first missing index is reported
-            () -> run(syncEsqlQueryRequest().query("FROM index-2,nonexisting-1,nonexisting-2"))
+            VerificationException.class,
+            // TODO ES-13095 both [nonexisting-1,nonexisting-2] missing should be reported as missing.
+            containsString("Unknown index [nonexisting-1]"), // only the first missing index is reported
+            () -> run(syncEsqlQueryRequest().query("FROM index-1,nonexisting-1,nonexisting-2"))
         );
     }
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java

@@ -41,6 +41,10 @@ public static IndexResolution invalid(String invalid) {
         return new IndexResolution(null, invalid, Set.of(), Map.of());
     }
 
+    public static IndexResolution empty(String indexPattern) {
+        return valid(new EsIndex(indexPattern, Map.of(), Map.of()));
+    }
+
     public static IndexResolution notFound(String name) {
         Objects.requireNonNull(name, "name must not be null");
         return invalid("Unknown index [" + name + "]");

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java

@@ -696,9 +696,7 @@ private void preAnalyzeMainIndices(
             String indexExpressionToResolve = EsqlCCSUtils.createIndexExpressionFromAvailableClusters(executionInfo);
             if (indexExpressionToResolve.isEmpty()) {
                 // if this was a pure remote CCS request (no local indices) and all remotes are offline, return an empty IndexResolution
-                listener.onResponse(
-                    result.withIndices(IndexResolution.valid(new EsIndex(preAnalysis.indexPattern().indexPattern(), Map.of(), Map.of())))
-                );
+                listener.onResponse(result.withIndices(IndexResolution.empty(preAnalysis.indexPattern().indexPattern())));
             } else {
                 indexResolver.resolveAsMergedMapping(
                     indexExpressionToResolve,

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -56,7 +57,7 @@ public class IndexResolver {
     public static final String UNMAPPED = "unmapped";
 
     public static final IndicesOptions FIELD_CAPS_INDICES_OPTIONS = IndicesOptions.builder()
-        .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
+        .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
         .wildcardOptions(
             IndicesOptions.WildcardOptions.builder()
                 .matchOpen(true)
@@ -91,10 +92,17 @@ public void resolveAsMergedMapping(
         client.execute(
             EsqlResolveFieldsAction.TYPE,
             createFieldCapsRequest(indexWildcard, fieldNames, requestFilter, includeAllDimensions),
-            listener.delegateFailureAndWrap(
-                (l, response) -> l.onResponse(
-                    mergedMappings(indexWildcard, new FieldsInfo(response, supportsAggregateMetricDouble, supportsDenseVector))
-                )
+            ActionListener.wrap(
+                r -> listener.onResponse(
+                    mergedMappings(indexWildcard, new FieldsInfo(r, supportsAggregateMetricDouble, supportsDenseVector))
+                ),
+                f -> {
+                    if (f instanceof IndexNotFoundException e) {
+                        listener.onResponse(IndexResolution.notFound(e.getIndex().getName()));
+                    } else {
+                        listener.onFailure(f);
+                    }
+                }
             )
         );
     }
@@ -106,7 +114,7 @@ public static IndexResolution mergedMappings(String indexPattern, FieldsInfo fie
         assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker
         int numberOfIndices = fieldsInfo.caps.getIndexResponses().size();
         if (numberOfIndices == 0) {
-            return IndexResolution.notFound(indexPattern);
+            return IndexResolution.empty(indexPattern);
         }
 
         // For each field name, store a list of the field caps responses from each index