Index resolution error

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

@@ -38,6 +38,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.injection.guice.Inject;
@@ -157,21 +158,23 @@ private void doExecuteForked(
         final Map<String, OriginalIndices> remoteClusterIndices = transportService.getRemoteClusterService()
             .groupIndices(request.indicesOptions(), request.indices());
         final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
-        final String[] concreteIndices;
-        if (localIndices == null) {
+        final FailureCollector indexFailures = new FailureCollector();
+        final String[] concreteIndices = localIndices == null
             // in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote indices
-            concreteIndices = Strings.EMPTY_ARRAY;
-        } else {
-            concreteIndices = indexNameExpressionResolver.concreteIndexNames(projectState.metadata(), localIndices);
-        }
+            ? Strings.EMPTY_ARRAY
+            : indexNameExpressionResolver.concreteIndexNames(
+                projectState.metadata(),
+                localIndices,
+                expr -> indexFailures.collect(expr, new IndexNotFoundException(expr))
+            );
 
         if (concreteIndices.length == 0 && remoteClusterIndices.isEmpty()) {
             listener.onResponse(new FieldCapabilitiesResponse(new String[0], Collections.emptyMap()));
             return;
         }
 
         checkIndexBlocks(projectState, concreteIndices);
-        final FailureCollector indexFailures = new FailureCollector();
+
         final Map<String, FieldCapabilitiesIndexResponse> indexResponses = new HashMap<>();
         // This map is used to share the index response for indices which have the same index mapping hash to reduce the memory usage.
         final Map<String, FieldCapabilitiesIndexResponse> indexMappingHashToResponses = new HashMap<>();

server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java

@@ -64,10 +64,10 @@ public static IndicesOptions.Builder builder(IndicesOptions indicesOptions) {
      * Controls the way the target indices will be handled.
      * @param allowUnavailableTargets, if false when any of the concrete targets requested does not exist, throw an error
      */
-    public record ConcreteTargetOptions(boolean allowUnavailableTargets) implements ToXContentFragment {
+    public record ConcreteTargetOptions(boolean allowUnavailableTargets, boolean reportFailuresToResolve) implements ToXContentFragment {
         public static final String IGNORE_UNAVAILABLE = "ignore_unavailable";
-        public static final ConcreteTargetOptions ALLOW_UNAVAILABLE_TARGETS = new ConcreteTargetOptions(true);
-        public static final ConcreteTargetOptions ERROR_WHEN_UNAVAILABLE_TARGETS = new ConcreteTargetOptions(false);
+        public static final ConcreteTargetOptions ALLOW_UNAVAILABLE_TARGETS = new ConcreteTargetOptions(true, false);
+        public static final ConcreteTargetOptions ERROR_WHEN_UNAVAILABLE_TARGETS = new ConcreteTargetOptions(false, false);
 
         public static ConcreteTargetOptions fromParameter(Object ignoreUnavailableString, ConcreteTargetOptions defaultOption) {
             if (ignoreUnavailableString == null && defaultOption != null) {
@@ -1295,7 +1295,7 @@ public static IndicesOptions fromXContent(XContentParser parser, @Nullable Indic
             );
         }
         return IndicesOptions.builder()
-            .concreteTargetOptions(new ConcreteTargetOptions(ignoreUnavailable))
+            .concreteTargetOptions(new ConcreteTargetOptions(ignoreUnavailable, false))
             .wildcardOptions(wildcards)
             .gatekeeperOptions(generalOptions)
             .build();

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -60,6 +60,7 @@
 import java.util.SortedMap;
 import java.util.function.BiFunction;
 import java.util.function.BiPredicate;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
 import java.util.function.Predicate;
@@ -151,6 +152,23 @@ public String[] concreteIndexNames(ProjectMetadata project, IndicesRequest reque
         return concreteIndexNames(context, request.indices());
     }
 
+    public String[] concreteIndexNames(ProjectMetadata project, IndicesRequest request, Consumer<String> onResolutionFailure) {
+        Context context = new Context(
+            project,
+            request.indicesOptions(),
+            System.currentTimeMillis(),
+            false,
+            false,
+            request.includeDataStreams(),
+            false,
+            getSystemIndexAccessLevel(),
+            getSystemIndexAccessPredicate(),
+            getNetNewSystemIndexPredicate(),
+            onResolutionFailure
+        );
+        return concreteIndexNames(context, request.indices());
+    }
+
     /**
      * Same as {@link #concreteIndexNames(ClusterState, IndicesRequest)}, but access to system indices is always allowed.
      */
@@ -450,7 +468,10 @@ protected static Collection<ResolvedExpression> resolveExpressionsToResources(Co
                     resources.remove(new ResolvedExpression(baseExpression, selector));
                 } else if (ensureAliasOrIndexExists(context, baseExpression, selector)) {
                     resources.add(new ResolvedExpression(baseExpression, selector));
-                }
+                } else if (context.getOptions().concreteTargetOptions().reportFailuresToResolve()
+                    && context.onResolutionFailure() != null) {
+                        context.onResolutionFailure().accept(baseExpression);
+                    }
             }
         }
         return resources;
@@ -592,7 +613,8 @@ public Index[] concreteIndices(ProjectMetadata project, IndicesRequest request,
             false,
             getSystemIndexAccessLevel(),
             getSystemIndexAccessPredicate(),
-            getNetNewSystemIndexPredicate()
+            getNetNewSystemIndexPredicate(),
+            null
         );
         return concreteIndices(context, request.indices());
     }
@@ -1542,6 +1564,7 @@ public static class Context {
         private final SystemIndexAccessLevel systemIndexAccessLevel;
         private final Predicate<String> systemIndexAccessPredicate;
         private final Predicate<String> netNewSystemIndexPredicate;
+        private final Consumer<String> onResolutionFailure;
 
         Context(ProjectMetadata project, IndicesOptions options, SystemIndexAccessLevel systemIndexAccessLevel) {
             this(project, options, systemIndexAccessLevel, Predicates.always(), Predicates.never());
@@ -1584,7 +1607,8 @@ public static class Context {
                 false,
                 systemIndexAccessLevel,
                 systemIndexAccessPredicate,
-                netNewSystemIndexPredicate
+                netNewSystemIndexPredicate,
+                null
             );
         }
 
@@ -1609,7 +1633,8 @@ public static class Context {
                 preserveDataStreams,
                 systemIndexAccessLevel,
                 systemIndexAccessPredicate,
-                netNewSystemIndexPredicate
+                netNewSystemIndexPredicate,
+                null
             );
         }
 
@@ -1631,7 +1656,8 @@ public static class Context {
                 false,
                 systemIndexAccessLevel,
                 systemIndexAccessPredicate,
-                netNewSystemIndexPredicate
+                netNewSystemIndexPredicate,
+                null
             );
         }
 
@@ -1645,7 +1671,8 @@ protected Context(
             boolean preserveDataStreams,
             SystemIndexAccessLevel systemIndexAccessLevel,
             Predicate<String> systemIndexAccessPredicate,
-            Predicate<String> netNewSystemIndexPredicate
+            Predicate<String> netNewSystemIndexPredicate,
+            Consumer<String> onResolutionFailure
         ) {
             this.project = project;
             this.options = options;
@@ -1657,6 +1684,7 @@ protected Context(
             this.systemIndexAccessLevel = systemIndexAccessLevel;
             this.systemIndexAccessPredicate = systemIndexAccessPredicate;
             this.netNewSystemIndexPredicate = netNewSystemIndexPredicate;
+            this.onResolutionFailure = onResolutionFailure;
         }
 
         public ProjectMetadata getProject() {
@@ -1702,6 +1730,10 @@ public boolean isPreserveDataStreams() {
         public Predicate<String> getSystemIndexAccessPredicate() {
             return systemIndexAccessPredicate;
         }
+
+        public Consumer<String> onResolutionFailure() {
+            return onResolutionFailure;
+        }
     }
 
     /**

server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotRequestTests.java

@@ -79,7 +79,7 @@ public void testToXContent() throws IOException {
             boolean defaultResolveAliasForThisRequest = original.indicesOptions().ignoreAliases() == false;
             original.indicesOptions(
                 IndicesOptions.builder()
-                    .concreteTargetOptions(new IndicesOptions.ConcreteTargetOptions(randomBoolean()))
+                    .concreteTargetOptions(new IndicesOptions.ConcreteTargetOptions(randomBoolean(), randomBoolean()))
                     .wildcardOptions(
                         new IndicesOptions.WildcardOptions(
                             randomBoolean(),

server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java

@@ -79,7 +79,7 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) {
         if (randomBoolean()) {
             instance.indicesOptions(
                 IndicesOptions.builder()
-                    .concreteTargetOptions(new IndicesOptions.ConcreteTargetOptions(randomBoolean()))
+                    .concreteTargetOptions(new IndicesOptions.ConcreteTargetOptions(randomBoolean(), randomBoolean()))
                     .wildcardOptions(
                         new IndicesOptions.WildcardOptions(
                             randomBoolean(),

server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java

@@ -333,7 +333,7 @@ public void testFromMap() {
     }
 
     public void testToXContent() throws IOException {
-        ConcreteTargetOptions concreteTargetOptions = new ConcreteTargetOptions(randomBoolean());
+        ConcreteTargetOptions concreteTargetOptions = new ConcreteTargetOptions(randomBoolean(), randomBoolean());
         WildcardOptions wildcardOptions = new WildcardOptions(
             randomBoolean(),
             randomBoolean(),
@@ -374,7 +374,7 @@ public void testFromXContent() throws IOException {
             randomBoolean(),
             randomBoolean()
         );
-        ConcreteTargetOptions concreteTargetOptions = new ConcreteTargetOptions(randomBoolean());
+        ConcreteTargetOptions concreteTargetOptions = new ConcreteTargetOptions(randomBoolean(), randomBoolean());
 
         IndicesOptions indicesOptions = IndicesOptions.builder()
             .concreteTargetOptions(concreteTargetOptions)

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java

@@ -1184,9 +1184,7 @@ private static IndexRequest createDoc(String dsName, String id, String ts, Objec
         return new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE).id(id).source("@timestamp", ts, "count", count);
     }
 
-    public void testOverlappingIndexPatterns() throws Exception {
-        String[] indexNames = { "test_overlapping_index_patterns_1", "test_overlapping_index_patterns_2" };
-
+    public void testOverlappingIndexPatterns() {
         assertAcked(
             client().admin()
                 .indices()
@@ -1217,13 +1215,27 @@ public void testOverlappingIndexPatterns() throws Exception {
     }
 
     public void testErrorMessageForUnknownColumn() {
-        var e = expectThrows(VerificationException.class, () -> run("row a = 1 | eval x = b"));
-        assertThat(e.getMessage(), containsString("Unknown column [b]"));
+        expectThrows(VerificationException.class, containsString("Unknown column [b]"), () -> run("row a = 1 | eval x = b"));
     }
 
     public void testErrorMessageForEmptyParams() {
-        var e = expectThrows(ParsingException.class, () -> run("row a = 1 | eval x = ?"));
-        assertThat(e.getMessage(), containsString("Not enough actual parameters 0"));
+        expectThrows(ParsingException.class, containsString("Not enough actual parameters 0"), () -> run("row a = 1 | eval x = ?"));
+    }
+
+    public void testErrorMessageForUnknownIndex() {
+        expectThrows(
+            VerificationException.class,
+            containsString("Unknown index [no-such-index]"),
+            () -> run("from no-such-index", randomPragmas(), null, false)
+        );
+    }
+
+    public void testErrorMessageForUnknownIndexInPatternList() {
+        expectThrows(
+            VerificationException.class,
+            containsString("Unknown index [no-such-index]"),
+            () -> run("from test,no-such-index", randomPragmas(), null, false)
+        );
     }
 
     public void testEmptyIndex() {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java

@@ -9,6 +9,7 @@
 import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.index.IndexNotFoundException;
 
 import java.util.Map;
 import java.util.Objects;
@@ -27,25 +28,27 @@ public static IndexResolution valid(
         EsIndex index,
         Set<String> resolvedIndices,
         Set<NoShardAvailableActionException> unavailableShards,
+        Set<IndexNotFoundException> unavailableIndices,
         Map<String, FieldCapabilitiesFailure> unavailableClusters
     ) {
         Objects.requireNonNull(index, "index must not be null if it was found");
         Objects.requireNonNull(resolvedIndices, "resolvedIndices must not be null");
         Objects.requireNonNull(unavailableShards, "unavailableShards must not be null");
+        Objects.requireNonNull(unavailableIndices, "unavailableIndices must not be null");
         Objects.requireNonNull(unavailableClusters, "unavailableClusters must not be null");
-        return new IndexResolution(index, null, resolvedIndices, unavailableShards, unavailableClusters);
+        return new IndexResolution(index, null, resolvedIndices, unavailableShards, unavailableIndices, unavailableClusters);
     }
 
     /**
      * Use this method only if the set of concrete resolved indices is the same as EsIndex#concreteIndices().
      */
     public static IndexResolution valid(EsIndex index) {
-        return valid(index, index.concreteIndices(), Set.of(), Map.of());
+        return valid(index, index.concreteIndices(), Set.of(), Set.of(), Map.of());
     }
 
     public static IndexResolution invalid(String invalid) {
         Objects.requireNonNull(invalid, "invalid must not be null to signal that the index is invalid");
-        return new IndexResolution(null, invalid, Set.of(), Set.of(), Map.of());
+        return new IndexResolution(null, invalid, Set.of(), Set.of(), Set.of(), Map.of());
     }
 
     public static IndexResolution notFound(String name) {
@@ -60,6 +63,7 @@ public static IndexResolution notFound(String name) {
     // all indices found by field-caps
     private final Set<String> resolvedIndices;
     private final Set<NoShardAvailableActionException> unavailableShards;
+    private final Set<IndexNotFoundException> unavailableIndices;
     // remote clusters included in the user's index expression that could not be connected to
     private final Map<String, FieldCapabilitiesFailure> unavailableClusters;
 
@@ -68,12 +72,14 @@ private IndexResolution(
         @Nullable String invalid,
         Set<String> resolvedIndices,
         Set<NoShardAvailableActionException> unavailableShards,
+        Set<IndexNotFoundException> unavailableIndices,
         Map<String, FieldCapabilitiesFailure> unavailableClusters
     ) {
         this.index = index;
         this.invalid = invalid;
         this.resolvedIndices = resolvedIndices;
         this.unavailableShards = unavailableShards;
+        this.unavailableIndices = unavailableIndices;
         this.unavailableClusters = unavailableClusters;
     }
 
@@ -122,6 +128,13 @@ public Set<NoShardAvailableActionException> getUnavailableShards() {
         return unavailableShards;
     }
 
+    /**
+     * @return set of unavailable indices during index resolution
+     */
+    public Set<IndexNotFoundException> getUnavailableIndices() {
+        return unavailableIndices;
+    }
+
     @Override
     public boolean equals(Object obj) {
         if (obj == null || obj.getClass() != getClass()) {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java

@@ -104,6 +104,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static java.util.stream.Collectors.joining;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD;
 
@@ -421,6 +422,7 @@ private void initializeClusterData(List<IndexPattern> indices, EsqlExecutionInfo
         }
     }
 
+    @SuppressWarnings("checkstyle:LineLength")
     private void preAnalyzeMainIndices(
         PreAnalyzer.PreAnalysis preAnalysis,
         EsqlExecutionInfo executionInfo,
@@ -460,11 +462,25 @@ private void preAnalyzeMainIndices(
                     result.fieldNames,
                     requestFilter,
                     listener.delegateFailure((l, indexResolution) -> {
-                        if (configuration.allowPartialResults() == false && indexResolution.getUnavailableShards().isEmpty() == false) {
-                            l.onFailure(indexResolution.getUnavailableShards().iterator().next());
-                        } else {
-                            l.onResponse(result.withIndexResolution(indexResolution));
+                        if (configuration.allowPartialResults() == false) {
+                            if (indexResolution.getUnavailableShards().isEmpty() == false) {
+                                l.onFailure(indexResolution.getUnavailableShards().iterator().next());
+                                return;
+                            }
+                            if (indexResolution.getUnavailableIndices().isEmpty() == false) {
+                                l.onFailure(
+                                    new VerificationException(
+                                        "Unknown index {}",
+                                        indexResolution.getUnavailableIndices()
+                                            .stream()
+                                            .map(it -> it.getIndex().getName())
+                                            .collect(joining(", ", "[", "]"))
+                                    )
+                                );
+                                return;
+                            }
                         }
+                        l.onResponse(result.withIndexResolution(indexResolution));
                     })
                 );
             }