Local index resolution records resolved expressions (#135081)

n1v0lg · web-flow · commit a77d35865114 · 2025-09-24T13:55:02.000+02:00
This PR extends index resolution logic to capture the resolution mapping from original to resolution result via `ResolvedIndexExpressions` (introduced in #134783). This lets consumers such as cross-project search that require full resolution context to access it via the `IndicesRequest.Replaceable` interface. To maximize code re-use, the PR updates the existing `resolveIndexAbstractions` method to always return a `ResolvedIndexExpressions` instance. The result is only recorded on the request if a boolean flag is set to avoid unnecessary memory overhead in contexts where storing the expressions is not necessary. In a (much bigger) future refactor, we plan to move away from storing a raw indices list and always use `ResolvedIndexExpressions` as the source of truth for indices resources accessed by a request. Once that's complete, we will move to _always_ storing the full resolution result. This PR does not include the following, which we'll address in immediate follow ups: - Integrate the change into the wider implementation of cross-project search index expressions rewriting and error handling - Store resolution for `_all` index expression handling - Record full authorization exceptions on authorization failures - Record resolution for remote expressions (e.g., `p:logs -> p1:logs`) Relates: ES-12690
diff --git a/server/src/main/java/org/elasticsearch/action/IndicesRequest.java b/server/src/main/java/org/elasticsearch/action/IndicesRequest.java
@@ -53,6 +53,7 @@ interface Replaceable extends IndicesRequest {
          * Record the results of index resolution. See {@link ResolvedIndexExpressions} for details.
          * Note: this method does not replace {@link #indices(String...)}. {@link #indices(String...)} must still be called to update
          * the actual list of indices the request relates to.
+         * Note: the field is transient and not serialized.
          */
         default void setResolvedIndexExpressions(ResolvedIndexExpressions expressions) {}
 
diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java
@@ -12,7 +12,7 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.core.Nullable;
 
-import java.util.List;
+import java.util.Set;
 
 /**
  * This class allows capturing context about index expression replacements performed on an {@link IndicesRequest.Replaceable} during
@@ -40,23 +40,24 @@
  *                         and failure info
  * @param remoteExpressions the remote expressions that replace the original
  */
-public record ResolvedIndexExpression(String original, LocalExpressions localExpressions, List<String> remoteExpressions) {
+public record ResolvedIndexExpression(String original, LocalExpressions localExpressions, Set<String> remoteExpressions) {
     /**
      * Indicates if a local index resolution attempt was successful or failed.
-     * Failures can be due to missing concrete resources or unauthorized concrete resources.
+     * Failures can be due to concrete resources not being visible (either missing or not visible due to indices options)
+     * or unauthorized concrete resources.
      * A wildcard expression resolving to nothing is still considered a successful resolution.
      */
-    enum LocalIndexResolutionResult {
+    public enum LocalIndexResolutionResult {
         SUCCESS,
-        CONCRETE_RESOURCE_MISSING,
+        CONCRETE_RESOURCE_NOT_VISIBLE,
         CONCRETE_RESOURCE_UNAUTHORIZED,
     }
 
     /**
-     * Represents local (non-remote) resolution results, including expanded indices, and the resolution result.
+     * Represents local (non-remote) resolution results, including expanded indices, and a {@link LocalIndexResolutionResult}.
      */
     public record LocalExpressions(
-        List<String> expressions,
+        Set<String> expressions,
         LocalIndexResolutionResult localIndexResolutionResult,
         @Nullable ElasticsearchException exception
     ) {
diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java
@@ -9,32 +9,62 @@
 
 package org.elasticsearch.action;
 
-import java.util.Map;
+import org.elasticsearch.action.ResolvedIndexExpression.LocalExpressions;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 
 /**
- * A collection of {@link ResolvedIndexExpression}, keyed by the original expression.
- *
- * <p>An example structure is:</p>
- *
- * <pre>{@code
- * {
- *   "my-index-*": {
- *      "original": "my-index-*",
- *      "localExpressions": {
- *          "expressions": ["my-index-000001", "my-index-000002"],
- *          "localIndexResolutionResult": "SUCCESS"
- *      },
- *      "remoteExpressions": ["remote1:my-index-*", "remote2:my-index-*"]
- *   },
- *   "my-index-000001": {
- *      "original": "my-index-000001",
- *      "localExpressions": {
- *          "expressions": ["my-index-000001"],
- *          "localIndexResolutionResult": "SUCCESS"
- *      },
- *      "remoteExpressions": ["remote1:my-index-000001", "remote2:my-index-000001"]
- *   }
- * }
- * }</pre>
+ * A collection of {@link ResolvedIndexExpression}.
  */
-public record ResolvedIndexExpressions(Map<String, ResolvedIndexExpression> expressions) {}
+public record ResolvedIndexExpressions(List<ResolvedIndexExpression> expressions) {
+
+    public List<String> getLocalIndicesList() {
+        return expressions.stream().flatMap(e -> e.localExpressions().expressions().stream()).toList();
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static final class Builder {
+        private final List<ResolvedIndexExpression> expressions = new ArrayList<>();
+
+        /**
+         * @param original         the original expression that was resolved -- may be blank for "access all" cases
+         * @param localExpressions is a HashSet as an optimization -- the set needs to be mutable, and we want to avoid copying it.
+         *                         May be empty.
+         */
+        public void addLocalExpressions(
+            String original,
+            HashSet<String> localExpressions,
+            ResolvedIndexExpression.LocalIndexResolutionResult resolutionResult
+        ) {
+            Objects.requireNonNull(original);
+            Objects.requireNonNull(localExpressions);
+            Objects.requireNonNull(resolutionResult);
+            expressions.add(
+                new ResolvedIndexExpression(original, new LocalExpressions(localExpressions, resolutionResult, null), new HashSet<>())
+            );
+        }
+
+        /**
+         * Exclude the given expressions from the local expressions of all prior added {@link ResolvedIndexExpression}.
+         */
+        public void excludeFromLocalExpressions(Set<String> expressionsToExclude) {
+            Objects.requireNonNull(expressionsToExclude);
+            if (expressionsToExclude.isEmpty() == false) {
+                for (ResolvedIndexExpression prior : expressions) {
+                    prior.localExpressions().expressions().removeAll(expressionsToExclude);
+                }
+            }
+        }
+
+        public ResolvedIndexExpressions build() {
+            return new ResolvedIndexExpressions(expressions);
+        }
+    }
+}
diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.LegacyActionRequest;
+import org.elasticsearch.action.ResolvedIndexExpressions;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.common.Strings;
@@ -70,6 +71,9 @@ public class SearchRequest extends LegacyActionRequest implements IndicesRequest
 
     private String[] indices = Strings.EMPTY_ARRAY;
 
+    @Nullable
+    private ResolvedIndexExpressions resolvedIndexExpressions = null;
+
     @Nullable
     private String routing;
     @Nullable
@@ -400,6 +404,17 @@ public SearchRequest indices(String... indices) {
         return this;
     }
 
+    @Override
+    public void setResolvedIndexExpressions(ResolvedIndexExpressions expressions) {
+        this.resolvedIndexExpressions = expressions;
+    }
+
+    @Override
+    @Nullable
+    public ResolvedIndexExpressions getResolvedIndexExpressions() {
+        return resolvedIndexExpressions;
+    }
+
     private static void validateIndices(String... indices) {
         Objects.requireNonNull(indices, "indices must not be null");
         for (String index : indices) {
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
@@ -9,6 +9,8 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.action.ResolvedIndexExpression.LocalIndexResolutionResult;
+import org.elasticsearch.action.ResolvedIndexExpressions;
 import org.elasticsearch.action.support.IndexComponentSelector;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.UnsupportedSelectorException;
@@ -19,13 +21,16 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel;
 
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.function.BiPredicate;
 import java.util.function.Function;
 
+import static org.elasticsearch.action.ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE;
+import static org.elasticsearch.action.ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_UNAUTHORIZED;
+import static org.elasticsearch.action.ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS;
+
 public class IndexAbstractionResolver {
 
     private final IndexNameExpressionResolver indexNameExpressionResolver;
@@ -34,15 +39,16 @@ public IndexAbstractionResolver(IndexNameExpressionResolver indexNameExpressionR
         this.indexNameExpressionResolver = indexNameExpressionResolver;
     }
 
-    public List<String> resolveIndexAbstractions(
-        Iterable<String> indices,
+    public ResolvedIndexExpressions resolveIndexAbstractions(
+        List<String> indices,
         IndicesOptions indicesOptions,
         ProjectMetadata projectMetadata,
         Function<IndexComponentSelector, Set<String>> allAuthorizedAndAvailableBySelector,
         BiPredicate<String, IndexComponentSelector> isAuthorized,
         boolean includeDataStreams
     ) {
-        List<String> finalIndices = new ArrayList<>();
+        final ResolvedIndexExpressions.Builder resolvedExpressionsBuilder = ResolvedIndexExpressions.builder();
+
         boolean wildcardSeen = false;
         for (String index : indices) {
             String indexAbstraction;
@@ -55,8 +61,8 @@ public List<String> resolveIndexAbstractions(
             }
 
             // Always check to see if there's a selector on the index expression
-            Tuple<String, String> expressionAndSelector = IndexNameExpressionResolver.splitSelectorExpression(indexAbstraction);
-            String selectorString = expressionAndSelector.v2();
+            final Tuple<String, String> expressionAndSelector = IndexNameExpressionResolver.splitSelectorExpression(indexAbstraction);
+            final String selectorString = expressionAndSelector.v2();
             if (indicesOptions.allowSelectors() == false && selectorString != null) {
                 throw new UnsupportedSelectorException(indexAbstraction);
             }
@@ -68,7 +74,7 @@ public List<String> resolveIndexAbstractions(
 
             if (indicesOptions.expandWildcardExpressions() && Regex.isSimpleMatchPattern(indexAbstraction)) {
                 wildcardSeen = true;
-                Set<String> resolvedIndices = new HashSet<>();
+                final HashSet<String> resolvedIndices = new HashSet<>();
                 for (String authorizedIndex : allAuthorizedAndAvailableBySelector.apply(selector)) {
                     if (Regex.simpleMatch(indexAbstraction, authorizedIndex)
                         && isIndexVisible(
@@ -88,27 +94,46 @@ && isIndexVisible(
                     if (indicesOptions.allowNoIndices() == false) {
                         throw new IndexNotFoundException(indexAbstraction);
                     }
+                    resolvedExpressionsBuilder.addLocalExpressions(index, new HashSet<>(), SUCCESS);
                 } else {
                     if (minus) {
-                        finalIndices.removeAll(resolvedIndices);
+                        resolvedExpressionsBuilder.excludeFromLocalExpressions(resolvedIndices);
                     } else {
-                        finalIndices.addAll(resolvedIndices);
+                        resolvedExpressionsBuilder.addLocalExpressions(index, resolvedIndices, SUCCESS);
                     }
                 }
             } else {
-                Set<String> resolvedIndices = new HashSet<>();
+                final HashSet<String> resolvedIndices = new HashSet<>();
                 resolveSelectorsAndCollect(indexAbstraction, selectorString, indicesOptions, resolvedIndices, projectMetadata);
                 if (minus) {
-                    finalIndices.removeAll(resolvedIndices);
-                } else if (indicesOptions.ignoreUnavailable() == false || isAuthorized.test(indexAbstraction, selector)) {
-                    // Unauthorized names are considered unavailable, so if `ignoreUnavailable` is `true` they should be silently
-                    // discarded from the `finalIndices` list. Other "ways of unavailable" must be handled by the action
-                    // handler, see: https://github.com/elastic/elasticsearch/issues/90215
-                    finalIndices.addAll(resolvedIndices);
+                    resolvedExpressionsBuilder.excludeFromLocalExpressions(resolvedIndices);
+                } else {
+                    final boolean authorized = isAuthorized.test(indexAbstraction, selector);
+                    if (authorized) {
+                        final boolean visible = indexExists(projectMetadata, indexAbstraction)
+                            && isIndexVisible(
+                                indexAbstraction,
+                                selectorString,
+                                indexAbstraction,
+                                indicesOptions,
+                                projectMetadata,
+                                indexNameExpressionResolver,
+                                includeDataStreams
+                            );
+                        final LocalIndexResolutionResult result = visible ? SUCCESS : CONCRETE_RESOURCE_NOT_VISIBLE;
+                        resolvedExpressionsBuilder.addLocalExpressions(index, resolvedIndices, result);
+                    } else if (indicesOptions.ignoreUnavailable()) {
+                        // ignoreUnavailable implies that the request should not fail if an index is not authorized
+                        // so we map this expression to an empty list,
+                        resolvedExpressionsBuilder.addLocalExpressions(index, new HashSet<>(), CONCRETE_RESOURCE_UNAUTHORIZED);
+                    } else {
+                        // store the calculated expansion as unauthorized, it will be rejected later
+                        resolvedExpressionsBuilder.addLocalExpressions(index, resolvedIndices, CONCRETE_RESOURCE_UNAUTHORIZED);
+                    }
                 }
             }
         }
-        return finalIndices;
+        return resolvedExpressionsBuilder.build();
     }
 
     private static void resolveSelectorsAndCollect(
@@ -260,4 +285,8 @@ private static boolean isSystemIndexVisible(IndexNameExpressionResolver resolver
     private static boolean isVisibleDueToImplicitHidden(String expression, String index) {
         return index.startsWith(".") && expression.startsWith(".") && Regex.isSimpleMatchPattern(expression);
     }
+
+    private static boolean indexExists(ProjectMetadata projectMetadata, String indexAbstraction) {
+        return projectMetadata.getIndicesLookup().get(indexAbstraction) != null;
+    }
 }
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java
@@ -369,9 +369,9 @@ private List<String> resolveAbstractions(List<String> expressions, IndicesOption
             indicesOptions,
             projectMetadata,
             (ignored) -> mask,
-            (ignored, nothing) -> true,
+            (ignored1, ignored2) -> true,
             true
-        );
+        ).getLocalIndicesList();
     }
 
     private boolean isIndexVisible(String index, String selector) {
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
@@ -171,7 +171,7 @@ public AuthorizationService(
         this.clusterService = clusterService;
         this.auditTrailService = auditTrailService;
         this.restrictedIndices = restrictedIndices;
-        this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, linkedProjectConfigService, resolver);
+        this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, linkedProjectConfigService, resolver, false);
         this.authcFailureHandler = authcFailureHandler;
         this.threadContext = threadPool.getThreadContext();
         this.securityContext = new SecurityContext(settings, this.threadContext);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
@@ -8,6 +8,7 @@
 
 import org.elasticsearch.action.AliasesRequest;
 import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.ResolvedIndexExpressions;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
@@ -57,15 +58,18 @@ class IndicesAndAliasesResolver {
     private final IndexNameExpressionResolver nameExpressionResolver;
     private final IndexAbstractionResolver indexAbstractionResolver;
     private final RemoteClusterResolver remoteClusterResolver;
+    private final boolean recordResolvedIndexExpressions;
 
     IndicesAndAliasesResolver(
         Settings settings,
         LinkedProjectConfigService linkedProjectConfigService,
-        IndexNameExpressionResolver resolver
+        IndexNameExpressionResolver resolver,
+        boolean recordResolvedIndexExpressions
     ) {
         this.nameExpressionResolver = resolver;
         this.indexAbstractionResolver = new IndexAbstractionResolver(resolver);
         this.remoteClusterResolver = new RemoteClusterResolver(settings, linkedProjectConfigService);
+        this.recordResolvedIndexExpressions = recordResolvedIndexExpressions;
     }
 
     /**
@@ -348,15 +352,21 @@ ResolvedIndices resolveIndicesAndAliases(
                 } else {
                     split = new ResolvedIndices(Arrays.asList(indicesRequest.indices()), Collections.emptyList());
                 }
-                List<String> replaced = indexAbstractionResolver.resolveIndexAbstractions(
+                final ResolvedIndexExpressions resolved = indexAbstractionResolver.resolveIndexAbstractions(
                     split.getLocal(),
                     indicesOptions,
                     projectMetadata,
                     authorizedIndices::all,
                     authorizedIndices::check,
                     indicesRequest.includeDataStreams()
                 );
-                resolvedIndicesBuilder.addLocal(replaced);
+                // only store resolved expressions if configured, to avoid unnecessary memory usage
+                // once we've migrated from `indices()` to using resolved expressions holistically,
+                // we will always store them
+                if (recordResolvedIndexExpressions) {
+                    replaceable.setResolvedIndexExpressions(resolved);
+                }
+                resolvedIndicesBuilder.addLocal(resolved.getLocalIndicesList());
                 resolvedIndicesBuilder.addRemote(split.getRemote());
             }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
