POC: plug ES|QL project routing into index resolution

n1v0lg · n1v0lg · commit ab8b9338c451 · 2025-08-15T12:16:05.000+02:00
diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
@@ -46,6 +47,9 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen
 
     private String clusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY;
 
+    @Nullable
+    private String projectRouting;
+
     private String[] indices = Strings.EMPTY_ARRAY;
     private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
     private String[] fields = Strings.EMPTY_ARRAY;
@@ -113,6 +117,15 @@ String clusterAlias() {
         return clusterAlias;
     }
 
+    @Nullable
+    public String projectRouting() {
+        return projectRouting;
+    }
+
+    public void projectRouting(String projectRouting) {
+        this.projectRouting = projectRouting;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/esql/EsqlAsyncActionNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/esql/EsqlAsyncActionNames.java
@@ -13,4 +13,5 @@
 public class EsqlAsyncActionNames {
     public static final String ESQL_ASYNC_GET_RESULT_ACTION_NAME = "indices:data/read/esql/async/get";
     public static final String ESQL_ASYNC_STOP_ACTION_NAME = "indices:data/read/esql/async/stop";
+
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlProjectRoutingAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlProjectRoutingAction.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.action.ActionType;
+
+public class EsqlProjectRoutingAction extends ActionType<EsqlProjectRoutingResponse> {
+    public static final String NAME = "cluster:monitor/xpack/esql/project_routing";
+    public static final EsqlProjectRoutingAction INSTANCE = new EsqlProjectRoutingAction(NAME);
+
+    public EsqlProjectRoutingAction(String name) {
+        super(NAME);
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlProjectRoutingRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlProjectRoutingRequest.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.List;
+
+public class EsqlProjectRoutingRequest extends ActionRequest {
+    public EsqlProjectRoutingRequest(List<String> projects, String projectRoutingQuery) {
+
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        TransportAction.localOnly();
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlProjectRoutingResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlProjectRoutingResponse.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.List;
+
+public class EsqlProjectRoutingResponse extends ActionResponse {
+
+    private final List<String> projects;
+
+    public EsqlProjectRoutingResponse(List<String> projects) {
+        this.projects = projects;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeGenericList(projects, StreamOutput::writeString);
+    }
+
+    public List<String> getProjects() {
+        return projects;
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/TransportEsqlProjectRoutingAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/TransportEsqlProjectRoutingAction.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.List;
+
+public class TransportEsqlProjectRoutingAction extends TransportAction<EsqlProjectRoutingRequest, EsqlProjectRoutingResponse> {
+    @Inject
+    public TransportEsqlProjectRoutingAction(
+        Settings settings,
+        TransportService transportService,
+        ActionFilters actionFilters,
+        ClusterService clusterService
+    ) {
+        super(EsqlProjectRoutingAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE);
+    }
+
+    @Override
+    protected void doExecute(Task task, EsqlProjectRoutingRequest request, ActionListener<EsqlProjectRoutingResponse> listener) {
+        // whatever fancy ES|QL needs to happen here can happen here
+        listener.onResponse(new EsqlProjectRoutingResponse(List.of("project1", "project2", "project3")));
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java
@@ -57,6 +57,7 @@
 import org.elasticsearch.xpack.esql.action.EsqlAsyncStopAction;
 import org.elasticsearch.xpack.esql.action.EsqlGetQueryAction;
 import org.elasticsearch.xpack.esql.action.EsqlListQueriesAction;
+import org.elasticsearch.xpack.esql.action.EsqlProjectRoutingAction;
 import org.elasticsearch.xpack.esql.action.EsqlQueryAction;
 import org.elasticsearch.xpack.esql.action.EsqlQueryRequestBuilder;
 import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction;
@@ -67,6 +68,7 @@
 import org.elasticsearch.xpack.esql.action.RestEsqlListQueriesAction;
 import org.elasticsearch.xpack.esql.action.RestEsqlQueryAction;
 import org.elasticsearch.xpack.esql.action.RestEsqlStopAsyncAction;
+import org.elasticsearch.xpack.esql.action.TransportEsqlProjectRoutingAction;
 import org.elasticsearch.xpack.esql.analysis.PlanCheckerProvider;
 import org.elasticsearch.xpack.esql.common.Failures;
 import org.elasticsearch.xpack.esql.enrich.EnrichLookupOperator;
@@ -276,7 +278,8 @@ public List<ActionHandler> getActions() {
             new ActionHandler(EsqlSearchShardsAction.TYPE, EsqlSearchShardsAction.class),
             new ActionHandler(EsqlAsyncStopAction.INSTANCE, TransportEsqlAsyncStopAction.class),
             new ActionHandler(EsqlListQueriesAction.INSTANCE, TransportEsqlListQueriesAction.class),
-            new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class)
+            new ActionHandler(EsqlGetQueryAction.INSTANCE, TransportEsqlGetQueryAction.class),
+            new ActionHandler(EsqlProjectRoutingAction.INSTANCE, TransportEsqlProjectRoutingAction.class)
         );
     }
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java
@@ -270,6 +270,8 @@ private static FieldCapabilitiesRequest createFieldCapsRequest(String index, Set
         req.fields(fieldNames.toArray(String[]::new));
         req.includeUnmapped(true);
         req.indexFilter(requestFilter);
+        // Note: this is just some bogus query to demonstrate that we can invoke the full ES|QL engine from the security layer
+        req.projectRouting("row a = 1, b = \"x\", c = 1000000000000, d = 1.1");
         // lenient because we throw our own errors looking at the response e.g. if something was not resolved
         // also because this way security doesn't throw authorization exceptions but rather honors ignore_unavailable
         req.indicesOptions(FIELD_CAPS_INDICES_OPTIONS);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/EsqlProjectRouter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/EsqlProjectRouter.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.xpack.core.esql.action.EsqlQueryRequest;
+import org.elasticsearch.xpack.core.esql.action.EsqlQueryRequestBuilder;
+import org.elasticsearch.xpack.core.esql.action.EsqlQueryResponse;
+
+import java.util.List;
+
+public class EsqlProjectRouter {
+
+    private static final Logger logger = LogManager.getLogger(EsqlProjectRouter.class);
+
+    private final Client client;
+
+    public EsqlProjectRouter(Client client) {
+        this.client = client;
+    }
+
+    public List<String> route(List<String> projects, String projectRoutingQuery) {
+        // Note: this just demonstrates that we can invoke the full ES|QL engine from the security layer
+        // we certainly don't want to just run a generic ES|QL query
+
+        // Instead we should expose the EsqlProjectRoutingAction to be called the same way we
+        // expose the EsqlQueryAction and call EsqlProjectRoutingAction here
+        // (we will need to repeat the dance done in https://github.com/elastic/elasticsearch/issues/104413)
+
+        // EsqlProjectRoutingAction will be localOnly as it will only access an on-the fly in-memory index
+        // so there are no network costs associated
+
+        @SuppressWarnings("unchecked")
+        EsqlQueryRequestBuilder<EsqlQueryRequest, EsqlQueryResponse> b = (EsqlQueryRequestBuilder<
+            EsqlQueryRequest,
+            EsqlQueryResponse>) EsqlQueryRequestBuilder.newRequestBuilder(client);
+
+        b.query(projectRoutingQuery).execute(new ActionListener<>() {
+            @Override
+            public void onResponse(EsqlQueryResponse response) {
+                logger.info("EsqlProjectRouter response: {}", response);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.warn("EsqlProjectRouter failure", e);
+            }
+        });
+
+        return projects;
+    }
+}
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java
@@ -1130,7 +1130,8 @@ Collection<Object> createComponents(
             operatorPrivilegesService.get(),
             restrictedIndices,
             authorizationDenialMessages.get(),
-            projectResolver
+            projectResolver,
+            new EsqlProjectRouter(client)
         );
 
         components.add(nativeRolesStore); // used by roles actions
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java
@@ -81,6 +81,7 @@
 import org.elasticsearch.xpack.core.security.user.InternalUser;
 import org.elasticsearch.xpack.core.security.user.SystemUser;
 import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.EsqlProjectRouter;
 import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.audit.AuditLevel;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
@@ -164,12 +165,13 @@ public AuthorizationService(
         OperatorPrivilegesService operatorPrivilegesService,
         RestrictedIndices restrictedIndices,
         AuthorizationDenialMessages authorizationDenialMessages,
-        ProjectResolver projectResolver
+        ProjectResolver projectResolver,
+        EsqlProjectRouter router
     ) {
         this.clusterService = clusterService;
         this.auditTrailService = auditTrailService;
         this.restrictedIndices = restrictedIndices;
-        this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, clusterService, resolver);
+        this.indicesAndAliasesResolver = new IndicesAndAliasesResolver(settings, clusterService, resolver, router);
         this.authcFailureHandler = authcFailureHandler;
         this.threadContext = threadPool.getThreadContext();
         this.securityContext = new SecurityContext(settings, this.threadContext);
diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolver.java
@@ -11,6 +11,7 @@
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.search.SearchContextId;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndexComponentSelector;
@@ -38,6 +39,7 @@
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
 import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField;
 import org.elasticsearch.xpack.core.security.authz.ResolvedIndices;
+import org.elasticsearch.xpack.security.EsqlProjectRouter;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -58,11 +60,19 @@ class IndicesAndAliasesResolver {
     private final IndexNameExpressionResolver nameExpressionResolver;
     private final IndexAbstractionResolver indexAbstractionResolver;
     private final RemoteClusterResolver remoteClusterResolver;
+    private final EsqlProjectRouter router;
 
-    IndicesAndAliasesResolver(Settings settings, ClusterService clusterService, IndexNameExpressionResolver resolver) {
+    IndicesAndAliasesResolver(
+        Settings settings,
+        ClusterService clusterService,
+        IndexNameExpressionResolver resolver,
+        EsqlProjectRouter router
+    ) {
         this.nameExpressionResolver = resolver;
         this.indexAbstractionResolver = new IndexAbstractionResolver(resolver);
         this.remoteClusterResolver = new RemoteClusterResolver(settings, clusterService.getClusterSettings());
+        this.router = router;
+
     }
 
     /**
@@ -124,6 +134,13 @@ ResolvedIndices resolve(
         if (request instanceof IndicesRequest == false) {
             throw new IllegalStateException("Request [" + request + "] is not an Indices request, but should be.");
         }
+
+        if (request instanceof FieldCapabilitiesRequest fieldCapabilitiesRequest) {
+            if (fieldCapabilitiesRequest.projectRouting() != null) {
+                router.route(List.of("project1", "project2", "project3"), fieldCapabilitiesRequest.projectRouting());
+            }
+        }
+
         return resolveIndicesAndAliases(action, (IndicesRequest) request, projectMetadata, authorizedIndices);
     }
 
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java
@@ -336,7 +336,8 @@ public void setup() {
             operatorPrivilegesService,
             RESTRICTED_INDICES,
             new AuthorizationDenialMessages.Default(),
-            projectResolver
+            projectResolver,
+            null
         );
     }
 
@@ -1769,7 +1770,8 @@ public void testDenialForAnonymousUser() {
             operatorPrivilegesService,
             RESTRICTED_INDICES,
             new AuthorizationDenialMessages.Default(),
-            projectResolver
+            projectResolver,
+            null
         );
 
         RoleDescriptor role = new RoleDescriptor(
@@ -1819,7 +1821,8 @@ public void testDenialForAnonymousUserAuthorizationExceptionDisabled() {
             operatorPrivilegesService,
             RESTRICTED_INDICES,
             new AuthorizationDenialMessages.Default(),
-            projectResolver
+            projectResolver,
+            null
         );
 
         RoleDescriptor role = new RoleDescriptor(
@@ -3357,7 +3360,8 @@ public void testAuthorizationEngineSelectionForCheckPrivileges() throws Exceptio
             operatorPrivilegesService,
             RESTRICTED_INDICES,
             new AuthorizationDenialMessages.Default(),
-            projectResolver
+            projectResolver,
+            null
         );
 
         Subject subject = new Subject(new User("test", "a role"), mock(RealmRef.class));
@@ -3513,7 +3517,8 @@ public void getUserPrivileges(AuthorizationInfo authorizationInfo, ActionListene
             operatorPrivilegesService,
             RESTRICTED_INDICES,
             new AuthorizationDenialMessages.Default(),
-            projectResolver
+            projectResolver,
+            null
         );
         Authentication authentication;
         try (StoredContext ignore = threadContext.stashContext()) {
diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java

Original file line number	Diff line number	Diff line change
`@@ -13,4 +13,5 @@`
`13`	`13`	`public class EsqlAsyncActionNames {`
`14`	`14`	`public static final String ESQL_ASYNC_GET_RESULT_ACTION_NAME = "indices:data/read/esql/async/get";`
`15`	`15`	`public static final String ESQL_ASYNC_STOP_ACTION_NAME = "indices:data/read/esql/async/stop";`
	`16`	`+`
`16`	`17`	`}`