Add Support for Providing a custom ServiceAccountTokenStore through SecurityExtensions

docs/changelog/126612.yaml

@@ -0,0 +1,5 @@
+pr: 126612
+summary: Add Support for Providing a custom `ServiceAccountTokenStore` through `SecurityExtensions`
+area: Authentication
+type: enhancement
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java

@@ -16,6 +16,8 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationFailureHandler;
 import org.elasticsearch.xpack.core.security.authc.Realm;
+import org.elasticsearch.xpack.core.security.authc.service.NodeLocalServiceAccountTokenStore;
+import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountTokenStore;
 import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
@@ -114,6 +116,18 @@ default List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> getRo
         return Collections.emptyList();
     }
 
+    /**
+     * Returns a {@link NodeLocalServiceAccountTokenStore} used to authenticate service account tokens.
+     * If {@code null} is returned, the default service account token stores will be used.
+     *
+     * Providing a custom {@link NodeLocalServiceAccountTokenStore} here overrides the default implementation.
+     *
+     * @param components Access to components that can be used to authenticate service account tokens
+     */
+    default ServiceAccountTokenStore getServiceAccountTokenStore(SecurityComponents components) {
+        return null;
+    }
+
     /**
      * Returns a authorization engine for authorizing requests, or null to use the default authorization mechanism.
      *

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/service/NodeLocalServiceAccountTokenStore.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.authc.service;
+
+import org.elasticsearch.xpack.core.security.action.service.TokenInfo;
+
+import java.util.List;
+
+public interface NodeLocalServiceAccountTokenStore extends ServiceAccountTokenStore {
+    default List<TokenInfo> findNodeLocalTokensFor(ServiceAccount.ServiceAccountId accountId) {
+        throw new IllegalStateException("Find node local tokens not supported by [" + this.getClass() + "]");
+    }
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ServiceAccount.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/service/ServiceAccount.java

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.security.authc.service;
+package org.elasticsearch.xpack.core.security.authc.service;
 
 import org.apache.logging.log4j.util.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountToken.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/service/ServiceAccountToken.java

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.security.authc.service;
+package org.elasticsearch.xpack.core.security.authc.service;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -14,9 +14,9 @@
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.core.CharArrays;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
+import org.elasticsearch.xpack.core.security.authc.service.ServiceAccount.ServiceAccountId;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.support.Validation;
-import org.elasticsearch.xpack.security.authc.service.ServiceAccount.ServiceAccountId;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -51,7 +51,6 @@ public class ServiceAccountToken implements AuthenticationToken, Closeable {
     private final ServiceAccountTokenId tokenId;
     private final SecureString secret;
 
-    // pkg private for testing
     ServiceAccountToken(ServiceAccountId accountId, String tokenName, SecureString secret) {
         tokenId = new ServiceAccountTokenId(accountId, tokenName);
         this.secret = Objects.requireNonNull(secret, "service account token secret cannot be null");

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountTokenStore.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/service/ServiceAccountTokenStore.java

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.security.authc.service;
+package org.elasticsearch.xpack.core.security.authc.service;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.xpack.core.security.action.service.TokenInfo.TokenSource;
@@ -24,11 +24,23 @@ class StoreAuthenticationResult {
         private final boolean success;
         private final TokenSource tokenSource;
 
-        public StoreAuthenticationResult(boolean success, TokenSource tokenSource) {
+        private StoreAuthenticationResult(TokenSource tokenSource, boolean success) {
             this.success = success;
             this.tokenSource = tokenSource;
         }
 
+        public static StoreAuthenticationResult successful(TokenSource tokenSource) {
+            return new StoreAuthenticationResult(tokenSource, true);
+        }
+
+        public static StoreAuthenticationResult failed(TokenSource tokenSource) {
+            return new StoreAuthenticationResult(tokenSource, false);
+        }
+
+        public static StoreAuthenticationResult fromBooleanResult(TokenSource tokenSource, boolean result) {
+            return result ? successful(tokenSource) : failed(tokenSource);
+        }
+
         public boolean isSuccess() {
             return success;
         }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/ServiceAccountTokenTests.java → x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/service/ServiceAccountTokenTests.java

@@ -5,13 +5,13 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.security.authc.service;
+package org.elasticsearch.xpack.core.security.authc.service;
 
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authc.service.ServiceAccount.ServiceAccountId;
 import org.elasticsearch.xpack.core.security.support.Validation;
 import org.elasticsearch.xpack.core.security.support.ValidationTests;
-import org.elasticsearch.xpack.security.authc.service.ServiceAccount.ServiceAccountId;
 
 import java.io.IOException;
 

x-pack/plugin/security/src/main/java/module-info.java

@@ -74,6 +74,7 @@
     exports org.elasticsearch.xpack.security.rest.action.apikey to org.elasticsearch.internal.security;
     exports org.elasticsearch.xpack.security.support to org.elasticsearch.internal.security;
     exports org.elasticsearch.xpack.security.authz.store to org.elasticsearch.internal.security;
+    exports org.elasticsearch.xpack.security.authc.service;
 
     provides org.elasticsearch.index.SlowLogFieldProvider with org.elasticsearch.xpack.security.slowlog.SecuritySlowLogFieldProvider;
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -208,6 +208,8 @@
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.Subject;
+import org.elasticsearch.xpack.core.security.authc.service.NodeLocalServiceAccountTokenStore;
+import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountTokenStore;
 import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
@@ -310,6 +312,7 @@
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authc.jwt.JwtRealm;
 import org.elasticsearch.xpack.security.authc.service.CachingServiceAccountTokenStore;
+import org.elasticsearch.xpack.security.authc.service.CompositeServiceAccountTokenStore;
 import org.elasticsearch.xpack.security.authc.service.FileServiceAccountTokenStore;
 import org.elasticsearch.xpack.security.authc.service.IndexServiceAccountTokenStore;
 import org.elasticsearch.xpack.security.authc.service.ServiceAccountService;
@@ -915,12 +918,34 @@ Collection<Object> createComponents(
         this.realms.set(realms);
 
         systemIndices.getMainIndexManager().addStateListener(nativeRoleMappingStore::onSecurityIndexStateChange);
-
         final CacheInvalidatorRegistry cacheInvalidatorRegistry = new CacheInvalidatorRegistry();
-        cacheInvalidatorRegistry.registerAlias("service", Set.of("file_service_account_token", "index_service_account_token"));
         components.add(cacheInvalidatorRegistry);
-        systemIndices.getMainIndexManager().addStateListener(cacheInvalidatorRegistry::onSecurityIndexStateChange);
 
+        ServiceAccountService serviceAccountService = createServiceAccountService(
+            components,
+            cacheInvalidatorRegistry,
+            extensionComponents,
+            () -> new IndexServiceAccountTokenStore(
+                settings,
+                threadPool,
+                getClock(),
+                client,
+                systemIndices.getMainIndexManager(),
+                clusterService,
+                cacheInvalidatorRegistry
+            ),
+            () -> new FileServiceAccountTokenStore(
+                environment,
+                resourceWatcherService,
+                threadPool,
+                clusterService,
+                cacheInvalidatorRegistry
+            )
+        );
+
+        components.add(serviceAccountService);
+
+        systemIndices.getMainIndexManager().addStateListener(cacheInvalidatorRegistry::onSecurityIndexStateChange);
         final NativePrivilegeStore privilegeStore = new NativePrivilegeStore(
             settings,
             client,
@@ -1004,33 +1029,6 @@ Collection<Object> createComponents(
         );
         components.add(apiKeyService);
 
-        final IndexServiceAccountTokenStore indexServiceAccountTokenStore = new IndexServiceAccountTokenStore(
-            settings,
-            threadPool,
-            getClock(),
-            client,
-            systemIndices.getMainIndexManager(),
-            clusterService,
-            cacheInvalidatorRegistry
-        );
-        components.add(indexServiceAccountTokenStore);
-
-        final FileServiceAccountTokenStore fileServiceAccountTokenStore = new FileServiceAccountTokenStore(
-            environment,
-            resourceWatcherService,
-            threadPool,
-            clusterService,
-            cacheInvalidatorRegistry
-        );
-        components.add(fileServiceAccountTokenStore);
-
-        final ServiceAccountService serviceAccountService = new ServiceAccountService(
-            client,
-            fileServiceAccountTokenStore,
-            indexServiceAccountTokenStore
-        );
-        components.add(serviceAccountService);
-
         final RoleProviders roleProviders = new RoleProviders(
             reservedRolesStore,
             fileRolesStore.get(),
@@ -1250,6 +1248,74 @@ Collection<Object> createComponents(
         return components;
     }
 
+    private ServiceAccountService createServiceAccountService(
+        List<Object> components,
+        CacheInvalidatorRegistry cacheInvalidatorRegistry,
+        SecurityExtension.SecurityComponents extensionComponents,
+        Supplier<IndexServiceAccountTokenStore> indexServiceAccountTokenStoreSupplier,
+        Supplier<FileServiceAccountTokenStore> fileServiceAccountTokenStoreSupplier
+    ) {
+        Map<String, ServiceAccountTokenStore> accountTokenStoreByExtension = new HashMap<>();
+
+        for (var extension : securityExtensions) {
+            var serviceAccountTokenStore = extension.getServiceAccountTokenStore(extensionComponents);
+            if (serviceAccountTokenStore != null) {
+                if (isInternalExtension(extension) == false) {
+                    throw new IllegalStateException(
+                        "The ["
+                            + extension.getClass().getName()
+                            + "] extension tried to install a custom ServiceAccountTokenStore. This functionality is not available to "
+                            + "external extensions."
+                    );
+                }
+                accountTokenStoreByExtension.put(extension.extensionName(), serviceAccountTokenStore);
+            }
+        }
+
+        if (accountTokenStoreByExtension.size() > 1) {
+            throw new IllegalStateException(
+                "More than one extension provided a ServiceAccountTokenStore override: " + accountTokenStoreByExtension.keySet()
+            );
+        }
+
+        if (accountTokenStoreByExtension.isEmpty()) {
+            var fileServiceAccountTokenStore = fileServiceAccountTokenStoreSupplier.get();
+            var indexServiceAccountTokenStore = indexServiceAccountTokenStoreSupplier.get();
+
+            components.add(new PluginComponentBinding<>(NodeLocalServiceAccountTokenStore.class, fileServiceAccountTokenStore));
+            components.add(fileServiceAccountTokenStore);
+            components.add(indexServiceAccountTokenStore);
+            cacheInvalidatorRegistry.registerAlias("service", Set.of("file_service_account_token", "index_service_account_token"));
+
+            return new ServiceAccountService(
+                client.get(),
+                new CompositeServiceAccountTokenStore(
+                    List.of(fileServiceAccountTokenStore, indexServiceAccountTokenStore),
+                    client.get().threadPool().getThreadContext()
+                ),
+                indexServiceAccountTokenStore
+            );
+        }
+        // Completely handover service account token management to the extension if provided,
+        // this will disable the index managed
+        // service account tokens managed through the service account token API
+        var extensionStore = accountTokenStoreByExtension.values().stream().findFirst();
+        components.add(new PluginComponentBinding<>(NodeLocalServiceAccountTokenStore.class, (token, listener) -> {
+            throw new IllegalStateException("Node local config not supported by [" + extensionStore.get().getClass() + "]");
+        }));
+        components.add(extensionStore);
+        logger.debug("Service account authentication handled by extension, disabling file and index token stores");
+        return new ServiceAccountService(client.get(), extensionStore.get());
+    }
+
+    private static boolean isInternalExtension(SecurityExtension extension) {
+        final String canonicalName = extension.getClass().getCanonicalName();
+        if (canonicalName == null) {
+            return false;
+        }
+        return canonicalName.startsWith("org.elasticsearch.xpack.") || canonicalName.startsWith("co.elastic.elasticsearch.");
+    }
+
     @FixForMultiProject
     // TODO : The migration task needs to be project aware
     private void applyPendingSecurityMigrations(ProjectId projectId, SecurityIndexManager.IndexState newState) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountAction.java

@@ -19,7 +19,7 @@
 import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountRequest;
 import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountResponse;
 import org.elasticsearch.xpack.core.security.action.service.ServiceAccountInfo;
-import org.elasticsearch.xpack.security.authc.service.ServiceAccount;
+import org.elasticsearch.xpack.core.security.authc.service.ServiceAccount;
 import org.elasticsearch.xpack.security.authc.service.ServiceAccountService;
 
 import java.util.function.Predicate;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/service/TransportGetServiceAccountNodesCredentialsAction.java

@@ -21,8 +21,8 @@
 import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountCredentialsNodesResponse;
 import org.elasticsearch.xpack.core.security.action.service.GetServiceAccountNodesCredentialsAction;
 import org.elasticsearch.xpack.core.security.action.service.TokenInfo;
-import org.elasticsearch.xpack.security.authc.service.FileServiceAccountTokenStore;
-import org.elasticsearch.xpack.security.authc.service.ServiceAccount.ServiceAccountId;
+import org.elasticsearch.xpack.core.security.authc.service.NodeLocalServiceAccountTokenStore;
+import org.elasticsearch.xpack.core.security.authc.service.ServiceAccount.ServiceAccountId;
 
 import java.io.IOException;
 import java.util.List;
@@ -38,15 +38,15 @@ public class TransportGetServiceAccountNodesCredentialsAction extends TransportN
     GetServiceAccountCredentialsNodesResponse.Node,
     Void> {
 
-    private final FileServiceAccountTokenStore fileServiceAccountTokenStore;
+    private final NodeLocalServiceAccountTokenStore readOnlyServiceAccountTokenStore;
 
     @Inject
     public TransportGetServiceAccountNodesCredentialsAction(
         ThreadPool threadPool,
         ClusterService clusterService,
         TransportService transportService,
         ActionFilters actionFilters,
-        FileServiceAccountTokenStore fileServiceAccountTokenStore
+        NodeLocalServiceAccountTokenStore readOnlyServiceAccountTokenStore
     ) {
         super(
             GetServiceAccountNodesCredentialsAction.NAME,
@@ -56,7 +56,7 @@ public TransportGetServiceAccountNodesCredentialsAction(
             GetServiceAccountCredentialsNodesRequest.Node::new,
             threadPool.executor(ThreadPool.Names.GENERIC)
         );
-        this.fileServiceAccountTokenStore = fileServiceAccountTokenStore;
+        this.readOnlyServiceAccountTokenStore = readOnlyServiceAccountTokenStore;
     }
 
     @Override
@@ -84,7 +84,7 @@ protected GetServiceAccountCredentialsNodesResponse.Node nodeOperation(
         Task task
     ) {
         final ServiceAccountId accountId = new ServiceAccountId(request.getNamespace(), request.getServiceName());
-        final List<TokenInfo> tokenInfos = fileServiceAccountTokenStore.findTokensFor(accountId);
+        final List<TokenInfo> tokenInfos = readOnlyServiceAccountTokenStore.findNodeLocalTokensFor(accountId);
         return new GetServiceAccountCredentialsNodesResponse.Node(
             clusterService.localNode(),
             tokenInfos.stream().map(TokenInfo::getName).toArray(String[]::new)

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java

@@ -97,6 +97,7 @@
 import org.elasticsearch.xpack.core.security.authc.AuthenticationField;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSettings;
+import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountToken;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
@@ -110,7 +111,6 @@
 import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditUtil;
 import org.elasticsearch.xpack.security.authc.ApiKeyService;
-import org.elasticsearch.xpack.security.authc.service.ServiceAccountToken;
 import org.elasticsearch.xpack.security.rest.RemoteHostHeader;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ServiceAccountAuthenticator.java

@@ -15,8 +15,8 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
+import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountToken;
 import org.elasticsearch.xpack.security.authc.service.ServiceAccountService;
-import org.elasticsearch.xpack.security.authc.service.ServiceAccountToken;
 import org.elasticsearch.xpack.security.metric.InstrumentedSecurityActionListener;
 import org.elasticsearch.xpack.security.metric.SecurityMetricType;
 import org.elasticsearch.xpack.security.metric.SecurityMetrics;