Allow configuring SAML private attributes (elastic#133154)

This PR is twofold:
- adds a new `private_attributes` setting to the SAML realm, and
- introduces extension point that allows providing a custom `SamlAuthenticateResponseHandler`

The `private_attributes` setting can be used to define which SAML 
attributes should be treated as private. This implies that these 
attributes will not be logged or returned as part of user's metadata 
when `populate_user_metadata` is set to `true`.

Author: slobodanadamovic

docs/changelog/133154.yaml

@@ -0,0 +1,5 @@
+pr: 133154
+summary: Allow configuring SAML private attributes
+area: Authentication
+type: enhancement
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/saml/SamlRealmSettings.java

@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.authc.saml;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.util.set.Sets;
@@ -139,6 +140,56 @@ public class SamlRealmSettings {
         key -> Setting.positiveTimeSetting(key, TimeValue.timeValueMinutes(3), Setting.Property.NodeScope)
     );
 
+    /**
+     * The names of attributes that should be treated as private and never populated as part of the user's metadata
+     * (even when {@code #POPULATE_USER_METADATA} is configured).
+     */
+    public static final Function<String, Setting.AffixSetting<List<String>>> PRIVATE_ATTRIBUTES = (type) -> Setting.affixKeySetting(
+        RealmSettings.realmSettingPrefix(type),
+        "private_attributes",
+        (namespace, key) -> Setting.stringListSetting(key, new Setting.Validator<>() {
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final List<Setting<?>> settings = List.of(
+                    PRINCIPAL_ATTRIBUTE.apply(type).getAttribute().getConcreteSettingForNamespace(namespace),
+                    GROUPS_ATTRIBUTE.apply(type).getAttributeSetting().getAttribute().getConcreteSettingForNamespace(namespace),
+                    DN_ATTRIBUTE.apply(type).getAttribute().getConcreteSettingForNamespace(namespace),
+                    NAME_ATTRIBUTE.apply(type).getAttribute().getConcreteSettingForNamespace(namespace),
+                    MAIL_ATTRIBUTE.apply(type).getAttribute().getConcreteSettingForNamespace(namespace)
+                );
+                return settings.iterator();
+            }
+
+            @Override
+            public void validate(List<String> attributes) {
+                verifyNonNullNotEmpty(key, attributes);
+            }
+
+            @Override
+            public void validate(List<String> privateAttributes, Map<Setting<?>, Object> settings) {
+                if (false == privateAttributes.isEmpty()) {
+                    final Set<String> privateAttributesSet = Set.copyOf(privateAttributes);
+                    this.settings().forEachRemaining(attributeSetting -> {
+                        String attributeName = (String) settings.get(attributeSetting);
+
+                        if (false == Strings.isNullOrBlank(attributeName) && privateAttributesSet.contains(attributeName)) {
+                            throw new SettingsException(
+                                "SAML Attribute ["
+                                    + attributeName
+                                    + "] cannot be both configured for ["
+                                    + key
+                                    + "] and ["
+                                    + attributeSetting.getKey()
+                                    + "] settings."
+                            );
+                        }
+                    });
+                }
+            }
+        }, Setting.Property.NodeScope)
+    );
+
     public static final Function<String, Setting.AffixSetting<List<String>>> EXCLUDE_ROLES = (type) -> Setting.affixKeySetting(
         RealmSettings.realmSettingPrefix(type),
         "exclude_roles",
@@ -201,7 +252,8 @@ public static Set<Setting.AffixSetting<?>> getSettings(String type) {
             ENCRYPTION_KEY_ALIAS.apply(type),
             SIGNING_KEY_ALIAS.apply(type),
             SIGNING_MESSAGE_TYPES.apply(type),
-            REQUESTED_AUTHN_CONTEXT_CLASS_REF.apply(type)
+            REQUESTED_AUTHN_CONTEXT_CLASS_REF.apply(type),
+            PRIVATE_ATTRIBUTES.apply(type)
         );
 
         set.addAll(X509KeyPairSettings.affix(RealmSettings.realmSettingPrefix(type), ENCRYPTION_SETTING_KEY, false));

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -304,6 +304,7 @@
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authc.jwt.JwtRealm;
+import org.elasticsearch.xpack.security.authc.saml.SamlAuthenticateResponseHandler;
 import org.elasticsearch.xpack.security.authc.service.CachingServiceAccountTokenStore;
 import org.elasticsearch.xpack.security.authc.service.CompositeServiceAccountTokenStore;
 import org.elasticsearch.xpack.security.authc.service.FileServiceAccountTokenStore;
@@ -628,6 +629,7 @@ public class Security extends Plugin
     private final SetOnce<FileRoleValidator> fileRoleValidator = new SetOnce<>();
     private final SetOnce<SecondaryAuthActions> secondaryAuthActions = new SetOnce<>();
     private final SetOnce<QueryableBuiltInRolesProviderFactory> queryableRolesProviderFactory = new SetOnce<>();
+    private final SetOnce<SamlAuthenticateResponseHandler.Factory> samlAuthenticateResponseHandlerFactory = new SetOnce<>();
 
     private final SetOnce<SecurityMigrations.Manager> migrationManager = new SetOnce<>();
     private final SetOnce<List<Closeable>> closableComponents = new SetOnce<>();
@@ -957,6 +959,15 @@ Collection<Object> createComponents(
         if (fileRoleValidator.get() == null) {
             fileRoleValidator.set(new FileRoleValidator.Default());
         }
+        if (samlAuthenticateResponseHandlerFactory.get() == null) {
+            samlAuthenticateResponseHandlerFactory.set(new SamlAuthenticateResponseHandler.DefaultFactory());
+        }
+        components.add(
+            new PluginComponentBinding<>(
+                SamlAuthenticateResponseHandler.class,
+                samlAuthenticateResponseHandlerFactory.get().create(settings, tokenService, getClock())
+            )
+        );
         this.fileRolesStore.set(
             new FileRolesStore(settings, environment, resourceWatcherService, getLicenseState(), xContentRegistry, fileRoleValidator.get())
         );
@@ -2419,6 +2430,7 @@ public void loadExtensions(ExtensionLoader loader) {
         loadSingletonExtensionAndSetOnce(loader, fileRoleValidator, FileRoleValidator.class);
         loadSingletonExtensionAndSetOnce(loader, secondaryAuthActions, SecondaryAuthActions.class);
         loadSingletonExtensionAndSetOnce(loader, queryableRolesProviderFactory, QueryableBuiltInRolesProviderFactory.class);
+        loadSingletonExtensionAndSetOnce(loader, samlAuthenticateResponseHandlerFactory, SamlAuthenticateResponseHandler.Factory.class);
     }
 
     private <T> void loadSingletonExtensionAndSetOnce(ExtensionLoader loader, SetOnce<T> setOnce, Class<T> clazz) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlAuthenticateAction.java

@@ -12,7 +12,6 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -25,11 +24,9 @@
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
-import org.elasticsearch.xpack.security.authc.TokenService;
-import org.elasticsearch.xpack.security.authc.saml.SamlRealm;
+import org.elasticsearch.xpack.security.authc.saml.SamlAuthenticateResponseHandler;
 import org.elasticsearch.xpack.security.authc.saml.SamlToken;
 
-import java.util.Map;
 import java.util.concurrent.Executor;
 
 /**
@@ -39,7 +36,7 @@ public final class TransportSamlAuthenticateAction extends HandledTransportActio
 
     private final ThreadPool threadPool;
     private final AuthenticationService authenticationService;
-    private final TokenService tokenService;
+    private final SamlAuthenticateResponseHandler tokenHandler;
     private final SecurityContext securityContext;
     private final Executor genericExecutor;
 
@@ -49,7 +46,7 @@ public TransportSamlAuthenticateAction(
         TransportService transportService,
         ActionFilters actionFilters,
         AuthenticationService authenticationService,
-        TokenService tokenService,
+        SamlAuthenticateResponseHandler tokenHandler,
         SecurityContext securityContext
     ) {
         // TODO replace DIRECT_EXECUTOR_SERVICE when removing workaround for https://github.com/elastic/elasticsearch/issues/97916
@@ -62,7 +59,7 @@ public TransportSamlAuthenticateAction(
         );
         this.threadPool = threadPool;
         this.authenticationService = authenticationService;
-        this.tokenService = tokenService;
+        this.tokenHandler = tokenHandler;
         this.securityContext = securityContext;
         this.genericExecutor = threadPool.generic();
     }
@@ -88,25 +85,9 @@ private void doExecuteForked(Task task, SamlAuthenticateRequest request, ActionL
                 }
                 assert authentication != null : "authentication should never be null at this point";
                 assert false == authentication.isRunAs() : "saml realm authentication cannot have run-as";
-                @SuppressWarnings("unchecked")
-                final Map<String, Object> tokenMeta = (Map<String, Object>) result.getMetadata().get(SamlRealm.CONTEXT_TOKEN_DATA);
-                tokenService.createOAuth2Tokens(
-                    authentication,
-                    originatingAuthentication,
-                    tokenMeta,
-                    true,
-                    ActionListener.wrap(tokenResult -> {
-                        final TimeValue expiresIn = tokenService.getExpirationDelay();
-                        listener.onResponse(
-                            new SamlAuthenticateResponse(
-                                authentication,
-                                tokenResult.getAccessToken(),
-                                tokenResult.getRefreshToken(),
-                                expiresIn
-                            )
-                        );
-                    }, listener::onFailure)
-                );
+                assert result.isAuthenticated();
+                tokenHandler.handleTokenResponse(authentication, originatingAuthentication, result, listener);
+
             }, e -> {
                 logger.debug(() -> "SamlToken [" + saml + "] could not be authenticated", e);
                 listener.onFailure(e);

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/DefaultSamlAuthenticateResponseHandler.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.security.authc.saml;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xpack.core.security.action.saml.SamlAuthenticateResponse;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authc.TokenService;
+
+import java.util.Map;
+
+/**
+ * Default implementation of {@link SamlAuthenticateResponseHandler} that returns tokens crested using the {@link TokenService}.
+ */
+public final class DefaultSamlAuthenticateResponseHandler implements SamlAuthenticateResponseHandler {
+
+    private final TokenService tokenService;
+
+    public DefaultSamlAuthenticateResponseHandler(TokenService tokenService) {
+        this.tokenService = tokenService;
+    }
+
+    @Override
+    public void handleTokenResponse(
+        Authentication authentication,
+        Authentication originatingAuthentication,
+        AuthenticationResult<User> authenticationResult,
+        ActionListener<SamlAuthenticateResponse> listener
+    ) {
+        @SuppressWarnings("unchecked")
+        final Map<String, Object> tokenMeta = (Map<String, Object>) authenticationResult.getMetadata().get(SamlRealm.CONTEXT_TOKEN_DATA);
+        tokenService.createOAuth2Tokens(authentication, originatingAuthentication, tokenMeta, true, ActionListener.wrap(tokenResult -> {
+            final TimeValue expiresIn = tokenService.getExpirationDelay();
+            listener.onResponse(
+                new SamlAuthenticateResponse(authentication, tokenResult.getAccessToken(), tokenResult.getRefreshToken(), expiresIn)
+            );
+        }, listener::onFailure));
+    }
+}