Adding tracing support for the authenticate action by wrapping the authenticator.authenticate call in AuthenticatorChain with a startTrace/stopTrace action.

In synchronous contexts, spans are stored as ThreadLocals under the hood and the currently active span look up (needed to end a span, append metadata, or spawn a child span) can be implicit. However, to track a span across threads as ES requires, we need to make `Authenticator.Context` implement `Traceable` which defines a random, unique `spanId` for each Context object, enabling us to uniquely locate the associated span regardless of which thread is currently executing.

Author: ankit--sethi

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java

@@ -285,6 +285,11 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
         NodeScope
     );
 
+    // not a setting to reduce the footprint of this change
+    public static final List<String> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES_EXCLUDES = List.of(
+        "security.authenticator.type"
+    );
+
     public static final Setting<Boolean> TELEMETRY_TRACING_ENABLED_SETTING = Setting.boolSetting(
         TELEMETRY_SETTING_PREFIX + "tracing.enabled",
         false,

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java

@@ -90,7 +90,7 @@ public APMTracer(Settings settings) {
         this.labelFilters = APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES.get(settings);
 
         this.filterAutomaton = buildAutomaton(includeNames, excludeNames);
-        this.labelFilterAutomaton = buildAutomaton(labelFilters, List.of());
+        this.labelFilterAutomaton = buildAutomaton(labelFilters, APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES_EXCLUDES);
         this.enabled = APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING.get(settings);
     }
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -1085,7 +1085,7 @@ Collection<Object> createComponents(
                 serviceAccountService,
                 operatorPrivilegesService.get(),
                 customApiKeyAuthenticator,
-                telemetryProvider.getMeterRegistry()
+                telemetryProvider
             )
         );
         components.add(authcService.get());

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java

@@ -22,7 +22,9 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.http.HttpPreRequest;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.telemetry.metric.MeterRegistry;
+import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
@@ -94,7 +96,7 @@ public AuthenticationService(
         ServiceAccountService serviceAccountService,
         OperatorPrivilegesService operatorPrivilegesService,
         CustomApiKeyAuthenticator customApiKeyAuthenticator,
-        MeterRegistry meterRegistry
+        TelemetryProvider telemetryProvider
     ) {
         this.realms = realms;
         this.auditTrailService = auditTrailService;
@@ -108,13 +110,15 @@ public AuthenticationService(
         } else {
             this.lastSuccessfulAuthCache = null;
         }
-
+        MeterRegistry meterRegistry = telemetryProvider.getMeterRegistry();
+        Tracer tracer = telemetryProvider.getTracer();
         final String nodeName = Node.NODE_NAME_SETTING.get(settings);
         this.authenticatorChain = new AuthenticatorChain(
             settings,
             operatorPrivilegesService,
             anonymousUser,
             new AuthenticationContextSerializer(),
+            tracer,
             new ServiceAccountAuthenticator(serviceAccountService, nodeName, meterRegistry),
             new OAuth2TokenAuthenticator(tokenService, meterRegistry),
             new PluggableApiKeyAuthenticator(customApiKeyAuthenticator),

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/Authenticator.java

@@ -9,10 +9,13 @@
 
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.telemetry.tracing.Traceable;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
@@ -90,7 +93,7 @@ static SecureString extractApiKeyFromHeader(ThreadContext threadContext) {
      * the next {@link Authenticator} is tried.
      * The extracted tokens are all appended with {@link #addAuthenticationToken(AuthenticationToken)}.
      */
-    class Context implements Closeable {
+    class Context implements Closeable, Traceable {
         private final ThreadContext threadContext;
         private final AuthenticationService.AuditableRequest request;
         private final User fallbackUser;
@@ -104,6 +107,7 @@ class Context implements Closeable {
         private SecureString apiKeyString = null;
         private List<Realm> defaultOrderedRealmList = null;
         private List<Realm> unlicensedRealms = null;
+        private String requestId;
 
         /**
          * Context constructor that provides the authentication token directly as an argument.
@@ -129,6 +133,7 @@ class Context implements Closeable {
             // if handleNullToken is false, fallbackUser and allowAnonymous are irrelevant
             this.fallbackUser = null;
             this.allowAnonymous = false;
+            this.requestId = UUIDs.randomBase64UUID(Randomness.get());;
         }
 
         /**
@@ -149,6 +154,7 @@ public Context(
             this.fallbackUser = fallbackUser;
             this.allowAnonymous = allowAnonymous;
             this.realms = realms;
+            this.requestId = UUIDs.randomBase64UUID(Randomness.get());;
         }
 
         public ThreadContext getThreadContext() {
@@ -241,5 +247,10 @@ public void addUnsuccessfulMessageToMetadata(final ElasticsearchSecurityExceptio
                 ese.addMetadata("es.additional_unsuccessful_credentials", getUnsuccessfulMessages());
             }
         }
+
+        @Override
+        public String getSpanId() {
+            return requestId;
+        }
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticatorChain.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.xpack.core.common.IteratingActionListener;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
@@ -26,6 +27,7 @@
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.operator.OperatorPrivileges.OperatorPrivilegesService;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.function.BiConsumer;
@@ -46,12 +48,14 @@ class AuthenticatorChain {
     private final AuthenticationContextSerializer authenticationSerializer;
     private final RealmsAuthenticator realmsAuthenticator;
     private final List<Authenticator> allAuthenticators;
+    private final Tracer tracer;
 
     AuthenticatorChain(
         Settings settings,
         OperatorPrivilegesService operatorPrivilegesService,
         AnonymousUser anonymousUser,
         AuthenticationContextSerializer authenticationSerializer,
+        Tracer tracer,
         ServiceAccountAuthenticator serviceAccountAuthenticator,
         OAuth2TokenAuthenticator oAuth2TokenAuthenticator,
         PluggableApiKeyAuthenticator pluggableApiKeyAuthenticator,
@@ -65,6 +69,7 @@ class AuthenticatorChain {
         this.isAnonymousUserEnabled = AnonymousUser.isAnonymousEnabled(settings);
         this.authenticationSerializer = authenticationSerializer;
         this.realmsAuthenticator = realmsAuthenticator;
+        this.tracer = tracer;
         this.allAuthenticators = List.of(
             serviceAccountAuthenticator,
             oAuth2TokenAuthenticator,
@@ -123,7 +128,7 @@ private void doAuthenticate(Authenticator.Context context, ActionListener<Authen
                     }
                 }
             }),
-            getAuthenticatorConsumer(context),
+            getAuthenticatorConsumer(context, tracer),
             allAuthenticators,
             context.getThreadContext(),
             Function.identity(),
@@ -133,7 +138,7 @@ private void doAuthenticate(Authenticator.Context context, ActionListener<Authen
     }
 
     private static BiConsumer<Authenticator, ActionListener<AuthenticationResult<Authentication>>> getAuthenticatorConsumer(
-        Authenticator.Context context
+        Authenticator.Context context, Tracer tracer
     ) {
         return (authenticator, listener) -> {
             if (context.shouldExtractCredentials()) {
@@ -171,7 +176,7 @@ private static BiConsumer<Authenticator, ActionListener<AuthenticationResult<Aut
                 listener.onFailure(e);
             };
 
-            authenticator.authenticate(context, ActionListener.wrap(result -> {
+            ActionListener<AuthenticationResult<Authentication>> authResultListener = ActionListener.wrap(result -> {
                 if (result.getStatus() == AuthenticationResult.Status.TERMINATE) {
                     onFailure.accept(result.getException());
                     return;
@@ -180,10 +185,22 @@ private static BiConsumer<Authenticator, ActionListener<AuthenticationResult<Aut
                     context.addUnsuccessfulMessage(authenticator.name() + ": " + result.getMessage());
                 }
                 listener.onResponse(result);
-            }, onFailure));
+            }, onFailure);
+
+            ActionListener<AuthenticationResult<Authentication>> afterAuthResultListener = ActionListener.runAfter(authResultListener,
+                () -> tracer.stopTrace(context));
+
+            tracer.startTrace(context.getThreadContext(), context, "authenticate", addUsefulMetadata(authenticator));
+            authenticator.authenticate(context, afterAuthResultListener);
         };
     }
 
+    private static Map<String, Object> addUsefulMetadata(Authenticator authenticator) {
+        Map<String, Object> attribs = new HashMap<>();
+        attribs.put("security.authenticator.type", authenticator.name());
+        return attribs;
+    }
+
     // Package private for test
     void maybeLookupRunAsUser(Authenticator.Context context, Authentication authentication, ActionListener<Authentication> listener) {
         if (false == runAsEnabled) {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticatorChainTests.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.node.Node;
+import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLog;
 import org.elasticsearch.xpack.core.security.action.apikey.ApiKey;
@@ -103,6 +104,7 @@ public void init() {
             operatorPrivilegesService,
             anonymousUser,
             authenticationContextSerializer,
+            Tracer.NOOP,
             serviceAccountAuthenticator,
             oAuth2TokenAuthenticator,
             pluggableApiKeyAuthenticator,