Make redaction configurable for APM tracing (#92358)

Closes #92338.

When tracing REST requests with APM, we capture HTTP headers as labels
on the trace, but redact sensitive values. However, we can't know ahead
of time what are all possible sensitive values.

Push this redaction into the tracer, and make the redaction terms
configurable. Switch the defaults to the APM Java agent's defaults.

Author: pugnascotia

modules/apm/src/main/java/org/elasticsearch/tracing/apm/APM.java

@@ -101,6 +101,7 @@ public List<Setting<?>> getSettings() {
             APMAgentSettings.APM_ENABLED_SETTING,
             APMAgentSettings.APM_TRACING_NAMES_INCLUDE_SETTING,
             APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING,
+            APMAgentSettings.APM_TRACING_SANITIZE_FIELD_NAMES,
             APMAgentSettings.APM_AGENT_SETTINGS,
             APMAgentSettings.APM_SECRET_TOKEN_SETTING,
             APMAgentSettings.APM_API_KEY_SETTING

modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMAgentSettings.java

@@ -57,6 +57,7 @@ void addClusterSettingsListeners(ClusterService clusterService, APMTracer apmTra
         });
         clusterSettings.addSettingsUpdateConsumer(APM_TRACING_NAMES_INCLUDE_SETTING, apmTracer::setIncludeNames);
         clusterSettings.addSettingsUpdateConsumer(APM_TRACING_NAMES_EXCLUDE_SETTING, apmTracer::setExcludeNames);
+        clusterSettings.addSettingsUpdateConsumer(APM_TRACING_SANITIZE_FIELD_NAMES, apmTracer::setLabelFilters);
         clusterSettings.addAffixMapUpdateConsumer(APM_AGENT_SETTINGS, map -> map.forEach(this::setAgentSetting), (x, y) -> {});
     }
 
@@ -143,6 +144,27 @@ void setAgentSetting(String key, String value) {
         NodeScope
     );
 
+    static final Setting<List<String>> APM_TRACING_SANITIZE_FIELD_NAMES = Setting.listSetting(
+        APM_SETTING_PREFIX + "sanitize_field_names",
+        List.of(
+            "password",
+            "passwd",
+            "pwd",
+            "secret",
+            "*key",
+            "*token*",
+            "*session*",
+            "*credit*",
+            "*card*",
+            "*auth*",
+            "*principal*",
+            "set-cookie"
+        ),
+        Function.identity(),
+        OperatorDynamic,
+        NodeScope
+    );
+
     static final Setting<Boolean> APM_ENABLED_SETTING = Setting.boolSetting(
         APM_SETTING_PREFIX + "enabled",
         false,

modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMTracer.java

@@ -44,6 +44,7 @@
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_ENABLED_SETTING;
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING;
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_INCLUDE_SETTING;
+import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_SANITIZE_FIELD_NAMES;
 
 /**
  * This is an implementation of the {@link org.elasticsearch.tracing.Tracer} interface, which uses
@@ -65,8 +66,10 @@ public class APMTracer extends AbstractLifecycleComponent implements org.elastic
 
     private List<String> includeNames;
     private List<String> excludeNames;
+    private List<String> labelFilters;
     /** Built using {@link #includeNames} and {@link #excludeNames}, and filters out spans based on their name. */
     private volatile CharacterRunAutomaton filterAutomaton;
+    private volatile CharacterRunAutomaton labelFilterAutomaton;
     private String clusterName;
     private String nodeName;
 
@@ -86,7 +89,10 @@ record APMServices(Tracer tracer, OpenTelemetry openTelemetry) {}
     public APMTracer(Settings settings) {
         this.includeNames = APM_TRACING_NAMES_INCLUDE_SETTING.get(settings);
         this.excludeNames = APM_TRACING_NAMES_EXCLUDE_SETTING.get(settings);
+        this.labelFilters = APM_TRACING_SANITIZE_FIELD_NAMES.get(settings);
+
         this.filterAutomaton = buildAutomaton(includeNames, excludeNames);
+        this.labelFilterAutomaton = buildAutomaton(labelFilters, List.of());
         this.enabled = APM_ENABLED_SETTING.get(settings);
     }
 
@@ -109,6 +115,16 @@ void setExcludeNames(List<String> excludeNames) {
         this.filterAutomaton = buildAutomaton(includeNames, excludeNames);
     }
 
+    void setLabelFilters(List<String> labelFilters) {
+        this.labelFilters = labelFilters;
+        this.labelFilterAutomaton = buildAutomaton(labelFilters, List.of());
+    }
+
+    // package-private for testing
+    CharacterRunAutomaton getLabelFilterAutomaton() {
+        return labelFilterAutomaton;
+    }
+
     @Override
     protected void doStart() {
         if (enabled) {
@@ -271,6 +287,12 @@ private void setSpanAttributes(@Nullable Map<String, Object> spanAttributes, Spa
             for (Map.Entry<String, Object> entry : spanAttributes.entrySet()) {
                 final String key = entry.getKey();
                 final Object value = entry.getValue();
+
+                if (this.labelFilterAutomaton.run(key)) {
+                    spanBuilder.setAttribute(key, "[REDACTED]");
+                    continue;
+                }
+
                 if (value instanceof String) {
                     spanBuilder.setAttribute(key, (String) value);
                 } else if (value instanceof Long) {
@@ -394,9 +416,9 @@ Map<String, Context> getSpans() {
         return spans;
     }
 
-    private static CharacterRunAutomaton buildAutomaton(List<String> includeNames, List<String> excludeNames) {
-        Automaton includeAutomaton = patternsToAutomaton(includeNames);
-        Automaton excludeAutomaton = patternsToAutomaton(excludeNames);
+    private static CharacterRunAutomaton buildAutomaton(List<String> includePatterns, List<String> excludePatterns) {
+        Automaton includeAutomaton = patternsToAutomaton(includePatterns);
+        Automaton excludeAutomaton = patternsToAutomaton(excludePatterns);
 
         if (includeAutomaton == null) {
             includeAutomaton = Automata.makeAnyString();

modules/apm/src/test/java/org/elasticsearch/tracing/apm/APMTracerTests.java

@@ -8,12 +8,14 @@
 
 package org.elasticsearch.tracing.apm;
 
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.List;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_ENABLED_SETTING;
 import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING;
@@ -166,6 +168,43 @@ public void test_whenTraceStarted_andSpanNameExcluded_thenSpanIsNotStarted() {
         assertThat(apmTracer.getSpans(), hasKey("id3"));
     }
 
+    /**
+     * Check that sensitive attributes are not added verbatim to a span, but instead the value is redacted.
+     */
+    public void test_whenAddingAttributes_thenSensitiveValuesAreRedacted() {
+        Settings settings = Settings.builder().put(APM_ENABLED_SETTING.getKey(), false).build();
+        APMTracer apmTracer = buildTracer(settings);
+        CharacterRunAutomaton labelFilterAutomaton = apmTracer.getLabelFilterAutomaton();
+
+        Stream.of(
+            "auth",
+            "auth-header",
+            "authValue",
+            "card",
+            "card-details",
+            "card-number",
+            "credit",
+            "credit-card",
+            "key",
+            "my-credit-number",
+            "my_session_id",
+            "passwd",
+            "password",
+            "principal",
+            "principal-value",
+            "pwd",
+            "secret",
+            "secure-key",
+            "sensitive-token*",
+            "session",
+            "session_id",
+            "set-cookie",
+            "some-auth",
+            "some-principal",
+            "token-for login"
+        ).forEach(key -> assertTrue("Expected label filter automaton to redact [" + key + "]", labelFilterAutomaton.run(key)));
+    }
+
     private APMTracer buildTracer(Settings settings) {
         APMTracer tracer = new APMTracer(settings);
         tracer.doStart();

server/src/main/java/org/elasticsearch/rest/RestController.java

@@ -461,14 +461,10 @@ private void startTrace(ThreadContext threadContext, RestChannel channel, String
             name = restPath;
         }
 
-        Map<String, Object> attributes = Maps.newMapWithExpectedSize(req.getHeaders().size() + 3);
+        final Map<String, Object> attributes = Maps.newMapWithExpectedSize(req.getHeaders().size() + 3);
         req.getHeaders().forEach((key, values) -> {
             final String lowerKey = key.toLowerCase(Locale.ROOT).replace('-', '_');
-            final String value = switch (lowerKey) {
-                case "authorization", "cookie", "secret", "session", "set_cookie", "token", "x_elastic_app_auth" -> "[REDACTED]";
-                default -> String.join("; ", values);
-            };
-            attributes.put("http.request.headers." + lowerKey, value);
+            attributes.put("http.request.headers." + lowerKey, values.size() == 1 ? values.get(0) : String.join("; ", values));
         });
         attributes.put("http.method", method);
         attributes.put("http.url", req.uri());