JMS - some more enhancements (#911)

Author: eyalkoren

CHANGELOG.md

@@ -1,6 +1,18 @@
 # (Next)
 
 ## Features
+ * JMS Enhancements (#911):
+    * Add special handling for temporary queues/topics
+    * Capture message bodies of text Messages
+        * Rely on the existing `ELASTIC_APM_CAPTURE_BODY` agent config option (off by default).
+        * Send as `context.message.body`
+        * Limit size to 10000 characters. If longer than this size, trim to 9999 and append with ellipsis
+    * Introduce the `ignore_message_queues` configuration to disable instrumentation (message tagging) for specific 
+      queues/topics as suggested in #710
+    * Capture predefined message headers and all properties
+        * Rely on the existing `ELASTIC_APM_CAPTURE_HEADERS` agent config option.
+        * Send as `context.message.headers`
+        * Sanitize sensitive headers/properties based on the `sanitize_field_names` config option
 
 ## Bug Fixes
  * JDBC regression- `PreparedStatement#executeUpdate()` and `PreparedStatement#executeLargeUpdate()` are not traced (#918)

apm-agent-benchmarks/pom.xml

@@ -33,7 +33,7 @@
         </dependency>
         <!--
         we can't add elastic-apm-agent as it causes shading problems:
-        java.util.ServiceConfigurationError: org.stagemonitor.configuration.ConfigurationOptionProvider: co.elastic.apm.agent.web.WebConfiguration not a subtype
+        java.util.ServiceConfigurationError: org.stagemonitor.configuration.ConfigurationOptionProvider: co.elastic.apm.agent.impl.context.web.WebConfiguration not a subtype
         -->
         <dependency>
             <groupId>${project.groupId}</groupId>

apm-agent-core/src/main/java/co/elastic/apm/agent/configuration/CoreConfiguration.java

@@ -267,6 +267,38 @@ public class CoreConfiguration extends ConfigurationOptionProvider {
         .dynamic(true)
         .buildWithDefault(Collections.<WildcardMatcher>emptyList());
 
+    private final ConfigurationOption<EventType> captureBody = ConfigurationOption.enumOption(EventType.class)
+        .key("capture_body")
+        .configurationCategory(CORE_CATEGORY)
+        .tags("performance")
+        .description("For transactions that are HTTP requests, the Java agent can optionally capture the request body (e.g. POST \n" +
+            "variables). For transactions that are initiated by receiving a JMS text message, the agent can capture the \n" +
+            "textual message body.\n" +
+            "\n" +
+            "If the HTTP request or the JMS message has a body and this setting is disabled, the body will be shown as [REDACTED].\n" +
+            "\n" +
+            "This option is case-insensitive.\n" +
+            "\n" +
+            "NOTE: Currently, only UTF-8 encoded plain text HTTP content types are supported.\n" +
+            "The option <<config-capture-body-content-types>> determines which content types are captured.\n" +
+            "\n" +
+            "WARNING: Request bodies often contain sensitive values like passwords, credit card numbers etc.\n" +
+            "If your service handles data like this, we advise to only enable this feature with care.\n" +
+            "Turning on body capturing can also significantly increase the overhead in terms of heap usage,\n" +
+            "network utilisation and Elasticsearch index size.")
+        .dynamic(true)
+        .buildWithDefault(EventType.OFF);
+
+    private final ConfigurationOption<Boolean> captureHeaders = ConfigurationOption.booleanOption()
+        .key("capture_headers")
+        .configurationCategory(CORE_CATEGORY)
+        .tags("performance")
+        .description("If set to `true`, the agent will capture request and response headers, including cookies.\n" +
+            "\n" +
+            "NOTE: Setting this to `false` reduces network bandwidth, disk space and object allocations.")
+        .dynamic(true)
+        .buildWithDefault(true);
+
     private final ConfigurationOption<Map<String, String>> globalLabels = ConfigurationOption
         .builder(new MapValueConverter<String, String>(StringValueConverter.INSTANCE, StringValueConverter.INSTANCE, "=", ","), Map.class)
         .key("global_labels")
@@ -514,6 +546,14 @@ public List<WildcardMatcher> getIgnoreExceptions(){
         return ignoreExceptions.get();
     }
 
+    public EventType getCaptureBody() {
+        return captureBody.get();
+    }
+
+    public boolean isCaptureHeaders() {
+        return captureHeaders.get();
+    }
+
     public boolean isTypePoolCacheEnabled() {
         return typePoolCache.get();
     }
@@ -589,4 +629,28 @@ public static String getConfigFileLocation(List<ConfigurationSource> configurati
             return configFileLocation;
         }
     }
+
+    public enum EventType {
+        /**
+         * Request bodies will never be reported
+         */
+        OFF,
+        /**
+         * Request bodies will only be reported with errors
+         */
+        ERRORS,
+        /**
+         * Request bodies will only be reported with request transactions
+         */
+        TRANSACTIONS,
+        /**
+         * Request bodies will be reported with both errors and request transactions
+         */
+        ALL;
+
+        @Override
+        public String toString() {
+            return name().toLowerCase();
+        }
+    }
 }

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/AbstractContext.java

@@ -31,6 +31,9 @@
 import java.util.concurrent.ConcurrentHashMap;
 
 public abstract class AbstractContext implements Recyclable {
+
+    public static final String REDACTED_CONTEXT_STRING = "[REDACTED]";
+
     /**
      * A flat mapping of user-defined labels with {@link String} keys and {@link String}, {@link Number} or {@link Boolean} values
      * (formerly known as tags).

apm-agent-plugins/apm-web-plugin/src/main/java/co/elastic/apm/agent/web/BodyProcessor.java renamed to apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/BodyProcessor.java

@@ -11,9 +11,9 @@
  * the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,28 +22,29 @@
  * under the License.
  * #L%
  */
-package co.elastic.apm.agent.web;
+package co.elastic.apm.agent.impl.context;
 
-import co.elastic.apm.agent.impl.context.TransactionContext;
-import co.elastic.apm.agent.impl.context.Request;
 import co.elastic.apm.agent.impl.error.ErrorCapture;
 import co.elastic.apm.agent.impl.transaction.Transaction;
 import co.elastic.apm.agent.report.processor.Processor;
+import co.elastic.apm.agent.configuration.CoreConfiguration;
 import org.stagemonitor.configuration.ConfigurationRegistry;
 
-import static co.elastic.apm.agent.web.WebConfiguration.EventType.ALL;
-import static co.elastic.apm.agent.web.WebConfiguration.EventType.ERRORS;
-import static co.elastic.apm.agent.web.WebConfiguration.EventType.TRANSACTIONS;
+import static co.elastic.apm.agent.configuration.CoreConfiguration.EventType.ALL;
+import static co.elastic.apm.agent.configuration.CoreConfiguration.EventType.ERRORS;
+import static co.elastic.apm.agent.configuration.CoreConfiguration.EventType.TRANSACTIONS;
 
 /**
- * This processor redacts the body according to the {@link WebConfiguration#captureBody} configuration option
+ * This processor redacts the body according to the {@link co.elastic.apm.agent.configuration.CoreConfiguration#captureBody}
+ * configuration option
  */
+@SuppressWarnings("JavadocReference")
 public class BodyProcessor implements Processor {
 
-    private final WebConfiguration webConfiguration;
+    private final CoreConfiguration coreConfiguration;
 
     public BodyProcessor(ConfigurationRegistry configurationRegistry) {
-        webConfiguration = configurationRegistry.getConfig(WebConfiguration.class);
+        coreConfiguration = configurationRegistry.getConfig(CoreConfiguration.class);
     }
 
     @Override
@@ -56,14 +57,15 @@ public void processBeforeReport(ErrorCapture error) {
         redactBodyIfNecessary(error.getContext(), ERRORS);
     }
 
-    private void redactBodyIfNecessary(TransactionContext context, WebConfiguration.EventType eventType) {
-        final WebConfiguration.EventType eventTypeConfig = webConfiguration.getCaptureBody();
-        if (hasBody(context.getRequest()) && eventTypeConfig != eventType && eventTypeConfig != ALL) {
-            context.getRequest().redactBody();
+    private void redactBodyIfNecessary(TransactionContext context, CoreConfiguration.EventType eventType) {
+        final CoreConfiguration.EventType eventTypeConfig = coreConfiguration.getCaptureBody();
+        if (eventTypeConfig != eventType && eventTypeConfig != ALL) {
+            if (context.getRequest().getBody() != null) {
+                context.getRequest().redactBody();
+            }
+            if (context.getMessage().getBody() != null) {
+                context.getMessage().redactBody();
+            }
         }
     }
-
-    private boolean hasBody(Request request) {
-        return request.getBody() != null;
-    }
 }

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/Message.java

@@ -25,9 +25,12 @@
 package co.elastic.apm.agent.impl.context;
 
 import co.elastic.apm.agent.objectpool.Recyclable;
+import co.elastic.apm.agent.util.NoRandomAccessMap;
 
 import javax.annotation.Nullable;
 
+import static co.elastic.apm.agent.impl.context.AbstractContext.REDACTED_CONTEXT_STRING;
+
 public class Message implements Recyclable {
 
     @Nullable
@@ -36,6 +39,14 @@ public class Message implements Recyclable {
     @Nullable
     private String topicName;
 
+    @Nullable
+    private String body;
+
+    /**
+     * A mapping of message headers (in JMS includes properties as well)
+     */
+    private final NoRandomAccessMap<String, String> headers = new NoRandomAccessMap<>();
+
     @Nullable
     public String getQueueName() {
         return queueName;
@@ -56,18 +67,45 @@ public Message withTopic(String topicName) {
         return this;
     }
 
+    @Nullable
+    public String getBody() {
+        return body;
+    }
+
+    public Message withBody(@Nullable String body) {
+        this.body = body;
+        return this;
+    }
+
+    public void redactBody() {
+        body = REDACTED_CONTEXT_STRING;
+    }
+
+    public Message addHeader(String key, String value) {
+        headers.add(key, value);
+        return this;
+    }
+
+    public NoRandomAccessMap<String, String> getHeaders() {
+        return headers;
+    }
+
     public boolean hasContent() {
-        return queueName != null || topicName != null;
+        return queueName != null || topicName != null || body != null || headers.size() > 0;
     }
 
     @Override
     public void resetState() {
         queueName = null;
         topicName = null;
+        body = null;
+        headers.resetState();
     }
 
     public void copyFrom(Message other) {
         this.queueName = other.getQueueName();
         this.topicName = other.getTopicName();
+        this.body = other.body;
+        this.headers.copyFrom(other.getHeaders());
     }
 }

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/Request.java

@@ -38,6 +38,8 @@
 import java.nio.CharBuffer;
 import java.util.Enumeration;
 
+import static co.elastic.apm.agent.impl.context.AbstractContext.REDACTED_CONTEXT_STRING;
+
 
 /**
  * Request
@@ -130,7 +132,7 @@ public void setRawBody(String rawBody) {
     }
 
     public void redactBody() {
-        setRawBody("[REDACTED]");
+        setRawBody(REDACTED_CONTEXT_STRING);
     }
 
     public Request addFormUrlEncodedParameter(String key, String value) {

apm-agent-plugins/apm-web-plugin/src/main/java/co/elastic/apm/agent/web/SanitizingWebProcessor.java renamed to apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/SanitizingWebProcessor.java

@@ -11,9 +11,9 @@
  * the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,7 +22,7 @@
  * under the License.
  * #L%
  */
-package co.elastic.apm.agent.web;
+package co.elastic.apm.agent.impl.context;
 
 import co.elastic.apm.agent.configuration.CoreConfiguration;
 import co.elastic.apm.agent.impl.context.Request;
@@ -34,12 +34,13 @@
 import co.elastic.apm.agent.util.PotentiallyMultiValuedMap;
 import org.stagemonitor.configuration.ConfigurationRegistry;
 
+import static co.elastic.apm.agent.impl.context.AbstractContext.REDACTED_CONTEXT_STRING;
+
 /**
  * Sanitizes web-related fields according to the {@link CoreConfiguration#sanitizeFieldNames} setting
  */
 public class SanitizingWebProcessor implements Processor {
 
-    static final String REDACTED = "[REDACTED]";
     private final CoreConfiguration config;
 
     public SanitizingWebProcessor(ConfigurationRegistry configurationRegistry) {
@@ -75,7 +76,7 @@ private void sanitizeRequest(Request request) {
     private void sanitizeMap(PotentiallyMultiValuedMap map) {
         for (int i = 0; i < map.size(); i++) {
             if (isSensitive(map.getKey(i))) {
-                map.set(i, REDACTED);
+                map.set(i, REDACTED_CONTEXT_STRING);
             }
         }
     }

apm-agent-plugins/apm-web-plugin/src/main/java/co/elastic/apm/agent/web/ResultUtil.java renamed to apm-agent-core/src/main/java/co/elastic/apm/agent/impl/context/web/ResultUtil.java

@@ -11,9 +11,9 @@
  * the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -22,7 +22,7 @@
  * under the License.
  * #L%
  */
-package co.elastic.apm.agent.web;
+package co.elastic.apm.agent.impl.context.web;
 
 import javax.annotation.Nullable;