Use display-name or context path as the default service name (#514)

Enables to have multiple service names per JVM,
one per class loader.

closes #136

Author: felixbarny

CHANGELOG.md

@@ -1,5 +1,13 @@
 # next (1.5.0)
 
+## Potentially breaking changes
+ * If you didn't explicitly set the [`service_name`](https://www.elastic.co/guide/en/apm/agent/java/master/config-core.html#config-service-name)
+   previously and you are dealing with a servlet-based application (including Spring Boot),
+   your `service_name` will change.
+   See the documentation for [`service_name`](https://www.elastic.co/guide/en/apm/agent/java/master/config-core.html#config-service-name)
+   and the corresponding section in _Features_ for more information.
+   Note: this requires APM Server 7.0+. If using previous versions, nothing will change.
+
 ## Features
  * Support for number and boolean labels in the public API (#497).
    This change also renames `tag` to `label` on the API level to be compliant with the [Elastic Common Schema (ECS)](https://github.com/elastic/ecs#-base-fields).
@@ -9,6 +17,11 @@
  * Support async queries made by Elasticsearch REST client 
  * Added `setStartTimestamp(long epochMicros)` and `end(long epochMicros)` API methods to `Span` and `Transaction`,
    allowing to set custom start and end timestamps.
+ * Auto-detection of the `service_name` based on the `<display-name>` element of the `web.xml` with a fallback to the servlet context path.
+   If you are using a spring-based application, the agent will use the setting for `spring.application.name` for its `service_name`.
+   See the documentation for [`service_name`](https://www.elastic.co/guide/en/apm/agent/java/master/config-core.html#config-service-name)
+   for more information.
+   Note: this requires APM Server 7.0+. If using previous versions, nothing will change.
 
 ## Bug Fixes
 

apm-agent-core/src/main/java/co/elastic/apm/agent/bci/methodmatching/TraceMethodInstrumentation.java

@@ -23,6 +23,7 @@
 import co.elastic.apm.agent.bci.bytebuddy.SimpleMethodSignatureOffsetMappingFactory;
 import co.elastic.apm.agent.configuration.CoreConfiguration;
 import co.elastic.apm.agent.impl.transaction.AbstractSpan;
+import co.elastic.apm.agent.impl.transaction.TraceContext;
 import co.elastic.apm.agent.impl.transaction.TraceContextHolder;
 import co.elastic.apm.agent.matcher.WildcardMatcher;
 import net.bytebuddy.asm.Advice;
@@ -58,12 +59,13 @@ public TraceMethodInstrumentation(MethodMatcher methodMatcher) {
     }
 
     @Advice.OnMethodEnter(suppress = Throwable.class)
-    public static void onMethodEnter(@SimpleMethodSignatureOffsetMappingFactory.SimpleMethodSignature String signature,
+    public static void onMethodEnter(@Advice.Origin Class<?> clazz,
+                                     @SimpleMethodSignatureOffsetMappingFactory.SimpleMethodSignature String signature,
                                      @Advice.Local("span") AbstractSpan<?> span) {
         if (tracer != null) {
             final TraceContextHolder<?> parent = tracer.getActive();
             if (parent == null) {
-                span = tracer.startTransaction()
+                span = tracer.startTransaction(TraceContext.asRoot(), null, clazz.getClassLoader())
                     .withName(signature)
                     .activate();
             } else {

apm-agent-core/src/main/java/co/elastic/apm/agent/configuration/CoreConfiguration.java

@@ -73,8 +73,20 @@ public class CoreConfiguration extends ConfigurationOptionProvider {
         .description("This is used to keep all the errors and transactions of your service together\n" +
             "and is the primary filter in the Elastic APM user interface.\n" +
             "\n" +
-            "NOTE: The service name must conform to this regular expression: ^[a-zA-Z0-9 _-]+$. In less regexy terms: Your service name " +
-            "must only contain characters from the ASCII alphabet, numbers, dashes, underscores and spaces.")
+            "The service name must conform to this regular expression: `^[a-zA-Z0-9 _-]+$`.\n" +
+            "In less regexy terms:\n" +
+            "Your service name must only contain characters from the ASCII alphabet, numbers, dashes, underscores and spaces.\n" +
+            "\n" +
+            "NOTE: When relying on auto-discovery of the service name in Servlet environments (including Spring Boot),\n" +
+            "there is currently a caveat related to metrics.\n" +
+            "The consequence is that the 'Metrics' tab of a service does not show process-global metrics like CPU utilization.\n" +
+            "The reason is that metrics are reported with the detected default service name for the JVM,\n" +
+            "for example `tomcat-application`.\n" +
+            "That is because there may be multiple web applications deployed to a single JVM/servlet container.\n" +
+            "However, you can view those metrics by selecting the `tomcat-application` service name, for example.\n" +
+            "Future versions of the Elastic APM stack will have better support for that scenario.\n" +
+            "A workaround is to explicitly set the `service_name` which means all applications deployed to the same servlet container will have the same name\n" +
+            "or to disable the corresponding `*-service-name` detecting instrumentations via <<config-disable-instrumentations>>.")
         .addValidator(RegexValidator.of("^[a-zA-Z0-9 _-]+$", "Your service name \"{0}\" must only contain characters " +
             "from the ASCII alphabet, numbers, dashes, underscores and spaces"))
         .buildWithDefault(ServiceNameUtil.getDefaultServiceName());
@@ -303,6 +315,10 @@ public String getServiceName() {
         return serviceName.get();
     }
 
+    public ConfigurationOption<String> getServiceNameConfig() {
+        return serviceName;
+    }
+
     public String getServiceVersion() {
         return serviceVersion.get();
     }

apm-agent-core/src/main/java/co/elastic/apm/agent/configuration/ServiceNameUtil.java

@@ -22,7 +22,7 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
-class ServiceNameUtil {
+public class ServiceNameUtil {
     private static final String JAR_VERSION_SUFFIX = "-(\\d+\\.)+(\\d+)(-.*)?$";
 
     static String getDefaultServiceName() {
@@ -70,7 +70,7 @@ private static String getSpecialServiceName(String command) {
         return null;
     }
 
-    private static String replaceDisallowedChars(String serviceName) {
+    public static String replaceDisallowedChars(String serviceName) {
         return serviceName.replaceAll("[^a-zA-Z0-9 _-]", "-");
     }
 

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/ElasticApmTracer.java

@@ -20,6 +20,7 @@
 package co.elastic.apm.agent.impl;
 
 import co.elastic.apm.agent.configuration.CoreConfiguration;
+import co.elastic.apm.agent.configuration.ServiceNameUtil;
 import co.elastic.apm.agent.context.LifecycleListener;
 import co.elastic.apm.agent.impl.async.ContextInScopeCallableWrapper;
 import co.elastic.apm.agent.impl.async.ContextInScopeRunnableWrapper;
@@ -40,6 +41,7 @@
 import co.elastic.apm.agent.objectpool.impl.QueueBasedObjectPool;
 import co.elastic.apm.agent.report.Reporter;
 import co.elastic.apm.agent.report.ReporterConfiguration;
+import com.blogspot.mydailyjava.weaklockfree.WeakConcurrentMap;
 import org.jctools.queues.atomic.AtomicQueueFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -97,6 +99,7 @@ protected Deque<TraceContextHolder<?>> initialValue() {
     private final MetricRegistry metricRegistry;
     private Sampler sampler;
     boolean assertionsEnabled = false;
+    private static final WeakConcurrentMap<ClassLoader, String> serviceNameByClassLoader = new WeakConcurrentMap.WithInlinedExpunction<>();
 
     ElasticApmTracer(ConfigurationRegistry configurationRegistry, Reporter reporter, Iterable<LifecycleListener> lifecycleListeners, List<ActivationListener> activationListeners) {
         this.metricRegistry = new MetricRegistry(configurationRegistry.getConfig(ReporterConfiguration.class));
@@ -178,15 +181,33 @@ public void onChange(ConfigurationOption<?> configurationOption, Double oldValue
         assert assertionsEnabled = true;
     }
 
-    public Transaction startTransaction() {
-        return startTransaction(TraceContext.asRoot(), null);
-    }
-
-    public <T> Transaction startTransaction(TraceContext.ChildContextCreator<T> childContextCreator, @Nullable T parent) {
-        return startTransaction(childContextCreator, parent, sampler, -1);
+    /**
+     * Starts a transaction as a child of the provided parent
+     *
+     * @param childContextCreator   used to make the transaction a child of the provided parent
+     * @param parent                the parent of the transaction. May be a traceparent header.
+     * @param initiatingClassLoader the class loader corresponding to the service which initiated the creation of the transaction.
+     *                              Used to determine the service name.
+     * @param <T>                   the type of the parent. {@code String} in case of a traceparent header.
+     * @return a transaction which is a child of the provided parent
+     */
+    public <T> Transaction startTransaction(TraceContext.ChildContextCreator<T> childContextCreator, @Nullable T parent, @Nullable ClassLoader initiatingClassLoader) {
+        return startTransaction(childContextCreator, parent, sampler, -1, initiatingClassLoader);
     }
 
-    public <T> Transaction startTransaction(TraceContext.ChildContextCreator<T> childContextCreator, @Nullable T parent, Sampler sampler, long epochMicros) {
+    /**
+     * Starts a transaction as a child of the provided parent
+     *
+     * @param childContextCreator   used to make the transaction a child of the provided parent
+     * @param parent                the parent of the transaction. May be a traceparent header.
+     * @param sampler               the {@link Sampler} instance which is responsible for determining the sampling decision if this is a root transaction
+     * @param epochMicros           the start timestamp
+     * @param initiatingClassLoader the class loader corresponding to the service which initiated the creation of the transaction
+     *                              Used to determine the service name.
+     * @param <T>                   the type of the parent. {@code String} in case of a traceparent header.
+     * @return a transaction which is a child of the provided parent
+     */
+    public <T> Transaction startTransaction(TraceContext.ChildContextCreator<T> childContextCreator, @Nullable T parent, Sampler sampler, long epochMicros, @Nullable ClassLoader initiatingClassLoader) {
         Transaction transaction;
         if (!coreConfiguration.isActive()) {
             transaction = noopTransaction();
@@ -200,6 +221,10 @@ public <T> Transaction startTransaction(TraceContext.ChildContextCreator<T> chil
                     new RuntimeException("this exception is just used to record where the transaction has been started from"));
             }
         }
+        final String serviceName = getServiceName(initiatingClassLoader);
+        if (serviceName != null) {
+            transaction.getTraceContext().setServiceName(serviceName);
+        }
         return transaction;
     }
 
@@ -269,11 +294,21 @@ private boolean isTransactionSpanLimitReached(Transaction transaction) {
         return coreConfiguration.getTransactionMaxSpans() <= transaction.getSpanCount().getStarted().get();
     }
 
-    public void captureException(@Nullable Throwable e) {
-        captureException(System.currentTimeMillis() * 1000, e, getActive());
+    /**
+     * Captures an exception without providing an explicit reference to a parent {@link TraceContextHolder}
+     *
+     * @param e                     the exception to capture
+     * @param initiatingClassLoader the class
+     */
+    public void captureException(@Nullable Throwable e, ClassLoader initiatingClassLoader) {
+        captureException(System.currentTimeMillis() * 1000, e, getActive(), initiatingClassLoader);
     }
 
-    public void captureException(long epochMicros, @Nullable Throwable e, @Nullable TraceContextHolder<?> parent) {
+    public void captureException(long epochMicros, @Nullable Throwable e, TraceContextHolder<?> parent) {
+        captureException(epochMicros, e, parent, null);
+    }
+
+    public void captureException(long epochMicros, @Nullable Throwable e, @Nullable TraceContextHolder<?> parent, @Nullable ClassLoader initiatingClassLoader) {
         if (e != null) {
             ErrorCapture error = errorPool.createInstance();
             error.withTimestamp(epochMicros);
@@ -287,6 +322,7 @@ public void captureException(long epochMicros, @Nullable Throwable e, @Nullable
                 error.asChildOf(parent);
             } else {
                 error.getTraceContext().getId().setToRandomValue();
+                error.getTraceContext().setServiceName(getServiceName(initiatingClassLoader));
             }
             reporter.report(error);
         }
@@ -463,4 +499,40 @@ private void assertIsActive(Object span, @Nullable Object currentlyActive) {
     public MetricRegistry getMetricRegistry() {
         return metricRegistry;
     }
+
+    /**
+     * Overrides the service name for all {@link Transaction}s,
+     * {@link Span}s and {@link ErrorCapture}s which are created by the service which corresponds to the provided {@link ClassLoader}.
+     * <p>
+     * The main use case is being able to differentiate between multiple services deployed to the same application server.
+     * </p>
+     *
+     * @param classLoader the class loader which corresponds to a particular service
+     * @param serviceName the service name for this class loader
+     */
+    public void overrideServiceNameForClassLoader(@Nullable ClassLoader classLoader, @Nullable String serviceName) {
+        // overriding the service name for the bootstrap class loader is not an actual use-case
+        // null may also mean we don't know about the initiating class loader
+        if (classLoader == null
+            || serviceName == null || serviceName.isEmpty()
+            // if the service name is set explicitly, don't override it
+            || !coreConfiguration.getServiceNameConfig().isDefault()) {
+            return;
+        }
+        if (!serviceNameByClassLoader.containsKey(classLoader)) {
+            serviceNameByClassLoader.putIfAbsent(classLoader, ServiceNameUtil.replaceDisallowedChars(serviceName));
+        }
+    }
+
+    @Nullable
+    private String getServiceName(@Nullable ClassLoader initiatingClassLoader) {
+        if (initiatingClassLoader == null) {
+            return null;
+        }
+        return serviceNameByClassLoader.get(initiatingClassLoader);
+    }
+
+    public void resetServiceNameOverrides() {
+        serviceNameByClassLoader.clear();
+    }
 }

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/transaction/AbstractSpan.java

@@ -114,7 +114,6 @@ public void resetState() {
         duration = 0;
         isLifecycleManagingThreadSwitch = false;
         traceContext.resetState();
-        // don't reset previouslyActive, as deactivate can be called after end
     }
 
     public boolean isChildOf(AbstractSpan<?> parent) {

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/transaction/TraceContext.java

@@ -25,6 +25,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nullable;
 import java.util.concurrent.Callable;
 
 /**
@@ -100,6 +101,8 @@ public boolean asChildOf(TraceContext child, Object ignore) {
      * @see EpochTickClock
      */
     private EpochTickClock clock = new EpochTickClock();
+    @Nullable
+    private String serviceName;
 
     private TraceContext(ElasticApmTracer tracer, Id id) {
         super(tracer);
@@ -214,6 +217,7 @@ public void asChildOf(TraceContext parent) {
         flags = parent.flags;
         id.setToRandomValue();
         clock.init(parent.clock);
+        serviceName = parent.serviceName;
         onMutation();
     }
 
@@ -229,6 +233,7 @@ public void resetState() {
         outgoingHeader.setLength(0);
         flags = 0;
         clock.resetState();
+        serviceName = null;
     }
 
     /**
@@ -335,6 +340,7 @@ public void copyFrom(TraceContext other) {
         outgoingHeader.append(other.outgoingHeader);
         flags = other.flags;
         clock.init(other.clock);
+        serviceName = other.serviceName;
         onMutation();
     }
 
@@ -351,6 +357,20 @@ public boolean isRoot() {
         return parentId.isEmpty();
     }
 
+    @Nullable
+    public String getServiceName() {
+        return serviceName;
+    }
+
+    /**
+     * Overrides the {@link co.elastic.apm.agent.impl.payload.Service#name} property sent via the meta data Intake V2 event.
+     *
+     * @param serviceName the service name for this event
+     */
+    public void setServiceName(@Nullable String serviceName) {
+        this.serviceName = serviceName;
+    }
+
     @Override
     public TraceContext getTraceContext() {
         return this;