Report the JVM/JMX metrics for each service name (#2233)

Author: tobiasstadler

CHANGELOG.asciidoc

@@ -44,6 +44,7 @@ by the agent may be different. This was done in order to improve the integration
 * Breakdown metrics are now tracked per service (when using APM Server 8.0) - {pull}2208[#2208]
 * Add support for Spring AMQP batch API - {pull}1716[#1716]
 * Add the (current) transaction name to the error (when using APM Server 8.0) - {pull}2235[#2235]
+* The JVM/JMX metrics are reported for each service name individually (when using APM Server 8.0) - {pull}2233[#2233]
 
 [float]
 ===== Performance improvements

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/ElasticApmTracer.java

@@ -54,8 +54,11 @@
 import javax.annotation.Nullable;
 import java.io.Closeable;
 import java.util.ArrayDeque;
+import java.util.ArrayList;
 import java.util.Deque;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -731,6 +734,14 @@ public MetricRegistry getMetricRegistry() {
         return metricRegistry;
     }
 
+    public List<String> getServiceNameOverrides() {
+        List<String> serviceNames = new ArrayList<>(serviceNameByClassLoader.approximateSize());
+        for (Map.Entry<ClassLoader, String> entry : serviceNameByClassLoader) {
+            serviceNames.add(entry.getValue());
+        }
+        return serviceNames;
+    }
+
     @Override
     public void overrideServiceNameForClassLoader(@Nullable ClassLoader classLoader, @Nullable String serviceName) {
         // overriding the service name for the bootstrap class loader is not an actual use-case

apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/DslJsonSerializer.java

@@ -77,6 +77,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import static co.elastic.apm.agent.util.ObjectUtils.defaultIfNull;
 import static com.dslplatform.json.JsonWriter.ARRAY_END;
 import static com.dslplatform.json.JsonWriter.ARRAY_START;
 import static com.dslplatform.json.JsonWriter.COMMA;
@@ -1055,9 +1056,9 @@ private static void serializeStringKeyScalarValueMap(Iterator<? extends Map.Entr
         jw.writeByte(OBJECT_END);
     }
 
-    static void serializeLabels(Labels labels, final StringBuilder replaceBuilder, final JsonWriter jw) {
+    static void serializeLabels(Labels labels, final String serviceName, final StringBuilder replaceBuilder, final JsonWriter jw) {
+        serializeServiceName(defaultIfNull(labels.getServiceName(), serviceName), replaceBuilder, jw);
         if (!labels.isEmpty()) {
-            serializeServiceName(labels.getServiceName(), replaceBuilder, jw);
             if (labels.getTransactionName() != null || labels.getTransactionType() != null) {
                 writeFieldName("transaction", jw);
                 jw.writeByte(OBJECT_START);

apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/MetricRegistryReporter.java

@@ -27,6 +27,7 @@
 import co.elastic.apm.agent.report.ReporterConfiguration;
 import com.dslplatform.json.JsonWriter;
 
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -67,8 +68,9 @@ public void stop() throws Exception {
     @Override
     public void report(Map<? extends Labels, MetricSet> metricSets) {
         if (tracer.isRunning()) {
+            List<String> serviceNames = tracer.getServiceNameOverrides();
             for (MetricSet metricSet : metricSets.values()) {
-                JsonWriter jw = serializer.serialize(metricSet);
+                JsonWriter jw = serializer.serialize(metricSet, serviceNames);
                 if (jw != null) {
                     reporter.report(jw);
                 }

apm-agent-core/src/main/java/co/elastic/apm/agent/report/serialize/MetricRegistrySerializer.java

@@ -27,6 +27,7 @@
 
 import javax.annotation.Nullable;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -47,18 +48,32 @@ public class MetricRegistrySerializer {
      * @return the serialized metric-set or {@code null} if no samples were serialized
      */
     @Nullable
-    public JsonWriter serialize(MetricSet metricSet) {
+    public JsonWriter serialize(MetricSet metricSet, List<String> serviceNames) {
         JsonWriter jw = dslJson.newWriter(maxSerializedSize);
-        final long timestamp = System.currentTimeMillis() * 1000;
-        boolean hasSamples = serialize(metricSet, timestamp, replaceBuilder, jw);
+        boolean hasSamples = false;
+        if (serviceNames.isEmpty() || metricSet.getLabels().getServiceName() != null) {
+            hasSamples = serialize(metricSet, null, jw);
+        } else {
+            hasSamples = serialize(metricSet, serviceNames.get(0), jw);
+            if (hasSamples) {
+                for (int i = 1; i < serviceNames.size(); ++i) {
+                    serialize(metricSet, serviceNames.get(i), jw);
+                }
+            }
+        }
         if (hasSamples) {
             maxSerializedSize = Math.max(Math.min(jw.size(), BUFFER_SIZE_LIMIT), maxSerializedSize);
             return jw;
         }
         return null;
     }
 
-    private static boolean serialize(MetricSet metricSet, long epochMicros, StringBuilder replaceBuilder, JsonWriter jw) {
+    private boolean serialize(MetricSet metricSet, String serviceName, JsonWriter jw) {
+        final long timestamp = System.currentTimeMillis() * 1000;
+        return serialize(metricSet, timestamp, serviceName, replaceBuilder, jw);
+    }
+
+    private static boolean serialize(MetricSet metricSet, long epochMicros, String serviceName, StringBuilder replaceBuilder, JsonWriter jw) {
         boolean hasSamples;
         jw.writeByte(JsonWriter.OBJECT_START);
         {
@@ -68,7 +83,7 @@ private static boolean serialize(MetricSet metricSet, long epochMicros, StringBu
                 DslJsonSerializer.writeFieldName("timestamp", jw);
                 NumberConverter.serialize(epochMicros, jw);
                 jw.writeByte(JsonWriter.COMMA);
-                DslJsonSerializer.serializeLabels(metricSet.getLabels(), replaceBuilder, jw);
+                DslJsonSerializer.serializeLabels(metricSet.getLabels(), serviceName, replaceBuilder, jw);
                 DslJsonSerializer.writeFieldName("samples", jw);
                 jw.writeByte(JsonWriter.OBJECT_START);
                 hasSamples = serializeGauges(metricSet.getGauges(), jw);

apm-agent-core/src/test/java/co/elastic/apm/agent/report/serialize/MetricRegistryReporterTest.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.report.serialize;
+
+import co.elastic.apm.agent.MockReporter;
+import co.elastic.apm.agent.configuration.SpyConfiguration;
+import co.elastic.apm.agent.configuration.source.PropertyFileConfigurationSource;
+import co.elastic.apm.agent.impl.ElasticApmTracer;
+import co.elastic.apm.agent.impl.ElasticApmTracerBuilder;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+import org.stagemonitor.configuration.source.SimpleSource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class MetricRegistryReporterTest {
+
+    @Test
+    void testReportedMetricsUseDefaultServiceNameIfServiceNameIsExplicitlySet() throws Exception {
+        ElasticApmTracer tracer = null;
+        try {
+            MockReporter reporter = new MockReporter();
+            tracer = new ElasticApmTracerBuilder()
+                .configurationRegistry(SpyConfiguration.createSpyConfig(SimpleSource.forTest("service_name", "foo")))
+                .reporter(reporter)
+                .buildAndStart();
+            tracer.overrideServiceNameForClassLoader(MetricRegistryReporterTest.class.getClassLoader(), "MetricRegistryReporterTest");
+
+            new MetricRegistryReporter(tracer).run();
+
+            assertThat(reporter.getBytes()).isNotEmpty();
+
+            ObjectMapper objectMapper = new ObjectMapper();
+            for (byte[] json : reporter.getBytes()) {
+                JsonNode jsonNode = objectMapper.readTree(json);
+                assertThat(jsonNode.get("metricset").get("service")).isNull();
+            }
+        } finally {
+            tracer.stop();
+        }
+    }
+}

apm-agent-core/src/test/java/co/elastic/apm/agent/report/serialize/MetricSetSerializationTest.java

@@ -27,8 +27,11 @@
 import org.junit.jupiter.api.Test;
 
 import javax.annotation.Nullable;
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 
@@ -185,11 +188,53 @@ void testServiceName() throws Exception {
         assertThat(serviceName.asText()).isEqualTo("bar");
     }
 
+    @Test
+    void testServiceNameOverrideWithOneService() throws Exception {
+        registry.updateTimer("foo", Labels.Mutable.of(), 1);
+
+        JsonNode jsonNode = reportAsJson(singletonList("bar"));
+        assertThat(jsonNode).isNotNull();
+        JsonNode serviceName = jsonNode.get("metricset").get("service").get("name");
+        assertThat(serviceName.asText()).isEqualTo("bar");
+    }
+
+    @Test
+    void testServiceNameOverrideWithMultipleService() throws Exception {
+        registry.updateTimer("foo", Labels.Mutable.of(), 1);
+
+        final CompletableFuture<JsonWriter> jwFuture = new CompletableFuture<>();
+        registry.flipPhaseAndReport(
+            metricSets -> jwFuture.complete(metricRegistrySerializer.serialize(metricSets.values().iterator().next(), List.of("bar1", "bar2")))
+        );
+
+        String[] jsonStrings = jwFuture.getNow(null).toString().split("\n");
+        assertThat(jsonStrings.length).isEqualTo(2);
+
+        JsonNode jsonNode1 = objectMapper.readTree(jsonStrings[0]);
+        String serviceName1 = jsonNode1.get("metricset").get("service").get("name").asText();
+        assertThat(serviceName1).isEqualTo("bar1");
+        JsonNode samples1 = jsonNode1.get("metricset").get("samples");
+        assertThat(samples1.get("foo.sum.us").get("value").intValue()).isOne();
+        assertThat(samples1.get("foo.count").get("value").intValue()).isOne();
+
+        JsonNode jsonNode2 = objectMapper.readTree(jsonStrings[1]);
+        String serviceName2 = jsonNode2.get("metricset").get("service").get("name").asText();
+        assertThat(serviceName2).isEqualTo("bar2");
+        JsonNode samples2 = jsonNode2.get("metricset").get("samples");
+        assertThat(samples2.get("foo.sum.us").get("value").intValue()).isOne();
+        assertThat(samples2.get("foo.count").get("value").intValue()).isOne();
+    }
+
     @Nullable
     private JsonNode reportAsJson() throws Exception {
+        return  reportAsJson(emptyList());
+    }
+
+    @Nullable
+    private JsonNode reportAsJson(List<String> serviceNames) throws Exception {
         final CompletableFuture<JsonWriter> jwFuture = new CompletableFuture<>();
         registry.flipPhaseAndReport(
-            metricSets -> jwFuture.complete(metricRegistrySerializer.serialize(metricSets.values().iterator().next()))
+            metricSets -> jwFuture.complete(metricRegistrySerializer.serialize(metricSets.values().iterator().next(), serviceNames))
         );
         JsonNode json = null;
         JsonWriter jw = jwFuture.getNow(null);

apm-agent-plugins/apm-jmx-plugin/src/test/java/co/elastic/apm/agent/jmx/JmxMetricTrackerTest.java

@@ -33,6 +33,7 @@
 import java.lang.management.ManagementFactory;
 import java.util.Arrays;
 
+import static java.util.Collections.emptyList;
 import static org.assertj.core.api.Assertions.assertThat;
 
 class JmxMetricTrackerTest {
@@ -162,7 +163,7 @@ private void printMetricSets() {
         metricRegistry.flipPhaseAndReport(
             metricSets -> {
                 metricSets.values().forEach(
-                    metricSet -> System.out.println(new MetricRegistrySerializer().serialize(metricSet).toString())
+                    metricSet -> System.out.println(new MetricRegistrySerializer().serialize(metricSet, emptyList()).toString())
                 );
             }
         );