From dda628b5657f1dc1482803e1be6747bb50541ed3 Mon Sep 17 00:00:00 2001 From: Reno Seo Date: Fri, 20 Sep 2024 16:35:17 -0700 Subject: [PATCH 1/4] feat: Add contract tests for runtime metrics (#893) *Issue #, if available:* N/A *Description of changes:* This PR adds contract tests for runtime metrics feature, where it validates: 1. Are all runtime metrics captured? 2. Do the runtime metrics have a realistic value? (non-negative) These changes can easily be extended for higher coverage in the future. By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. --- .../test/base/ContractTestBase.java | 5 + .../test/misc/RuntimeMetricsTest.java | 95 +++++++++++++++++++ .../test/utils/AppSignalsConstants.java | 12 +++ .../test/utils/MockCollectorClient.java | 14 ++- 4 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/misc/RuntimeMetricsTest.java diff --git a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/base/ContractTestBase.java b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/base/ContractTestBase.java index 16bd1af119..3dd7e89a24 100644 --- a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/base/ContractTestBase.java +++ b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/base/ContractTestBase.java @@ -79,6 +79,7 @@ public abstract class ContractTestBase { .withEnv("JAVA_TOOL_OPTIONS", "-javaagent:/opentelemetry-javaagent-all.jar") .withEnv("OTEL_METRIC_EXPORT_INTERVAL", "100") // 100 ms .withEnv("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", "true") + .withEnv("OTEL_AWS_APPLICATION_SIGNALS_RUNTIME_ENABLED", isRuntimeEnabled()) .withEnv("OTEL_METRICS_EXPORTER", "none") .withEnv("OTEL_BSP_SCHEDULE_DELAY", "0") // Don't wait to export spans to the collector .withEnv( @@ -159,4 +160,8 @@ protected String getApplicationOtelServiceName() { protected String getApplicationOtelResourceAttributes() { return "service.name=" + getApplicationOtelServiceName(); } + + protected String isRuntimeEnabled() { + return "false"; + } } diff --git a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/misc/RuntimeMetricsTest.java b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/misc/RuntimeMetricsTest.java new file mode 100644 index 0000000000..0d7bfc84b9 --- /dev/null +++ b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/misc/RuntimeMetricsTest.java @@ -0,0 +1,95 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.opentelemetry.appsignals.test.misc; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.opentelemetry.proto.metrics.v1.NumberDataPoint; +import java.util.List; +import java.util.Set; +import org.junit.Assert; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.opentelemetry.appsignals.test.base.ContractTestBase; +import software.amazon.opentelemetry.appsignals.test.utils.AppSignalsConstants; + +public class RuntimeMetricsTest { + private abstract static class RuntimeMetricsContractTestBase extends ContractTestBase { + @Override + protected String getApplicationImageName() { + return "aws-appsignals-tests-http-server-spring-mvc"; + } + + @Override + protected String isRuntimeEnabled() { + return "true"; + } + + protected String getApplicationWaitPattern() { + return ".*Started Application.*"; + } + + protected void doTestRuntimeMetrics() { + var response = appClient.get("/success").aggregate().join(); + + assertThat(response.status().isSuccess()).isTrue(); + assertRuntimeMetrics(); + } + + protected void assertRuntimeMetrics() { + var metrics = + mockCollectorClient.getRuntimeMetrics( + Set.of( + AppSignalsConstants.JVM_GC_METRIC, + AppSignalsConstants.JVM_GC_COUNT, + AppSignalsConstants.JVM_HEAP_USED, + AppSignalsConstants.JVM_NON_HEAP_USED, + AppSignalsConstants.JVM_AFTER_GC, + AppSignalsConstants.JVM_POOL_USED, + AppSignalsConstants.JVM_THREAD_COUNT, + AppSignalsConstants.JVM_CLASS_LOADED, + AppSignalsConstants.JVM_CPU_TIME, + AppSignalsConstants.JVM_CPU_UTILIZATION, + AppSignalsConstants.LATENCY_METRIC, + AppSignalsConstants.ERROR_METRIC, + AppSignalsConstants.FAULT_METRIC)); + metrics.forEach( + metric -> { + var dataPoints = metric.getMetric().getGauge().getDataPointsList(); + assertNonNegativeValue(dataPoints); + }); + } + + private void assertNonNegativeValue(List dps) { + dps.forEach( + datapoint -> { + Assert.assertTrue(datapoint.getAsInt() >= 0); + }); + } + } + + @Testcontainers(disabledWithoutDocker = true) + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + @Nested + class ValidateRuntimeMetricsTest extends RuntimeMetricsContractTestBase { + @Test + void testRuntimeMetrics() { + doTestRuntimeMetrics(); + } + } +} diff --git a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/AppSignalsConstants.java b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/AppSignalsConstants.java index b2cf569bb2..b7f0ebb2da 100644 --- a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/AppSignalsConstants.java +++ b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/AppSignalsConstants.java @@ -33,4 +33,16 @@ public class AppSignalsConstants { public static final String AWS_REMOTE_RESOURCE_IDENTIFIER = "aws.remote.resource.identifier"; public static final String AWS_SPAN_KIND = "aws.span.kind"; public static final String AWS_REMOTE_DB_USER = "aws.remote.db.user"; + + // JVM Metrics + public static final String JVM_GC_METRIC = "jvm.gc.collections.elapsed"; + public static final String JVM_GC_COUNT = "jvm.gc.collections.count"; + public static final String JVM_HEAP_USED = "jvm.memory.heap.used"; + public static final String JVM_NON_HEAP_USED = "jvm.memory.nonheap.used"; + public static final String JVM_AFTER_GC = "jvm.memory.pool.used_after_last_gc"; + public static final String JVM_POOL_USED = "jvm.memory.pool.used"; + public static final String JVM_THREAD_COUNT = "jvm.threads.count"; + public static final String JVM_CLASS_LOADED = "jvm.classes.loaded"; + public static final String JVM_CPU_TIME = "jvm.cpu.time"; + public static final String JVM_CPU_UTILIZATION = "jvm.cpu.recent_utilization"; } diff --git a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/MockCollectorClient.java b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/MockCollectorClient.java index 64ecc191d7..f32f6e0399 100644 --- a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/MockCollectorClient.java +++ b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/MockCollectorClient.java @@ -133,13 +133,21 @@ public List getTraces() { .collect(toImmutableList()); } + public List getRuntimeMetrics(Set presentMetrics) { + return fetchMetrics(presentMetrics, true); + } + + public List getMetrics(Set presentMetrics) { + return fetchMetrics(presentMetrics, false); + } + /** * Get all metrics that are currently stored in the mock collector. * * @return List of `ResourceScopeMetric` which is a flat list containing all metrics and their * related scope and resources. */ - public List getMetrics(Set presentMetrics) { + private List fetchMetrics(Set presentMetrics, boolean isRuntime) { List exportedMetrics = waitForContent( "/get-metrics", @@ -153,7 +161,9 @@ public List getMetrics(Set presentMetrics) { .map(x -> x.getName()) .collect(Collectors.toSet()); - return (!exported.isEmpty() && current.size() == exported.size()) + return (isRuntime + ? !exported.isEmpty() && receivedMetrics.size() == presentMetrics.size() + : !exported.isEmpty() && current.size() == exported.size()) && receivedMetrics.containsAll(presentMetrics); }); From 01a4486a9206e1ac939c7ed953cb0f131d1b2370 Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Tue, 15 Oct 2024 13:28:13 -0700 Subject: [PATCH 2/4] Enable runtime metrics by default --- ...AwsApplicationSignalsCustomizerProvider.java | 3 ++- .../src/main/resources/jmx/rules/jvm.yaml | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/awsagentprovider/src/main/java/software/amazon/opentelemetry/javaagent/providers/AwsApplicationSignalsCustomizerProvider.java b/awsagentprovider/src/main/java/software/amazon/opentelemetry/javaagent/providers/AwsApplicationSignalsCustomizerProvider.java index df657538e0..4c4415a89d 100644 --- a/awsagentprovider/src/main/java/software/amazon/opentelemetry/javaagent/providers/AwsApplicationSignalsCustomizerProvider.java +++ b/awsagentprovider/src/main/java/software/amazon/opentelemetry/javaagent/providers/AwsApplicationSignalsCustomizerProvider.java @@ -104,7 +104,8 @@ private boolean isApplicationSignalsEnabled(ConfigProperties configProps) { } private boolean isApplicationSignalsRuntimeEnabled(ConfigProperties configProps) { - return false; + return isApplicationSignalsEnabled(configProps) + && configProps.getBoolean(APPLICATION_SIGNALS_RUNTIME_ENABLED_CONFIG, true); } private Map customizeProperties(ConfigProperties configProps) { diff --git a/instrumentation/jmx-metrics/src/main/resources/jmx/rules/jvm.yaml b/instrumentation/jmx-metrics/src/main/resources/jmx/rules/jvm.yaml index 18a4c01355..9c86719230 100644 --- a/instrumentation/jmx-metrics/src/main/resources/jmx/rules/jvm.yaml +++ b/instrumentation/jmx-metrics/src/main/resources/jmx/rules/jvm.yaml @@ -58,6 +58,9 @@ rules: metricAttribute: name: param(name) mapping: + CollectionUsage.used: + metric: used_after_last_gc + desc: Memory used after the most recent gc event Usage.init: metric: init desc: The initial amount of memory that the JVM requests from the operating system for the memory pool @@ -81,37 +84,49 @@ rules: metric: jvm.daemon_threads.count desc: Number of daemon threads - bean: java.lang:type=OperatingSystem - type: gauge mapping: TotalSwapSpaceSize: metric: jvm.system.swap.space.total + type: gauge desc: The host swap memory size in bytes unit: by FreeSwapSpaceSize: metric: jvm.system.swap.space.free + type: gauge desc: The amount of available swap memory in bytes unit: by TotalPhysicalMemorySize: metric: jvm.system.physical.memory.total + type: gauge desc: The total physical memory size in host unit: by FreePhysicalMemorySize: metric: jvm.system.physical.memory.free + type: gauge desc: The amount of free physical memory in host unit: by AvailableProcessors: metric: jvm.system.available.processors + type: gauge desc: The number of available processors unit: "1" SystemCpuLoad: metric: jvm.system.cpu.utilization + type: gauge desc: The current load of CPU in host unit: "1" + ProcessCpuTime: + metric: jvm.cpu.time + type: counter + unit: ns + desc: CPU time used ProcessCpuLoad: metric: jvm.cpu.recent_utilization + type: gauge unit: "1" desc: Recent CPU utilization for the process OpenFileDescriptorCount: metric: jvm.open_file_descriptor.count + type: gauge desc: The number of opened file descriptors unit: "1" From b2967c80e64b6c98bfaa6249ae708e94ae3383f9 Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Tue, 15 Oct 2024 20:43:57 -0700 Subject: [PATCH 3/4] Update contract tests for runtime --- .../test/misc/RuntimeMetricsTest.java | 96 ++++++++++++++++--- .../test/utils/AppSignalsConstants.java | 2 +- .../test/utils/MockCollectorClient.java | 19 ++-- 3 files changed, 95 insertions(+), 22 deletions(-) diff --git a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/misc/RuntimeMetricsTest.java b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/misc/RuntimeMetricsTest.java index 0d7bfc84b9..6c63a1606d 100644 --- a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/misc/RuntimeMetricsTest.java +++ b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/misc/RuntimeMetricsTest.java @@ -17,16 +17,16 @@ import static org.assertj.core.api.Assertions.assertThat; -import io.opentelemetry.proto.metrics.v1.NumberDataPoint; +import io.opentelemetry.proto.metrics.v1.Metric; import java.util.List; import java.util.Set; -import org.junit.Assert; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.testcontainers.junit.jupiter.Testcontainers; import software.amazon.opentelemetry.appsignals.test.base.ContractTestBase; import software.amazon.opentelemetry.appsignals.test.utils.AppSignalsConstants; +import software.amazon.opentelemetry.appsignals.test.utils.ResourceScopeMetric; public class RuntimeMetricsTest { private abstract static class RuntimeMetricsContractTestBase extends ContractTestBase { @@ -55,7 +55,7 @@ protected void assertRuntimeMetrics() { var metrics = mockCollectorClient.getRuntimeMetrics( Set.of( - AppSignalsConstants.JVM_GC_METRIC, + AppSignalsConstants.JVM_GC_DURATION, AppSignalsConstants.JVM_GC_COUNT, AppSignalsConstants.JVM_HEAP_USED, AppSignalsConstants.JVM_NON_HEAP_USED, @@ -68,18 +68,88 @@ protected void assertRuntimeMetrics() { AppSignalsConstants.LATENCY_METRIC, AppSignalsConstants.ERROR_METRIC, AppSignalsConstants.FAULT_METRIC)); - metrics.forEach( - metric -> { - var dataPoints = metric.getMetric().getGauge().getDataPointsList(); - assertNonNegativeValue(dataPoints); - }); + + testResourceAttributes(metrics); + for (String metricName : List.of(AppSignalsConstants.JVM_POOL_USED)) { + testGaugeMetrics(metrics, metricName, "name"); + } + for (String metricName : + List.of( + AppSignalsConstants.JVM_HEAP_USED, + AppSignalsConstants.JVM_NON_HEAP_USED, + AppSignalsConstants.JVM_AFTER_GC, + AppSignalsConstants.JVM_THREAD_COUNT, + AppSignalsConstants.JVM_CLASS_LOADED, + AppSignalsConstants.JVM_CPU_UTILIZATION)) { + testGaugeMetrics(metrics, metricName, ""); + } + for (String metricName : + List.of(AppSignalsConstants.JVM_GC_DURATION, AppSignalsConstants.JVM_GC_COUNT)) { + testCounterMetrics(metrics, metricName, "name"); + } + for (String metricName : List.of(AppSignalsConstants.JVM_CPU_TIME)) { + testCounterMetrics(metrics, metricName, ""); + } + } + + private void testGaugeMetrics( + List resourceScopeMetrics, String metricName, String attributeKey) { + for (ResourceScopeMetric rsm : resourceScopeMetrics) { + Metric metric = rsm.getMetric(); + if (metricName.equals(metric.getName())) { + assertThat(metric.getGauge().getDataPointsList()) + .as(metricName + " is not empty") + .isNotEmpty(); + assertThat(metric.getGauge().getDataPointsList()) + .as(metricName + " is valid") + .allMatch( + dp -> { + boolean valid = true; + if (!attributeKey.isEmpty()) { + valid = + dp.getAttributesList().stream() + .anyMatch(attribute -> attribute.getKey().equals(attributeKey)); + } + return valid && dp.getAsInt() >= 0; + }); + } + } + } + + private void testCounterMetrics( + List resourceScopeMetrics, String metricName, String attributeKey) { + for (ResourceScopeMetric rsm : resourceScopeMetrics) { + Metric metric = rsm.getMetric(); + if (metricName.equals(metric.getName())) { + assertThat(metric.getSum().getDataPointsList()) + .as(metricName + " is not empty") + .isNotEmpty(); + assertThat(metric.getSum().getDataPointsList()) + .as(metricName + " is valid") + .allMatch( + dp -> { + boolean valid = true; + if (!attributeKey.isEmpty()) { + valid = + dp.getAttributesList().stream() + .anyMatch(attribute -> attribute.getKey().equals(attributeKey)); + } + return valid && dp.getAsInt() >= 0; + }); + } + } } - private void assertNonNegativeValue(List dps) { - dps.forEach( - datapoint -> { - Assert.assertTrue(datapoint.getAsInt() >= 0); - }); + private void testResourceAttributes(List resourceScopeMetrics) { + for (ResourceScopeMetric rsm : resourceScopeMetrics) { + assertThat(rsm.getResource().getResource().getAttributesList()) + .anyMatch( + attr -> + attr.getKey().equals(AppSignalsConstants.AWS_LOCAL_SERVICE) + && attr.getValue() + .getStringValue() + .equals(getApplicationOtelServiceName())); + } } } diff --git a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/AppSignalsConstants.java b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/AppSignalsConstants.java index b7f0ebb2da..675b69032b 100644 --- a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/AppSignalsConstants.java +++ b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/AppSignalsConstants.java @@ -35,7 +35,7 @@ public class AppSignalsConstants { public static final String AWS_REMOTE_DB_USER = "aws.remote.db.user"; // JVM Metrics - public static final String JVM_GC_METRIC = "jvm.gc.collections.elapsed"; + public static final String JVM_GC_DURATION = "jvm.gc.collections.elapsed"; public static final String JVM_GC_COUNT = "jvm.gc.collections.count"; public static final String JVM_HEAP_USED = "jvm.memory.heap.used"; public static final String JVM_NON_HEAP_USED = "jvm.memory.nonheap.used"; diff --git a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/MockCollectorClient.java b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/MockCollectorClient.java index f32f6e0399..98fd1e963b 100644 --- a/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/MockCollectorClient.java +++ b/appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/MockCollectorClient.java @@ -134,11 +134,11 @@ public List getTraces() { } public List getRuntimeMetrics(Set presentMetrics) { - return fetchMetrics(presentMetrics, true); + return fetchMetrics(presentMetrics, false); } public List getMetrics(Set presentMetrics) { - return fetchMetrics(presentMetrics, false); + return fetchMetrics(presentMetrics, true); } /** @@ -147,7 +147,7 @@ public List getMetrics(Set presentMetrics) { * @return List of `ResourceScopeMetric` which is a flat list containing all metrics and their * related scope and resources. */ - private List fetchMetrics(Set presentMetrics, boolean isRuntime) { + private List fetchMetrics(Set presentMetrics, boolean exactMatch) { List exportedMetrics = waitForContent( "/get-metrics", @@ -160,11 +160,14 @@ private List fetchMetrics(Set presentMetrics, boole .flatMap(x -> x.getMetricsList().stream()) .map(x -> x.getName()) .collect(Collectors.toSet()); - - return (isRuntime - ? !exported.isEmpty() && receivedMetrics.size() == presentMetrics.size() - : !exported.isEmpty() && current.size() == exported.size()) - && receivedMetrics.containsAll(presentMetrics); + if (!exported.isEmpty() && receivedMetrics.containsAll(presentMetrics)) { + if (exactMatch) { + return current.size() == exported.size(); + } else { + return true; + } + } + return false; }); return exportedMetrics.stream() From fe9c6e197cb457218ad263e17b21bc9ac413c638 Mon Sep 17 00:00:00 2001 From: Mengyi Zhou Date: Fri, 18 Oct 2024 17:26:17 -0700 Subject: [PATCH 4/4] Fix metric unit --- .../src/main/resources/jmx/rules/jvm.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/instrumentation/jmx-metrics/src/main/resources/jmx/rules/jvm.yaml b/instrumentation/jmx-metrics/src/main/resources/jmx/rules/jvm.yaml index 9c86719230..908e52b899 100644 --- a/instrumentation/jmx-metrics/src/main/resources/jmx/rules/jvm.yaml +++ b/instrumentation/jmx-metrics/src/main/resources/jmx/rules/jvm.yaml @@ -23,7 +23,7 @@ rules: unit: ms desc: The approximate accumulated collection elapsed time in milliseconds - bean: java.lang:type=Memory - unit: by + unit: By prefix: jvm.memory. type: gauge mapping: @@ -52,7 +52,7 @@ rules: metric: nonheap.max desc: The maximum amount of memory can be used for non-heap purposes - bean: java.lang:type=MemoryPool,name=* - unit: by + unit: By prefix: jvm.memory.pool. type: gauge metricAttribute: @@ -88,23 +88,23 @@ rules: TotalSwapSpaceSize: metric: jvm.system.swap.space.total type: gauge - desc: The host swap memory size in bytes - unit: by + desc: The host swap memory size in Bytes + unit: By FreeSwapSpaceSize: metric: jvm.system.swap.space.free type: gauge - desc: The amount of available swap memory in bytes - unit: by + desc: The amount of available swap memory in Bytes + unit: By TotalPhysicalMemorySize: metric: jvm.system.physical.memory.total type: gauge desc: The total physical memory size in host - unit: by + unit: By FreePhysicalMemorySize: metric: jvm.system.physical.memory.free type: gauge desc: The amount of free physical memory in host - unit: by + unit: By AvailableProcessors: metric: jvm.system.available.processors type: gauge