[DO NOT MERGE] feat: Add contract tests for runtime metrics (#893)

*Issue #, if available:*
N/A

*Description of changes:*
This PR adds contract tests for runtime metrics feature, where it
validates:
1. Are all runtime metrics captured?
2. Do the runtime metrics have a realistic value? (non-negative)

These changes can easily be extended for higher coverage in the future.

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

Author: Reno Seo

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/base/ContractTestBase.java

@@ -79,7 +79,7 @@ public abstract class ContractTestBase {
           .withEnv("JAVA_TOOL_OPTIONS", "-javaagent:/opentelemetry-javaagent-all.jar")
           .withEnv("OTEL_METRIC_EXPORT_INTERVAL", "100") // 100 ms
           .withEnv("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", "true")
-          .withEnv("OTEL_AWS_APPLICATION_SIGNALS_RUNTIME_ENABLED", "false")
+          .withEnv("OTEL_AWS_APPLICATION_SIGNALS_RUNTIME_ENABLED", isRuntimeEnabled())
           .withEnv("OTEL_METRICS_EXPORTER", "none")
           .withEnv("OTEL_BSP_SCHEDULE_DELAY", "0") // Don't wait to export spans to the collector
           .withEnv(
@@ -160,4 +160,6 @@ protected String getApplicationOtelServiceName() {
   protected String getApplicationOtelResourceAttributes() {
     return "service.name=" + getApplicationOtelServiceName();
   }
+
+  protected String isRuntimeEnabled() { return "false"; }
 }

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/misc/RuntimeMetricsTest.java

@@ -0,0 +1,77 @@
+package software.amazon.opentelemetry.appsignals.test.misc;
+
+import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
+import org.junit.Assert;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import software.amazon.opentelemetry.appsignals.test.base.ContractTestBase;
+import software.amazon.opentelemetry.appsignals.test.utils.AppSignalsConstants;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class RuntimeMetricsTest {
+    private abstract static class RuntimeMetricsContractTestBase extends ContractTestBase {
+        @Override
+        protected String getApplicationImageName() {
+            return "aws-appsignals-tests-http-server-spring-mvc";
+        }
+
+        @Override
+        protected String isRuntimeEnabled() { return "true"; }
+
+        protected String getApplicationWaitPattern() {
+            return ".*Started Application.*";
+        }
+
+        protected void doTestRuntimeMetrics() {
+            var response = appClient.get("/success").aggregate().join();
+
+            assertThat(response.status().isSuccess()).isTrue();
+            assertRuntimeMetrics();
+        }
+        protected void assertRuntimeMetrics() {
+            var metrics =
+                    mockCollectorClient.getRuntimeMetrics(
+                            Set.of(
+                                    AppSignalsConstants.JVM_GC_METRIC,
+                                    AppSignalsConstants.JVM_GC_COUNT,
+                                    AppSignalsConstants.JVM_HEAP_USED,
+                                    AppSignalsConstants.JVM_NON_HEAP_USED,
+                                    AppSignalsConstants.JVM_AFTER_GC,
+                                    AppSignalsConstants.JVM_POOL_USED,
+                                    AppSignalsConstants.JVM_THREAD_COUNT,
+                                    AppSignalsConstants.JVM_CLASS_LOADED,
+                                    AppSignalsConstants.JVM_CPU_TIME,
+                                    AppSignalsConstants.JVM_CPU_UTILIZATION,
+                                    AppSignalsConstants.LATENCY_METRIC,
+                                    AppSignalsConstants.ERROR_METRIC,
+                                    AppSignalsConstants.FAULT_METRIC
+                            ));
+            metrics.forEach(metric -> {
+                var dataPoints = metric.getMetric().getGauge().getDataPointsList();
+                assertNonNegativeValue(dataPoints);
+            });
+        }
+
+        private void assertNonNegativeValue(List<NumberDataPoint> dps) {
+            dps.forEach(datapoint -> {
+                Assert.assertTrue(datapoint.getAsInt() >= 0);
+            });
+        }
+    }
+
+    @Testcontainers(disabledWithoutDocker = true)
+    @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+    @Nested
+    class ValidateRuntimeMetricsTest extends RuntimeMetricsContractTestBase {
+        @Test
+        void testRuntimeMetrics() {
+            doTestRuntimeMetrics();
+        }
+    }
+}

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/AppSignalsConstants.java

@@ -33,4 +33,16 @@ public class AppSignalsConstants {
   public static final String AWS_REMOTE_RESOURCE_IDENTIFIER = "aws.remote.resource.identifier";
   public static final String AWS_SPAN_KIND = "aws.span.kind";
   public static final String AWS_REMOTE_DB_USER = "aws.remote.db.user";
+
+  // JVM Metrics
+  public static final String JVM_GC_METRIC = "jvm.gc.collections.elapsed";
+  public static final String JVM_GC_COUNT = "jvm.gc.collections.count";
+  public static final String JVM_HEAP_USED = "jvm.memory.heap.used";
+  public static final String JVM_NON_HEAP_USED = "jvm.memory.nonheap.used";
+  public static final String JVM_AFTER_GC = "jvm.memory.pool.used_after_last_gc";
+  public static final String JVM_POOL_USED = "jvm.memory.pool.used";
+  public static final String JVM_THREAD_COUNT = "jvm.threads.count";
+  public static final String JVM_CLASS_LOADED = "jvm.classes.loaded";
+  public static final String JVM_CPU_TIME = "jvm.cpu.time";
+  public static final String JVM_CPU_UTILIZATION = "jvm.cpu.recent_utilization";
 }

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/MockCollectorClient.java

@@ -132,39 +132,45 @@ public List<ResourceScopeSpan> getTraces() {
                     .map(x -> new ResourceScopeSpan(ss.getFirst(), ss.getSecond(), x)))
         .collect(toImmutableList());
   }
+  public List<ResourceScopeMetric> getRuntimeMetrics(Set<String> presentMetrics) {
+    return fetchMetrics(presentMetrics, true);
+  }
+
+  public List<ResourceScopeMetric> getMetrics(Set<String> presentMetrics) {
+    return fetchMetrics(presentMetrics, false);
+  }
 
   /**
    * Get all metrics that are currently stored in the mock collector.
    *
    * @return List of `ResourceScopeMetric` which is a flat list containing all metrics and their
    *     related scope and resources.
    */
-  public List<ResourceScopeMetric> getMetrics(Set<String> presentMetrics) {
+  private List<ResourceScopeMetric> fetchMetrics(Set<String> presentMetrics, boolean isRuntime) {
     List<ExportMetricsServiceRequest> exportedMetrics =
-        waitForContent(
-            "/get-metrics",
-            EXPORT_METRICS_SERVICE_REQUEST_LIST,
-            (exported, current) -> {
-              Set<String> receivedMetrics =
-                  current.stream()
-                      .flatMap(x -> x.getResourceMetricsList().stream())
-                      .flatMap(x -> x.getScopeMetricsList().stream())
-                      .flatMap(x -> x.getMetricsList().stream())
-                      .map(x -> x.getName())
-                      .collect(Collectors.toSet());
-
-              return (!exported.isEmpty() && current.size() == exported.size())
-                  && receivedMetrics.containsAll(presentMetrics);
-            });
+            waitForContent(
+                    "/get-metrics",
+                    EXPORT_METRICS_SERVICE_REQUEST_LIST,
+                    (exported, current) -> {
+                      Set<String> receivedMetrics =
+                              current.stream()
+                                      .flatMap(x -> x.getResourceMetricsList().stream())
+                                      .flatMap(x -> x.getScopeMetricsList().stream())
+                                      .flatMap(x -> x.getMetricsList().stream())
+                                      .map(x -> x.getName())
+                                      .collect(Collectors.toSet());
+
+                      return (isRuntime ? !exported.isEmpty() && receivedMetrics.size() == presentMetrics.size() : !exported.isEmpty() && current.size() == exported.size())
+                              && receivedMetrics.containsAll(presentMetrics);
+                    });
 
     return exportedMetrics.stream()
-        .flatMap(req -> req.getResourceMetricsList().stream())
-        .flatMap(rm -> rm.getScopeMetricsList().stream().map(x -> new Pair<>(rm, x)))
-        .flatMap(
-            sm ->
-                sm.getSecond().getMetricsList().stream()
-                    .map(x -> new ResourceScopeMetric(sm.getFirst(), sm.getSecond(), x)))
-        .collect(toImmutableList());
+            .flatMap(req -> req.getResourceMetricsList().stream())
+            .flatMap(rm -> rm.getScopeMetricsList().stream().map(x -> new Pair<>(rm, x)))
+            .flatMap(sm ->
+                    sm.getSecond().getMetricsList().stream()
+                            .map(x -> new ResourceScopeMetric(sm.getFirst(), sm.getSecond(), x)))
+            .collect(toImmutableList());
   }
 
   private <T> List<T> waitForContent(String url, TypeReference<List<T>> t) {