feat: Add contract tests for runtime metrics (aws-observability#893)

*Issue #, if available:*
N/A

*Description of changes:*
This PR adds contract tests for runtime metrics feature, where it
validates:
1. Are all runtime metrics captured?
2. Do the runtime metrics have a realistic value? (non-negative)

These changes can easily be extended for higher coverage in the future.

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

Author: Reno Seo

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/base/ContractTestBase.java

@@ -79,6 +79,7 @@ public abstract class ContractTestBase {
           .withEnv("JAVA_TOOL_OPTIONS", "-javaagent:/opentelemetry-javaagent-all.jar")
           .withEnv("OTEL_METRIC_EXPORT_INTERVAL", "100") // 100 ms
           .withEnv("OTEL_AWS_APPLICATION_SIGNALS_ENABLED", "true")
+          .withEnv("OTEL_AWS_APPLICATION_SIGNALS_RUNTIME_ENABLED", isRuntimeEnabled())
           .withEnv("OTEL_METRICS_EXPORTER", "none")
           .withEnv("OTEL_BSP_SCHEDULE_DELAY", "0") // Don't wait to export spans to the collector
           .withEnv(
@@ -159,4 +160,8 @@ protected String getApplicationOtelServiceName() {
   protected String getApplicationOtelResourceAttributes() {
     return "service.name=" + getApplicationOtelServiceName();
   }
+
+  protected String isRuntimeEnabled() {
+    return "false";
+  }
 }

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/misc/RuntimeMetricsTest.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.opentelemetry.appsignals.test.misc;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
+import java.util.List;
+import java.util.Set;
+import org.junit.Assert;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import software.amazon.opentelemetry.appsignals.test.base.ContractTestBase;
+import software.amazon.opentelemetry.appsignals.test.utils.AppSignalsConstants;
+
+public class RuntimeMetricsTest {
+  private abstract static class RuntimeMetricsContractTestBase extends ContractTestBase {
+    @Override
+    protected String getApplicationImageName() {
+      return "aws-appsignals-tests-http-server-spring-mvc";
+    }
+
+    @Override
+    protected String isRuntimeEnabled() {
+      return "true";
+    }
+
+    protected String getApplicationWaitPattern() {
+      return ".*Started Application.*";
+    }
+
+    protected void doTestRuntimeMetrics() {
+      var response = appClient.get("/success").aggregate().join();
+
+      assertThat(response.status().isSuccess()).isTrue();
+      assertRuntimeMetrics();
+    }
+
+    protected void assertRuntimeMetrics() {
+      var metrics =
+          mockCollectorClient.getRuntimeMetrics(
+              Set.of(
+                  AppSignalsConstants.JVM_GC_METRIC,
+                  AppSignalsConstants.JVM_GC_COUNT,
+                  AppSignalsConstants.JVM_HEAP_USED,
+                  AppSignalsConstants.JVM_NON_HEAP_USED,
+                  AppSignalsConstants.JVM_AFTER_GC,
+                  AppSignalsConstants.JVM_POOL_USED,
+                  AppSignalsConstants.JVM_THREAD_COUNT,
+                  AppSignalsConstants.JVM_CLASS_LOADED,
+                  AppSignalsConstants.JVM_CPU_TIME,
+                  AppSignalsConstants.JVM_CPU_UTILIZATION,
+                  AppSignalsConstants.LATENCY_METRIC,
+                  AppSignalsConstants.ERROR_METRIC,
+                  AppSignalsConstants.FAULT_METRIC));
+      metrics.forEach(
+          metric -> {
+            var dataPoints = metric.getMetric().getGauge().getDataPointsList();
+            assertNonNegativeValue(dataPoints);
+          });
+    }
+
+    private void assertNonNegativeValue(List<NumberDataPoint> dps) {
+      dps.forEach(
+          datapoint -> {
+            Assert.assertTrue(datapoint.getAsInt() >= 0);
+          });
+    }
+  }
+
+  @Testcontainers(disabledWithoutDocker = true)
+  @TestInstance(TestInstance.Lifecycle.PER_CLASS)
+  @Nested
+  class ValidateRuntimeMetricsTest extends RuntimeMetricsContractTestBase {
+    @Test
+    void testRuntimeMetrics() {
+      doTestRuntimeMetrics();
+    }
+  }
+}

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/AppSignalsConstants.java

@@ -33,4 +33,16 @@ public class AppSignalsConstants {
   public static final String AWS_REMOTE_RESOURCE_IDENTIFIER = "aws.remote.resource.identifier";
   public static final String AWS_SPAN_KIND = "aws.span.kind";
   public static final String AWS_REMOTE_DB_USER = "aws.remote.db.user";
+
+  // JVM Metrics
+  public static final String JVM_GC_METRIC = "jvm.gc.collections.elapsed";
+  public static final String JVM_GC_COUNT = "jvm.gc.collections.count";
+  public static final String JVM_HEAP_USED = "jvm.memory.heap.used";
+  public static final String JVM_NON_HEAP_USED = "jvm.memory.nonheap.used";
+  public static final String JVM_AFTER_GC = "jvm.memory.pool.used_after_last_gc";
+  public static final String JVM_POOL_USED = "jvm.memory.pool.used";
+  public static final String JVM_THREAD_COUNT = "jvm.threads.count";
+  public static final String JVM_CLASS_LOADED = "jvm.classes.loaded";
+  public static final String JVM_CPU_TIME = "jvm.cpu.time";
+  public static final String JVM_CPU_UTILIZATION = "jvm.cpu.recent_utilization";
 }

appsignals-tests/contract-tests/src/test/java/software/amazon/opentelemetry/appsignals/test/utils/MockCollectorClient.java

@@ -133,13 +133,21 @@ public List<ResourceScopeSpan> getTraces() {
         .collect(toImmutableList());
   }
 
+  public List<ResourceScopeMetric> getRuntimeMetrics(Set<String> presentMetrics) {
+    return fetchMetrics(presentMetrics, true);
+  }
+
+  public List<ResourceScopeMetric> getMetrics(Set<String> presentMetrics) {
+    return fetchMetrics(presentMetrics, false);
+  }
+
   /**
    * Get all metrics that are currently stored in the mock collector.
    *
    * @return List of `ResourceScopeMetric` which is a flat list containing all metrics and their
    *     related scope and resources.
    */
-  public List<ResourceScopeMetric> getMetrics(Set<String> presentMetrics) {
+  private List<ResourceScopeMetric> fetchMetrics(Set<String> presentMetrics, boolean isRuntime) {
     List<ExportMetricsServiceRequest> exportedMetrics =
         waitForContent(
             "/get-metrics",
@@ -153,7 +161,9 @@ public List<ResourceScopeMetric> getMetrics(Set<String> presentMetrics) {
                       .map(x -> x.getName())
                       .collect(Collectors.toSet());
 
-              return (!exported.isEmpty() && current.size() == exported.size())
+              return (isRuntime
+                      ? !exported.isEmpty() && receivedMetrics.size() == presentMetrics.size()
+                      : !exported.isEmpty() && current.size() == exported.size())
                   && receivedMetrics.containsAll(presentMetrics);
             });