diff --git a/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java b/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java index 86da0bfa5..222803435 100644 --- a/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java +++ b/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java @@ -104,6 +104,9 @@ public enum PredefinedExpectedTemplate implements FileConfig { /** Metric Limiter Test Case Validations */ JAVA_METRIC_LIMITER_METRIC("/expected-data-template/java/metric_limiter/metric-limiter-metric.mustache"), + JAVA_RUNTIME_METRIC_LOG("/expected-data-template/java/runtime/runtime-log.mustache"), + JAVA_RUNTIME_METRIC("/expected-data-template/java/runtime/runtime-metric.mustache"), + /** Python EKS Test Case Validations */ PYTHON_EKS_OUTGOING_HTTP_CALL_LOG("/expected-data-template/python/eks/outgoing-http-call-log.mustache"), PYTHON_EKS_OUTGOING_HTTP_CALL_METRIC("/expected-data-template/python/eks/outgoing-http-call-metric.mustache"), @@ -184,6 +187,9 @@ public enum PredefinedExpectedTemplate implements FileConfig { PYTHON_ECS_HC_CALL_METRIC("/expected-data-template/python/ecs/hc-metric.mustache"), PYTHON_ECS_HC_CALL_TRACE("/expected-data-template/python/ecs/hc-trace.mustache"), + PYTHON_RUNTIME_METRIC_LOG("/expected-data-template/python/runtime/runtime-log.mustache"), + PYTHON_RUNTIME_METRIC("/expected-data-template/python/runtime/runtime-metric.mustache"), + /** DotNet EC2 Default Test Case Validations */ DOTNET_EC2_DEFAULT_OUTGOING_HTTP_CALL_LOG( "/expected-data-template/dotnet/ec2/default/outgoing-http-call-log.mustache"), diff --git a/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java b/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java index d54f9e735..156644dfb 100644 --- a/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java +++ b/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java @@ -32,6 +32,8 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; + +import com.google.common.annotations.VisibleForTesting; import lombok.extern.log4j.Log4j2; @Log4j2 @@ -145,7 +147,14 @@ private Map getActualLog( dependencyFilter += String.format(" && ($.RemoteResourceType = %%%s%%) && ($.RemoteResourceIdentifier = %%%s%%)", remoteResourceType, remoteResourceIdentifier); } - String filterPattern = String.format("{ ($.Service = %s) && ($.Operation = \"%s\") %s }", context.getServiceName(), operation, dependencyFilter); + if (operation != null) { + dependencyFilter += String.format(" && ($.Operation = \"%s\")", operation); + } else { + // runtime metrics don't have Operation + dependencyFilter += "&& ($.Operation NOT EXISTS)"; + } + + String filterPattern = String.format("{ ($.Service = %s) %s }", context.getServiceName(), dependencyFilter); log.info("Filter Pattern for Log Search: " + filterPattern); List retrievedLogs = @@ -174,4 +183,14 @@ public void init( this.cloudWatchService = new CloudWatchService(context.getRegion()); this.maxRetryCount = DEFAULT_MAX_RETRY_COUNT; } + + @VisibleForTesting + public void setCloudWatchService(CloudWatchService cloudWatchService) { + this.cloudWatchService = cloudWatchService; + } + + @VisibleForTesting + public void setMaxRetryCount(int maxRetryCount) { + this.maxRetryCount = maxRetryCount; + } } diff --git a/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java b/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java index 072a1d3ad..012f63b7e 100644 --- a/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java +++ b/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java @@ -19,29 +19,31 @@ import com.amazon.aoc.exception.ExceptionCode; import com.amazon.aoc.fileconfigs.FileConfig; import com.amazon.aoc.helpers.CWMetricHelper; -import com.amazon.aoc.helpers.MustacheHelper; import com.amazon.aoc.helpers.RetryHelper; import com.amazon.aoc.models.Context; import com.amazon.aoc.models.ValidationConfig; import com.amazon.aoc.services.CloudWatchService; import com.amazonaws.services.cloudwatch.model.Dimension; import com.amazonaws.services.cloudwatch.model.Metric; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import lombok.extern.log4j.Log4j2; + import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.TreeSet; -import lombok.extern.log4j.Log4j2; @Log4j2 public class CWMetricValidator implements IValidator { - private static int DEFAULT_MAX_RETRY_COUNT = 80; + private static final int DEFAULT_MAX_RETRY_COUNT = 80; + private static final String ANY_VALUE = "ANY_VALUE"; - private MustacheHelper mustacheHelper = new MustacheHelper(); private Context context; private ValidationConfig validationConfig; private FileConfig expectedMetric; @@ -50,12 +52,12 @@ public class CWMetricValidator implements IValidator { private CWMetricHelper cwMetricHelper; private int maxRetryCount; - // for unit test + @VisibleForTesting public void setCloudWatchService(CloudWatchService cloudWatchService) { this.cloudWatchService = cloudWatchService; } - // for unit test so that we lower the count to 1 + @VisibleForTesting public void setMaxRetryCount(int maxRetryCount) { this.maxRetryCount = maxRetryCount; } @@ -134,7 +136,7 @@ public void validate() throws Exception { log.info("expected metricList is {}", expectedMetricList); compareMetricLists(expectedMetricList, actualMetricList); }); - + log.info("validation is passed for path {}", validationConfig.getHttpPath()); } @@ -152,50 +154,58 @@ private void addMetrics( } /** - * Check if every metric in toBeChckedMetricList is in baseMetricList. + * Check if every metric in expectedMetricList is in actualMetricList. * - * @param toBeCheckedMetricList toBeCheckedMetricList - * @param baseMetricList baseMetricList + * @param expectedMetricList expectedMetricList + * @param actualMetricList actualMetricList */ - private void compareMetricLists(List toBeCheckedMetricList, List baseMetricList) + private void compareMetricLists(List expectedMetricList, List actualMetricList) throws BaseException { - // load metrics into a hash set - Set metricSet = - new TreeSet<>( - (Metric o1, Metric o2) -> { - // check namespace - if (!o1.getNamespace().equals(o2.getNamespace())) { - return o1.getNamespace().compareTo(o2.getNamespace()); - } - - // check metric name - if (!o1.getMetricName().equals(o2.getMetricName())) { - return o1.getMetricName().compareTo(o2.getMetricName()); - } - - // sort and check dimensions - List dimensionList1 = o1.getDimensions(); - List dimensionList2 = o2.getDimensions(); - - // sort - dimensionList1.sort(Comparator.comparing(Dimension::getName)); - dimensionList2.sort(Comparator.comparing(Dimension::getName)); - - return dimensionList1.toString().compareTo(dimensionList2.toString()); - }); - for (Metric metric : baseMetricList) { - metricSet.add(metric); - } - for (Metric metric : toBeCheckedMetricList) { - if (!metricSet.contains(metric)) { - throw new BaseException( - ExceptionCode.EXPECTED_METRIC_NOT_FOUND, - String.format( - "metric in %ntoBeCheckedMetricList: %s is not found in %nbaseMetricList: %s %n", - metric, metricSet)); + Set matchAny = new HashSet<>(); + Set matchExact = new HashSet<>(); + for (Metric metric : expectedMetricList) { + metric.getDimensions().sort(Comparator.comparing(Dimension::getName)); + if (metric.getDimensions().stream().anyMatch(d -> ANY_VALUE.equals(d.getValue()))) { + matchAny.add(metric); + } else { + matchExact.add(metric); + } } - } + + Set actualMetricSet = new HashSet<>(); + for (Metric metric : actualMetricList) { + metric.getDimensions().sort(Comparator.comparing(Dimension::getName)); + actualMetricSet.add(metric); + } + Set actualMetricSnapshot = ImmutableSet.copyOf(actualMetricSet); + + actualMetricSet.removeAll(matchExact); + matchExact.removeAll(actualMetricSnapshot); + if (!matchExact.isEmpty()) { + throw new BaseException( + ExceptionCode.EXPECTED_METRIC_NOT_FOUND, + String.format( + "metric in %ntoBeCheckedMetricList: %s is not found in %nbaseMetricList: %s %n", + matchExact.stream().findAny().get(), actualMetricSnapshot)); + } + + Iterator iter = matchAny.iterator(); + while (iter.hasNext()) { + Metric expected = iter.next(); + for (Metric actual : actualMetricSet) { + if (metricEquals(expected, actual)) { + iter.remove(); + } + } + } + if (!matchAny.isEmpty()) { + throw new BaseException( + ExceptionCode.EXPECTED_METRIC_NOT_FOUND, + String.format( + "metric in %ntoBeCheckedMetricList: %s is not found in %nbaseMetricList: %s %n", + matchAny.stream().findAny().get(), actualMetricSnapshot)); + } } private List listMetricFromCloudWatch( @@ -220,6 +230,29 @@ private List listMetricFromCloudWatch( return result; } + private boolean metricEquals(Metric expected, Metric actual) { + if (expected.getNamespace().equals(actual.getNamespace()) + && expected.getMetricName().equals(actual.getMetricName())) { + if (expected.getDimensions().size() == actual.getDimensions().size()) { + for (int i = 0; i < expected.getDimensions().size(); i++) { + if (!dimensionEquals(expected.getDimensions().get(i), actual.getDimensions().get(i))) { + return false; + } + } + return true; + } + } + return false; + } + + private boolean dimensionEquals(Dimension expected, Dimension actual) { + if (expected.getName().equals(actual.getName())) { + return ANY_VALUE.equals(expected.getValue()) || + expected.getValue().equals(actual.getValue()); + } + return false; + } + @Override public void init( Context context, diff --git a/validator/src/main/resources/expected-data-template/java/runtime/runtime-log.mustache b/validator/src/main/resources/expected-data-template/java/runtime/runtime-log.mustache new file mode 100644 index 000000000..296d37f98 --- /dev/null +++ b/validator/src/main/resources/expected-data-template/java/runtime/runtime-log.mustache @@ -0,0 +1,4 @@ +[{ + "Telemetry.SDK": "opentelemetry,.*,java,Auto", + "Telemetry.Source": "^RuntimeMetric$" +}] \ No newline at end of file diff --git a/validator/src/main/resources/expected-data-template/java/runtime/runtime-metric.mustache b/validator/src/main/resources/expected-data-template/java/runtime/runtime-metric.mustache new file mode 100644 index 000000000..a55adf095 --- /dev/null +++ b/validator/src/main/resources/expected-data-template/java/runtime/runtime-metric.mustache @@ -0,0 +1,186 @@ +- + metricName: JVMGCDuration + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMGCCount + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMGCOldGenDuration + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMGCOldGenCount + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMGCYoungGenDuration + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMGCYoungGenCount + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMGCYoungGenCount + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMMemoryHeapUsed + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMMemoryNonHeapUsed + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMMemoryUsedAfterLastGC + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMMemoryOldGenUsed + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMMemorySurvivorSpaceUsed + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMMemoryEdenSpaceUsed + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMThreadCount + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMClassLoaded + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMCpuTime + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: JVMCpuRecentUtilization + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE \ No newline at end of file diff --git a/validator/src/main/resources/expected-data-template/python/runtime/runtime-log.mustache b/validator/src/main/resources/expected-data-template/python/runtime/runtime-log.mustache new file mode 100644 index 000000000..3f07b7c01 --- /dev/null +++ b/validator/src/main/resources/expected-data-template/python/runtime/runtime-log.mustache @@ -0,0 +1,4 @@ +[{ + "Telemetry.SDK": "opentelemetry,.*,python,Auto", + "Telemetry.Source": "^RuntimeMetric$" +}] \ No newline at end of file diff --git a/validator/src/main/resources/expected-data-template/python/runtime/runtime-metric.mustache b/validator/src/main/resources/expected-data-template/python/runtime/runtime-metric.mustache new file mode 100644 index 000000000..d13d3467a --- /dev/null +++ b/validator/src/main/resources/expected-data-template/python/runtime/runtime-metric.mustache @@ -0,0 +1,99 @@ +- + metricName: PythonProcessGCCount + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessGCGen0Count + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessGCGen1Count + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessGCGen2Count + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessVMSMemoryUsed + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessRSSMemoryUsed + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessThreadCount + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + + +- + metricName: PythonProcessCpuTime + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessCpuUtilization + namespace: {{metricNamespace}} + dimensions: + - + name: Service + value: {{serviceName}} + - + name: Environment + value: ANY_VALUE \ No newline at end of file diff --git a/validator/src/test/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplateTest.java b/validator/src/test/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplateTest.java index c64db6dcc..0b028f942 100644 --- a/validator/src/test/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplateTest.java +++ b/validator/src/test/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplateTest.java @@ -15,19 +15,22 @@ package com.amazon.aoc.fileconfigs; -import java.io.IOException; -import java.net.URL; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Test; +import java.net.URL; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + public class PredefinedExpectedTemplateTest { - @Test - public void ensureTemplatesAreExisting() throws IOException { - for (PredefinedExpectedTemplate predefinedExpectedTemplate : - PredefinedExpectedTemplate.values()) { - URL path = predefinedExpectedTemplate.getPath(); - // also check if tostring can return a valid filepath - IOUtils.toString(new URL(path.toString())); + @Test + public void ensureTemplatesAreExisting() throws Exception { + for (PredefinedExpectedTemplate predefinedExpectedTemplate : + PredefinedExpectedTemplate.values()) { + URL path = predefinedExpectedTemplate.getPath(); + assertNotNull(path); + // also check if tostring can return a valid filepath + IOUtils.toString(new URL(path.toString())); + } } - } } diff --git a/validator/src/test/java/com/amazon/aoc/validators/CWLogValidatorTest.java b/validator/src/test/java/com/amazon/aoc/validators/CWLogValidatorTest.java new file mode 100644 index 000000000..bb6030206 --- /dev/null +++ b/validator/src/test/java/com/amazon/aoc/validators/CWLogValidatorTest.java @@ -0,0 +1,72 @@ +package com.amazon.aoc.validators; + +import com.amazon.aoc.models.Context; +import com.amazon.aoc.models.ValidationConfig; +import com.amazon.aoc.services.CloudWatchService; +import com.amazonaws.services.logs.model.FilteredLogEvent; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.net.URL; +import java.nio.charset.Charset; +import java.util.List; + +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +@DisabledIf("isWindows") +public class CWLogValidatorTest extends ValidatorBaseTest { + private Context context; + @Mock + private CloudWatchService cloudWatchService; + + private final ObjectMapper mapper = new ObjectMapper(); + + static boolean isWindows() { + return System.getProperty("os.name").toLowerCase().startsWith("win"); + } + + @BeforeEach + public void setUp() throws Exception { + context = initContext(); + } + + @Test + public void testValidateRuntimeLogs() throws Exception { + String file = IOUtils.toString(new URL(TEMPLATE_ROOT + "log/actual-runtime.json"), Charset.defaultCharset()); + List result = mapper.readValue(file, new TypeReference>() { + }); + when(cloudWatchService.filterLogs( + Mockito.eq("/aws/application-signals/data"), + Mockito.eq("{ ($.Service = serviceName) && ($.RemoteService NOT EXISTS) && ($.RemoteOperation NOT EXISTS)&& ($.Operation NOT EXISTS) }"), + Mockito.anyLong(), + Mockito.anyInt() + )).thenReturn(result); + + ValidationConfig validationConfig = + initValidationConfig(TEMPLATE_ROOT + "log/expected-runtime.mustache"); + CWLogValidator cwLogValidator = new CWLogValidator(); + cwLogValidator.init(context, validationConfig, validationConfig.getExpectedLogStructureTemplate()); + cwLogValidator.setMaxRetryCount(1); + cwLogValidator.setCloudWatchService(cloudWatchService); + cwLogValidator.validate(); + } + + @Override + protected ValidationConfig initValidationConfig(String metricTemplate) { + ValidationConfig validationConfig = new ValidationConfig(); + validationConfig.setCallingType("none"); + validationConfig.setExpectedLogStructureTemplate(metricTemplate); + return validationConfig; + } + +} diff --git a/validator/src/test/java/com/amazon/aoc/validators/CWMetricValidatorTest.java b/validator/src/test/java/com/amazon/aoc/validators/CWMetricValidatorTest.java index bbe4250e8..da7a00129 100644 --- a/validator/src/test/java/com/amazon/aoc/validators/CWMetricValidatorTest.java +++ b/validator/src/test/java/com/amazon/aoc/validators/CWMetricValidatorTest.java @@ -144,6 +144,42 @@ public void testValidateEndToEnd_MissingRemoteService() throws Exception { } } + @Test + public void testValidateE2E_MatchAny() throws Exception{ + ValidationConfig validationConfig = + initValidationConfig(TEMPLATE_ROOT + "endToEnd_expectedRuntimeMetrics.mustache"); + + List localServiceMetrics = getTestMetrics("endToEnd_actualRuntimeMetrics"); + List remoteServiceMetrics = getTestMetrics("endToEnd_remoteMetricsWithService"); + // Skip remoteMetricsWithRemoteApp, which contains the [RemoteService] rollup. + List remoteMetricsWithRemoteApp = List.of(); + List remoteMetricsWithAmazon = getTestMetrics("endToEnd_remoteMetricsWithAmazon"); + List remoteMetricsWithAwsSdk = getTestMetrics("endToEnd_remoteMetricsWithAwsSdk"); + List remoteMetricsWithAwsSdkWithTarget = + getTestMetrics("endToEnd_remoteMetricsWithAwsSdk"); + + CloudWatchService cloudWatchService = + mockCloudWatchService( + localServiceMetrics, + remoteServiceMetrics, + remoteMetricsWithRemoteApp, + remoteMetricsWithAmazon, + remoteMetricsWithAwsSdk, + remoteMetricsWithAwsSdkWithTarget); + + try { + validate(validationConfig, cloudWatchService); + } catch (BaseException be) { + String actualMessage = be.getMessage(); + String expectedMessage = + "expectedMetricList: {Namespace: metricNamespace,MetricName: metricName,Dimensions: [{Name: RemoteService,Value: " + + REMOTE_SERVICE_DEPLOYMENT_NAME + + "}]} is not found in"; + assertTrue(actualMessage.contains(expectedMessage), actualMessage); + } + + } + private Context initContext() { // fake vars String testingId = "testingId"; diff --git a/validator/src/test/java/com/amazon/aoc/validators/ValidatorBaseTest.java b/validator/src/test/java/com/amazon/aoc/validators/ValidatorBaseTest.java index dd2030795..91e2a222f 100644 --- a/validator/src/test/java/com/amazon/aoc/validators/ValidatorBaseTest.java +++ b/validator/src/test/java/com/amazon/aoc/validators/ValidatorBaseTest.java @@ -28,6 +28,7 @@ protected Context initContext() { context.setRemoteServiceName(REMOTE_SERVICE_NAME); context.setRemoteServiceDeploymentName(REMOTE_SERVICE_DEPLOYMENT_NAME); context.setTestingId(TESTING_ID); + context.setLogGroup("/aws/application-signals/data"); return context; } diff --git a/validator/src/test/test-resources/endToEnd_actualRuntimeMetrics.mustache b/validator/src/test/test-resources/endToEnd_actualRuntimeMetrics.mustache new file mode 100644 index 000000000..12d438429 --- /dev/null +++ b/validator/src/test/test-resources/endToEnd_actualRuntimeMetrics.mustache @@ -0,0 +1,113 @@ +- + metricName: PythonProcessGCCount + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ec2:default + +- + metricName: PythonProcessGCGen0Count + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ec2:default + +- + metricName: PythonProcessGCGen1Count + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ec2:default + +- + metricName: PythonProcessGCGen2Count + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ec2:default + +- + metricName: PythonProcessVMSMemoryUsed + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ec2:default + +- + metricName: PythonProcessRSSMemoryUsed + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ec2:default + +- + metricName: PythonProcessThreadCount + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ec2:default + + +- + metricName: PythonProcessCpuTime + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ec2:default + +- + metricName: PythonProcessCpuUtilization + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ec2:default + +- + metricName: Fault + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ec2:default + - + name: Operation + value: operationName \ No newline at end of file diff --git a/validator/src/test/test-resources/endToEnd_expectedRuntimeMetrics.mustache b/validator/src/test/test-resources/endToEnd_expectedRuntimeMetrics.mustache new file mode 100644 index 000000000..64d1d3f50 --- /dev/null +++ b/validator/src/test/test-resources/endToEnd_expectedRuntimeMetrics.mustache @@ -0,0 +1,112 @@ +- + metricName: PythonProcessGCCount + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessGCGen0Count + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessGCGen1Count + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessGCGen2Count + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessVMSMemoryUsed + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessRSSMemoryUsed + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessThreadCount + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ANY_VALUE + + +- + metricName: PythonProcessCpuTime + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ANY_VALUE + +- + metricName: PythonProcessCpuUtilization + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ANY_VALUE +- + metricName: Fault + namespace: metricNamespace + dimensions: + - + name: Service + value: serviceName + - + name: Environment + value: ec2:default + - + name: Operation + value: operationName \ No newline at end of file diff --git a/validator/src/test/test-resources/log/actual-runtime.json b/validator/src/test/test-resources/log/actual-runtime.json new file mode 100644 index 000000000..03f293419 --- /dev/null +++ b/validator/src/test/test-resources/log/actual-runtime.json @@ -0,0 +1,12 @@ +[ + { + "timestamp": 1728448275567, + "message": "{\"EC2.InstanceId\":\"i-0aa390d458bf782b1\",\"Environment\":\"ec2:default\",\"Host\":\"ip-172-31-24-105.ec2.internal\",\"PlatformType\":\"AWS::EC2\",\"Service\":\"serviceName\",\"Telemetry.Agent\":\"CWAgent/1.300047.0-9-g132e908d\",\"Telemetry.SDK\":\"opentelemetry,1.33.0-aws-SNAPSHOT,java,Auto\",\"Telemetry.Source\":\"RuntimeMetric\",\"Version\":\"1\",\"_aws\":{\"CloudWatchMetrics\":[{\"Namespace\":\"ApplicationSignals\",\"Dimensions\":[[\"Environment\",\"Service\"]],\"Metrics\":[{\"Name\":\"JVMClassLoaded\",\"Unit\":\"1\"},{\"Name\":\"JVMMemoryNonHeapUsed\",\"Unit\":\"by\"},{\"Name\":\"JVMCpuRecentUtilization\",\"Unit\":\"1\"},{\"Name\":\"JVMMemoryHeapUsed\",\"Unit\":\"by\"},{\"Name\":\"JVMMemoryUsedAfterLastGC\",\"Unit\":\"by\"},{\"Name\":\"JVMMemoryOldGenUsed\",\"Unit\":\"by\"},{\"Name\":\"JVMMemorySurvivorSpaceUsed\",\"Unit\":\"by\"},{\"Name\":\"JVMMemoryEdenSpaceUsed\",\"Unit\":\"by\"},{\"Name\":\"JVMThreadCount\",\"Unit\":\"1\"},{\"Name\":\"JVMCpuTime\",\"Unit\":\"ns\"}]}],\"Timestamp\":1728448275567},\"JVMClassLoaded\":7685,\"JVMCpuRecentUtilization\":0.0007508212106992023,\"JVMCpuTime\":150530000000,\"JVMMemoryEdenSpaceUsed\":48234496,\"JVMMemoryHeapUsed\":74651136,\"JVMMemoryNonHeapUsed\":58388280,\"JVMMemoryOldGenUsed\":23270912,\"JVMMemorySurvivorSpaceUsed\":3145728,\"JVMMemoryUsedAfterLastGC\":20167680,\"JVMThreadCount\":16}", + "ingestionTime": 1728448275588 + }, + { + "timestamp": 1728448275567, + "message": "{\"EC2.InstanceId\":\"i-0aa390d458bf782b1\",\"Environment\":\"ec2:default\",\"Host\":\"ip-172-31-24-105.ec2.internal\",\"PlatformType\":\"AWS::EC2\",\"Service\":\"serviceName\",\"Telemetry.Agent\":\"CWAgent/1.300047.0-9-g132e908d\",\"Telemetry.SDK\":\"opentelemetry,1.33.0-aws-SNAPSHOT,java,Auto\",\"Telemetry.Source\":\"RuntimeMetric\",\"Version\":\"1\",\"_aws\":{\"CloudWatchMetrics\":[{\"Namespace\":\"ApplicationSignals\",\"Dimensions\":[[\"Environment\",\"Service\"]],\"Metrics\":[{\"Name\":\"JVMGCDuration\",\"Unit\":\"Milliseconds\"},{\"Name\":\"JVMGCCount\",\"Unit\":\"1\"},{\"Name\":\"JVMGCOldGenDuration\",\"Unit\":\"Milliseconds\"},{\"Name\":\"JVMGCYoungGenDuration\",\"Unit\":\"Milliseconds\"},{\"Name\":\"JVMGCOldGenCount\",\"Unit\":\"1\"},{\"Name\":\"JVMGCYoungGenCount\",\"Unit\":\"1\"}]}],\"Timestamp\":1728448275567},\"JVMGCCount\":0,\"JVMGCDuration\":0,\"JVMGCOldGenCount\":0,\"JVMGCOldGenDuration\":0,\"JVMGCYoungGenCount\":0,\"JVMGCYoungGenDuration\":0}", + "ingestionTime": 1728448275588 + } +] \ No newline at end of file diff --git a/validator/src/test/test-resources/log/expected-runtime.mustache b/validator/src/test/test-resources/log/expected-runtime.mustache new file mode 100644 index 000000000..296d37f98 --- /dev/null +++ b/validator/src/test/test-resources/log/expected-runtime.mustache @@ -0,0 +1,4 @@ +[{ + "Telemetry.SDK": "opentelemetry,.*,java,Auto", + "Telemetry.Source": "^RuntimeMetric$" +}] \ No newline at end of file