diff --git a/validator/src/main/java/com/amazon/aoc/App.java b/validator/src/main/java/com/amazon/aoc/App.java index 5074eccf4..674cfb2ef 100644 --- a/validator/src/main/java/com/amazon/aoc/App.java +++ b/validator/src/main/java/com/amazon/aoc/App.java @@ -157,6 +157,9 @@ public class App implements Callable { defaultValue = "defaultDnsName") private String privateDnsName; + @CommandLine.Option(names = {"--trace-id"}) + private String traceId; + private static final String TEST_CASE_DIM_KEY = "testcase"; private static final String CANARY_NAMESPACE = "Otel/Canary"; private static final String CANARY_METRIC_NAME = "Success"; @@ -196,6 +199,7 @@ public Integer call() throws Exception { context.setInstanceAmi(this.instanceAmi); context.setInstanceId(this.instanceId); context.setPrivateDnsName(this.privateDnsName); + context.setTraceId(this.traceId); log.info(context); diff --git a/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java b/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java index 4a23468d2..73f883c97 100644 --- a/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java +++ b/validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java @@ -246,6 +246,11 @@ public enum PredefinedExpectedTemplate implements FileConfig { /** Python EC2 ADOT SigV4 Log Exporter Test Case Validation */ PYTHON_EC2_ADOT_OTLP_LOG("/expected-data-template/python/ec2/adot-aws-otlp/application-log.mustache"), + /** Python EC2 ADOT Gen AI Test Case Validation */ + PYTHON_EC2_ADOT_GENAI_LOG("/expected-data-template/python/ec2/adot-genai/genai-log.mustache"), + PYTHON_EC2_ADOT_GENAI_TRACE("/expected-data-template/python/ec2/adot-genai/genai-trace.mustache"), + PYTHON_EC2_ADOT_GENAI_METRIC("/expected-data-template/python/ec2/adot-genai/genai-metric.mustache"), + /** Python K8S Test Case Validations */ PYTHON_K8S_OUTGOING_HTTP_CALL_LOG("/expected-data-template/python/k8s/outgoing-http-call-log.mustache"), PYTHON_K8S_OUTGOING_HTTP_CALL_METRIC("/expected-data-template/python/k8s/outgoing-http-call-metric.mustache"), diff --git a/validator/src/main/java/com/amazon/aoc/models/Context.java b/validator/src/main/java/com/amazon/aoc/models/Context.java index 6e607591a..6f5db130c 100644 --- a/validator/src/main/java/com/amazon/aoc/models/Context.java +++ b/validator/src/main/java/com/amazon/aoc/models/Context.java @@ -67,6 +67,8 @@ public class Context { private String privateDnsName; + private String traceId; + private ECSContext ecsContext; private CloudWatchContext cloudWatchContext; diff --git a/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java b/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java index 49af8d396..dec8d1dda 100644 --- a/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java +++ b/validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java @@ -27,6 +27,8 @@ import com.github.wnameless.json.flattener.FlattenMode; import com.github.wnameless.json.flattener.JsonFlattener; import com.github.wnameless.json.flattener.JsonifyArrayList; + +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -68,7 +70,23 @@ public void validate() throws Exception { Map actualLog; if (isAwsOtlpLog(expectedAttributes)) { - actualLog = this.getActualAwsOtlpLog(); + String otlpLogFilterPattern = String.format( + "{ ($.resource.attributes.['service.name'] = \"%s\") && ($.body = \"This is a custom log for validation testing\") }", + context.getServiceName() + ); + String addTraceIdFilter = (context.getTraceId() != null ? "&& ($.traceId = \"" + context.getTraceId() + "\") " : ""); + String genAILogFilterPattern = String.format( + "{ ($.resource.attributes.['service.name'] = \"%s\") " + + "&& ($.body.output.messages[0].role = \"assistant\") " + + "&& ($.body.input.messages[0].role = \"user\") " + + "&& ($.body.output.messages[1] NOT EXISTS) " + + "&& ($.body.input.messages[1] NOT EXISTS) " + + "%s" + + "}", + context.getServiceName(), + addTraceIdFilter + ); + actualLog = this.getActualAwsOtlpLog(Arrays.asList(otlpLogFilterPattern, genAILogFilterPattern)); } else { String operation = (String) expectedAttributes.get("Operation"); String remoteService = (String) expectedAttributes.get("RemoteService"); @@ -153,9 +171,12 @@ private JsonifyArrayList> getExpectedAttributes() throws Exc private boolean isAwsOtlpLog(Map expectedAttributes) { // OTLP SigV4 logs have 'body' as a top-level attribute - return expectedAttributes.containsKey("body") && - expectedAttributes.containsKey("severityNumber") && - expectedAttributes.containsKey("severityText"); + boolean hasBodyKey = expectedAttributes.keySet().stream() + .anyMatch(key -> key.startsWith("body")); + + return expectedAttributes.containsKey("severityNumber") && + expectedAttributes.containsKey("severityText") && + hasBodyKey; } private Map getActualLog( @@ -234,25 +255,33 @@ private Map getActualOtelSpanLog(String operation, String remote return JsonFlattener.flattenAsMap(retrievedLogs.get(0).getMessage()); } - private Map getActualAwsOtlpLog() throws Exception { - String filterPattern= String.format( - "{ ($.resource.attributes.['service.name'] = \"%s\") && ($.body = \"This is a custom log for validation testing\") }", - context.getServiceName() - ); - log.info("Filter Pattern for OTLP Log Search: " + filterPattern); + private Map getActualAwsOtlpLog(List filterPatterns) throws Exception { + log.info("Filter patterns {}", filterPatterns); - List retrievedLogs = - this.cloudWatchService.filterLogs( - context.getLogGroup(), - filterPattern, - System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5), - 10); + List retrievedLogs = null; + + for (String pattern : filterPatterns) { + log.info("Attempting filter Pattern for OTLP Log Search: {}", pattern); + + retrievedLogs = this.cloudWatchService.filterLogs( + context.getLogGroup(), + pattern, + System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5), + 10); + + if (retrievedLogs != null && !retrievedLogs.isEmpty()) { + log.info("Found logs for filter pattern {}", pattern); + break; + } + } if (retrievedLogs == null || retrievedLogs.isEmpty()) { - throw new BaseException(ExceptionCode.EMPTY_LIST); + throw new BaseException(ExceptionCode.EMPTY_LIST); } - return JsonFlattener.flattenAsMap(retrievedLogs.get(0).getMessage()); + return new JsonFlattener(retrievedLogs.get(0).getMessage()) + .withFlattenMode(FlattenMode.KEEP_ARRAYS) + .flattenAsMap(); } @Override diff --git a/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java b/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java index 3553bc1a8..dca2d964f 100644 --- a/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java +++ b/validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java @@ -91,6 +91,14 @@ public void validate() throws Exception { RetryHelper.retry( maxRetryCount, () -> { + String httpPath = validationConfig.getHttpPath(); + // Special handling for Genesis path - just check if any metrics exists in namespace + // since ADOT will just capture any OTel Metrics emitted from the instrumentation library used + // and convert them into EMF metrics, it's impossible to create a validation template for this. + if (httpPath != null && httpPath.contains("ai-chat")) { + validateAnyMetricExists(); + return; + } // We will query the Service, RemoteService, and RemoteTarget dimensions to ensure we // get all metrics from all aggregations, specifically the [RemoteService] aggregation. List serviceNames = @@ -210,6 +218,17 @@ private void compareMetricLists(List expectedMetricList, List ac matchAny.stream().findAny().get(), actualMetricSnapshot)); } } + + private void validateAnyMetricExists() throws Exception { + // This will grab all metrics from last 3 hours + // See: https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html + List allMetricsInNamespace = cloudWatchService.listMetrics(context.getMetricNamespace(), null, null, null); + log.info("Found {} metrics in namespace {}", allMetricsInNamespace.size(), context.getMetricNamespace()); + if (allMetricsInNamespace.isEmpty()) { + throw new BaseException(ExceptionCode.EXPECTED_METRIC_NOT_FOUND, "No metrics found in namespace: " + context.getMetricNamespace()); + } + log.info("validation is passed for path {}", validationConfig.getHttpPath()); + } private List listMetricFromCloudWatch( CloudWatchService cloudWatchService, diff --git a/validator/src/main/java/com/amazon/aoc/validators/TraceValidator.java b/validator/src/main/java/com/amazon/aoc/validators/TraceValidator.java index 21b762b9e..17886beb6 100644 --- a/validator/src/main/java/com/amazon/aoc/validators/TraceValidator.java +++ b/validator/src/main/java/com/amazon/aoc/validators/TraceValidator.java @@ -147,15 +147,15 @@ private Map getTrace() throws Exception { validationConfig.getHttpMethod().toUpperCase(), validationConfig.getHttpPath())); } + + if (validationConfig.getHttpPath() != null && validationConfig.getHttpPath().contains("ai-chat")) { + return this.getTraceById(Collections.singletonList(context.getTraceId())); + } + log.info("Trace Filter: {}", traceFilter); List retrieveTraceLists = xrayService.searchTraces(traceFilter); List traceIdLists = Collections.singletonList(retrieveTraceLists.get(0).getId()); - List retrievedTraceList = xrayService.listTraceByIds(traceIdLists); - - if (retrievedTraceList == null || retrievedTraceList.isEmpty()) { - throw new BaseException(ExceptionCode.EMPTY_LIST); - } - return this.flattenDocument(retrievedTraceList.get(0).getSegments()); + return getTraceById(traceIdLists); } private Map flattenDocument(List segmentList) { @@ -190,6 +190,14 @@ private Map flattenDocument(List segmentList) { return JsonFlattener.flattenAsMap(segmentsJson.toString()); } + private Map getTraceById(List traceIdLists) throws Exception { + List retrievedTraceList = xrayService.listTraceByIds(traceIdLists); + if (retrievedTraceList == null || retrievedTraceList.isEmpty()) { + throw new BaseException(ExceptionCode.EMPTY_LIST); + } + return this.flattenDocument(retrievedTraceList.get(0).getSegments()); + } + // This method will get the stored traces private Map getStoredTrace() throws Exception { Map flattenedJsonMapForStoredTraces = null; diff --git a/validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-log.mustache b/validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-log.mustache new file mode 100644 index 000000000..1eb79a202 --- /dev/null +++ b/validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-log.mustache @@ -0,0 +1,36 @@ +[{ + "resource": { + "attributes": { + "aws.local.service": "{{serviceName}}", + "aws.service.type": "gen_ai_agent", + "service.name": "{{serviceName}}" + } + }, + "scope": { + "name": "openinference.instrumentation.langchain" + }, + "severityNumber": "^[0-9]+$", + "severityText": ".*", + "body": { + "output": { + "messages": [ + { + "content": "^.+$", + "role": "assistant" + } + ] + }, + "input": { + "messages": [ + { + "content": "^.+$", + "role": "user" + } + ] + } + }, + "attributes": { + "event.name": "openinference.instrumentation.langchain" + }, + "traceId": "{{traceId}}" +}] \ No newline at end of file diff --git a/validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-metric.mustache b/validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-metric.mustache new file mode 100644 index 000000000..04d120344 --- /dev/null +++ b/validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-metric.mustache @@ -0,0 +1,4 @@ +- + metricName: ANY_VALUE + namespace: {{metricNamespace}} + dimensions: [] \ No newline at end of file diff --git a/validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-trace.mustache b/validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-trace.mustache new file mode 100644 index 000000000..bbad8e1ae --- /dev/null +++ b/validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-trace.mustache @@ -0,0 +1,20 @@ +[{ + "name": "^{{serviceName}}$", + "trace_id": "^{{traceId}}$", + "http": { + "request": { + "url": "^.*/ai-chat$", + "method": "^POST$" + } + }, + "aws": { + "service.type": "^gen_ai_agent$" + }, + "annotations": { + "aws.local.service": "^{{serviceName}}$", + "aws.local.operation": "^POST /ai-chat$" + }, + "metadata": { + "service.name": "^{{serviceName}}$" + } +}] \ No newline at end of file diff --git a/validator/src/main/resources/validations/python/ec2/adot-genai/log-validation.yml b/validator/src/main/resources/validations/python/ec2/adot-genai/log-validation.yml new file mode 100644 index 000000000..92c64fd06 --- /dev/null +++ b/validator/src/main/resources/validations/python/ec2/adot-genai/log-validation.yml @@ -0,0 +1,6 @@ +- + validationType: "cw-log" + httpPath: "ai-chat" + httpMethod: "post" + callingType: "http" + expectedLogStructureTemplate: "PYTHON_EC2_ADOT_GENAI_LOG" \ No newline at end of file diff --git a/validator/src/main/resources/validations/python/ec2/adot-genai/metric-validation.yml b/validator/src/main/resources/validations/python/ec2/adot-genai/metric-validation.yml new file mode 100644 index 000000000..2fd94b30a --- /dev/null +++ b/validator/src/main/resources/validations/python/ec2/adot-genai/metric-validation.yml @@ -0,0 +1,6 @@ +- + validationType: "cw-metric" + httpPath: "ai-chat" + httpMethod: "post" + callingType: "http" + expectedMetricTemplate: "PYTHON_EC2_ADOT_GENAI_METRIC" \ No newline at end of file diff --git a/validator/src/main/resources/validations/python/ec2/adot-genai/trace-validation.yml b/validator/src/main/resources/validations/python/ec2/adot-genai/trace-validation.yml new file mode 100644 index 000000000..34a3573c4 --- /dev/null +++ b/validator/src/main/resources/validations/python/ec2/adot-genai/trace-validation.yml @@ -0,0 +1,6 @@ +- + validationType: "trace" + httpPath: "ai-chat" + httpMethod: "post" + callingType: "http" + expectedTraceTemplate: "PYTHON_EC2_ADOT_GENAI_TRACE" \ No newline at end of file