[Bedrock AgentCore] Release Testing - Validator Changes

validator/src/main/java/com/amazon/aoc/App.java

@@ -157,6 +157,9 @@ public class App implements Callable<Integer> {
       defaultValue = "defaultDnsName")
   private String privateDnsName;
 
+  @CommandLine.Option(names = {"--trace-id"})
+  private String traceId;
+
   private static final String TEST_CASE_DIM_KEY = "testcase";
   private static final String CANARY_NAMESPACE = "Otel/Canary";
   private static final String CANARY_METRIC_NAME = "Success";
@@ -196,6 +199,7 @@ public Integer call() throws Exception {
     context.setInstanceAmi(this.instanceAmi);
     context.setInstanceId(this.instanceId);
     context.setPrivateDnsName(this.privateDnsName);
+    context.setTraceId(this.traceId);
 
     log.info(context);
 

validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java

@@ -246,6 +246,11 @@ public enum PredefinedExpectedTemplate implements FileConfig {
   /** Python EC2 ADOT SigV4 Log Exporter Test Case Validation */
   PYTHON_EC2_ADOT_OTLP_LOG("/expected-data-template/python/ec2/adot-aws-otlp/application-log.mustache"),
 
+  /** Python EC2 ADOT Gen AI Test Case Validation */
+  PYTHON_EC2_ADOT_GENAI_LOG("/expected-data-template/python/ec2/adot-genai/genai-log.mustache"),
+  PYTHON_EC2_ADOT_GENAI_TRACE("/expected-data-template/python/ec2/adot-genai/genai-trace.mustache"),
+  PYTHON_EC2_ADOT_GENAI_METRIC("/expected-data-template/python/ec2/adot-genai/genai-metric.mustache"),
+
   /** Python K8S Test Case Validations */
   PYTHON_K8S_OUTGOING_HTTP_CALL_LOG("/expected-data-template/python/k8s/outgoing-http-call-log.mustache"),
   PYTHON_K8S_OUTGOING_HTTP_CALL_METRIC("/expected-data-template/python/k8s/outgoing-http-call-metric.mustache"),

validator/src/main/java/com/amazon/aoc/models/Context.java

@@ -67,6 +67,8 @@ public class Context {
 
   private String privateDnsName;
 
+  private String traceId;
+
   private ECSContext ecsContext;
 
   private CloudWatchContext cloudWatchContext;

validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java

@@ -27,6 +27,8 @@
 import com.github.wnameless.json.flattener.FlattenMode;
 import com.github.wnameless.json.flattener.JsonFlattener;
 import com.github.wnameless.json.flattener.JsonifyArrayList;
+
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -68,7 +70,23 @@ public void validate() throws Exception {
           Map<String, Object> actualLog;
 
           if (isAwsOtlpLog(expectedAttributes)) {
-                    actualLog = this.getActualAwsOtlpLog();
+            String otlpLogFilterPattern = String.format(
+                "{ ($.resource.attributes.['service.name'] = \"%s\") && ($.body = \"This is a custom log for validation testing\") }",
+                context.getServiceName()
+            );
+            String addTraceIdFilter = (context.getTraceId() != null ? "&& ($.traceId = \"" + context.getTraceId() + "\") " : "");
+            String genAILogFilterPattern = String.format(
+                "{ ($.resource.attributes.['service.name'] = \"%s\") " +
+                "&& ($.body.output.messages[0].role = \"assistant\") " +
+                "&& ($.body.input.messages[0].role = \"user\") " +
+                "&& ($.body.output.messages[1] NOT EXISTS) " +
+                "&& ($.body.input.messages[1] NOT EXISTS) " +
+                "%s" +
+                "}",
+                context.getServiceName(),
+                addTraceIdFilter
+            );
+            actualLog = this.getActualAwsOtlpLog(Arrays.asList(otlpLogFilterPattern, genAILogFilterPattern));            
           } else {
             String operation = (String) expectedAttributes.get("Operation");
             String remoteService = (String) expectedAttributes.get("RemoteService");
@@ -153,9 +171,12 @@ private JsonifyArrayList<Map<String, Object>> getExpectedAttributes() throws Exc
 
   private boolean isAwsOtlpLog(Map<String, Object> expectedAttributes) {
     // OTLP SigV4 logs have 'body' as a top-level attribute
-    return expectedAttributes.containsKey("body") &&
-           expectedAttributes.containsKey("severityNumber") &&
-           expectedAttributes.containsKey("severityText");
+    boolean hasBodyKey = expectedAttributes.keySet().stream()
+        .anyMatch(key -> key.startsWith("body"));
+
+    return expectedAttributes.containsKey("severityNumber") && 
+           expectedAttributes.containsKey("severityText") &&
+           hasBodyKey;
   }
 
   private Map<String, Object> getActualLog(
@@ -234,25 +255,33 @@ private Map<String, Object> getActualOtelSpanLog(String operation, String remote
     return JsonFlattener.flattenAsMap(retrievedLogs.get(0).getMessage());
   }
 
-  private Map<String, Object> getActualAwsOtlpLog() throws Exception {
-    String filterPattern= String.format(
-            "{ ($.resource.attributes.['service.name'] = \"%s\") && ($.body = \"This is a custom log for validation testing\") }",
-            context.getServiceName()
-        );
-    log.info("Filter Pattern for OTLP Log Search: " + filterPattern);
+  private Map<String, Object> getActualAwsOtlpLog(List<String> filterPatterns) throws Exception {
+    log.info("Filter patterns {}", filterPatterns);
 
-    List<FilteredLogEvent> retrievedLogs =
-        this.cloudWatchService.filterLogs(
-            context.getLogGroup(),
-            filterPattern,
-            System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5),
-            10);
+    List<FilteredLogEvent> retrievedLogs = null;
+
+    for (String pattern : filterPatterns) {
+      log.info("Attempting filter Pattern for OTLP Log Search: {}", pattern);
+
+      retrievedLogs = this.cloudWatchService.filterLogs(
+          context.getLogGroup(),
+          pattern,
+          System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5),
+          10);
+
+      if (retrievedLogs != null && !retrievedLogs.isEmpty()) {
+        log.info("Found logs for filter pattern {}", pattern);
+        break;
+      }
+    }
 
     if (retrievedLogs == null || retrievedLogs.isEmpty()) {
-        throw new BaseException(ExceptionCode.EMPTY_LIST);
+      throw new BaseException(ExceptionCode.EMPTY_LIST);
     }
 
-    return JsonFlattener.flattenAsMap(retrievedLogs.get(0).getMessage());
+    return new JsonFlattener(retrievedLogs.get(0).getMessage())
+             .withFlattenMode(FlattenMode.KEEP_ARRAYS)
+             .flattenAsMap();
   }
 
   @Override

validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java

@@ -91,6 +91,14 @@ public void validate() throws Exception {
     RetryHelper.retry(
         maxRetryCount,
         () -> {
+          String httpPath = validationConfig.getHttpPath();
+          // Special handling for Genesis path - just check if any metrics exists in namespace
+          // since ADOT will just capture any OTel Metrics emitted from the instrumentation library used
+          // and convert them into EMF metrics, it's impossible to create a validation template for this.
+          if (httpPath != null && httpPath.contains("ai-chat")) {
+            validateAnyMetricExists();
+            return;
+          }
           // We will query the Service, RemoteService, and RemoteTarget dimensions to ensure we
           // get all metrics from all aggregations, specifically the [RemoteService] aggregation.
           List<String> serviceNames =
@@ -210,6 +218,17 @@ private void compareMetricLists(List<Metric> expectedMetricList, List<Metric> ac
                          matchAny.stream().findAny().get(), actualMetricSnapshot));
      }
   }
+
+  private void validateAnyMetricExists() throws Exception {
+    // This will grab all metrics from last 3 hours
+    // See: https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html
+    List<Metric> allMetricsInNamespace = cloudWatchService.listMetrics(context.getMetricNamespace(), null, null, null);
+    log.info("Found {} metrics in namespace {}", allMetricsInNamespace.size(), context.getMetricNamespace());
+    if (allMetricsInNamespace.isEmpty()) {
+      throw new BaseException(ExceptionCode.EXPECTED_METRIC_NOT_FOUND, "No metrics found in namespace: " + context.getMetricNamespace());
+    }
+    log.info("validation is passed for path {}", validationConfig.getHttpPath());
+  }
 
   private List<Metric> listMetricFromCloudWatch(
       CloudWatchService cloudWatchService,

validator/src/main/java/com/amazon/aoc/validators/TraceValidator.java

@@ -147,15 +147,15 @@ private Map<String, Object> getTrace() throws Exception {
               validationConfig.getHttpMethod().toUpperCase(),
               validationConfig.getHttpPath()));
     }
+
+    if (validationConfig.getHttpPath() != null && validationConfig.getHttpPath().contains("ai-chat")) {
+      return this.getTraceById(Collections.singletonList(context.getTraceId()));
+    }
+
     log.info("Trace Filter: {}", traceFilter);
     List<TraceSummary> retrieveTraceLists = xrayService.searchTraces(traceFilter);
     List<String> traceIdLists = Collections.singletonList(retrieveTraceLists.get(0).getId());
-    List<Trace> retrievedTraceList = xrayService.listTraceByIds(traceIdLists);
-
-    if (retrievedTraceList == null || retrievedTraceList.isEmpty()) {
-      throw new BaseException(ExceptionCode.EMPTY_LIST);
-    }
-    return this.flattenDocument(retrievedTraceList.get(0).getSegments());
+    return getTraceById(traceIdLists);
   }
 
   private Map<String, Object> flattenDocument(List<Segment> segmentList) {
@@ -190,6 +190,14 @@ private Map<String, Object> flattenDocument(List<Segment> segmentList) {
     return JsonFlattener.flattenAsMap(segmentsJson.toString());
   }
 
+  private Map<String, Object> getTraceById(List<String> traceIdLists) throws Exception {
+    List<Trace> retrievedTraceList = xrayService.listTraceByIds(traceIdLists);
+    if (retrievedTraceList == null || retrievedTraceList.isEmpty()) {
+      throw new BaseException(ExceptionCode.EMPTY_LIST);
+    }
+    return this.flattenDocument(retrievedTraceList.get(0).getSegments());
+  }
+
   // This method will get the stored traces
   private Map<String, Object> getStoredTrace() throws Exception {
     Map<String, Object> flattenedJsonMapForStoredTraces = null;

validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-log.mustache

@@ -0,0 +1,36 @@
+[{
+  "resource": {
+    "attributes": {
+      "aws.local.service": "{{serviceName}}",
+      "aws.service.type": "gen_ai_agent",
+      "service.name": "{{serviceName}}"
+    }
+  },
+  "scope": {
+    "name": "openinference.instrumentation.langchain"
+  },
+  "severityNumber": "^[0-9]+$",
+  "severityText": ".*",
+  "body": {
+    "output": {
+      "messages": [
+        {
+          "content": "^.+$",
+          "role": "assistant"
+        }
+      ]
+    },
+    "input": {
+      "messages": [
+        {
+          "content": "^.+$",
+          "role": "user"
+        }
+      ]
+    }
+  },
+  "attributes": {
+    "event.name": "openinference.instrumentation.langchain"
+  },
+  "traceId": "{{traceId}}"
+}]

validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-metric.mustache

@@ -0,0 +1,4 @@
+-
+  metricName: ANY_VALUE
+  namespace: {{metricNamespace}}
+  dimensions: []

validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-trace.mustache

@@ -0,0 +1,20 @@
+[{
+  "name": "^{{serviceName}}$",
+  "trace_id": "^{{traceId}}$",
+  "http": {
+    "request": {
+      "url": "^.*/ai-chat$",
+      "method": "^POST$"
+    }
+  },
+  "aws": {
+    "service.type": "^gen_ai_agent$"
+  },
+  "annotations": {
+    "aws.local.service": "^{{serviceName}}$",
+    "aws.local.operation": "^POST /ai-chat$"
+  },
+  "metadata": {
+    "service.name": "^{{serviceName}}$"  
+  }
+}]

validator/src/main/resources/validations/python/ec2/adot-genai/log-validation.yml

@@ -0,0 +1,6 @@
+-
+  validationType: "cw-log"
+  httpPath: "ai-chat"
+  httpMethod: "post"
+  callingType: "http"
+  expectedLogStructureTemplate: "PYTHON_EC2_ADOT_GENAI_LOG"

validator/src/main/resources/validations/python/ec2/adot-genai/metric-validation.yml

@@ -0,0 +1,6 @@
+-
+  validationType: "cw-metric"
+  httpPath: "ai-chat"
+  httpMethod: "post"
+  callingType: "http"
+  expectedMetricTemplate: "PYTHON_EC2_ADOT_GENAI_METRIC"

validator/src/main/resources/validations/python/ec2/adot-genai/trace-validation.yml

@@ -0,0 +1,6 @@
+-
+  validationType: "trace"
+  httpPath: "ai-chat"
+  httpMethod: "post"
+  callingType: "http"
+  expectedTraceTemplate: "PYTHON_EC2_ADOT_GENAI_TRACE"