Skip to content

Commit 7b03654

Browse files
liustveliustve
authored
[Genesis] Release Testing - Validator Changes (#425)
*Description of changes:* This PR is part 1 of the release testing changes to support the incoming release of the latest Genesis/Agentic AI Observability changes done in ADOT Python which contains all of the changes to `validator` We need to validate that we are correctly emitting all 3 telemetry signals to CW backend. 1. Expands `Context` class to also take in a given traceId - as a part of the Agent Observability, we've split off user input/output prompts from Gen AI span events as a separate log event. The idea is that users will be able to rebuild their entire Gen AI span event by querying for logs with the same trace id in `aws/spans` and a custom log group. As a part of this testing we need to verify that trace Id is being propagated from their `Agentic AI application + ADOT --> CW` correctly and can be linked together. 2. Updated `AWS OTLP log filter` patterns to capture and validate GenAI input/output prompt log events. 3. Updated `TraceValidator` to validate pre-generated trace IDs for GenAI release testing 4. Updated `MetricValidator` to handle GenAI metrics special case: ADOT Python captures all OTel metrics from the Agentic AI application Instrumentor and converts them to EMF format. Validation for this path just checks for any metric emission to a custom namespace since the exact metric structure is unknown *Ensure you've run the following tests on your changes and include the link below:* Example of a successful run of this test: https://github.com/liustve/aws-application-signals-test-framework/actions/runs/16172487807/job/45649328445 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent 29a7b5f commit 7b03654

File tree

12 files changed

+169
-24
lines changed

12 files changed

+169
-24
lines changed

‎validator/src/main/java/com/amazon/aoc/App.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,9 @@ public class App implements Callable<Integer> {
157157
defaultValue = "defaultDnsName")
158158
private String privateDnsName;
159159

160+
@CommandLine.Option(names = {"--trace-id"})
161+
private String traceId;
162+
160163
private static final String TEST_CASE_DIM_KEY = "testcase";
161164
private static final String CANARY_NAMESPACE = "Otel/Canary";
162165
private static final String CANARY_METRIC_NAME = "Success";
@@ -196,6 +199,7 @@ public Integer call() throws Exception {
196199
context.setInstanceAmi(this.instanceAmi);
197200
context.setInstanceId(this.instanceId);
198201
context.setPrivateDnsName(this.privateDnsName);
202+
context.setTraceId(this.traceId);
199203

200204
log.info(context);
201205

‎validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,11 @@ public enum PredefinedExpectedTemplate implements FileConfig {
246246
/** Python EC2 ADOT SigV4 Log Exporter Test Case Validation */
247247
PYTHON_EC2_ADOT_OTLP_LOG("/expected-data-template/python/ec2/adot-aws-otlp/application-log.mustache"),
248248

249+
/** Python EC2 ADOT Gen AI Test Case Validation */
250+
PYTHON_EC2_ADOT_GENAI_LOG("/expected-data-template/python/ec2/adot-genai/genai-log.mustache"),
251+
PYTHON_EC2_ADOT_GENAI_TRACE("/expected-data-template/python/ec2/adot-genai/genai-trace.mustache"),
252+
PYTHON_EC2_ADOT_GENAI_METRIC("/expected-data-template/python/ec2/adot-genai/genai-metric.mustache"),
253+
249254
/** Python K8S Test Case Validations */
250255
PYTHON_K8S_OUTGOING_HTTP_CALL_LOG("/expected-data-template/python/k8s/outgoing-http-call-log.mustache"),
251256
PYTHON_K8S_OUTGOING_HTTP_CALL_METRIC("/expected-data-template/python/k8s/outgoing-http-call-metric.mustache"),

‎validator/src/main/java/com/amazon/aoc/models/Context.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ public class Context {
6767

6868
private String privateDnsName;
6969

70+
private String traceId;
71+
7072
private ECSContext ecsContext;
7173

7274
private CloudWatchContext cloudWatchContext;

‎validator/src/main/java/com/amazon/aoc/validators/CWLogValidator.java‎

Lines changed: 47 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
import com.github.wnameless.json.flattener.FlattenMode;
2828
import com.github.wnameless.json.flattener.JsonFlattener;
2929
import com.github.wnameless.json.flattener.JsonifyArrayList;
30+
31+
import java.util.Arrays;
3032
import java.util.List;
3133
import java.util.Map;
3234
import java.util.concurrent.TimeUnit;
@@ -68,7 +70,23 @@ public void validate() throws Exception {
6870
Map<String, Object> actualLog;
6971

7072
if (isAwsOtlpLog(expectedAttributes)) {
71-
actualLog = this.getActualAwsOtlpLog();
73+
String otlpLogFilterPattern = String.format(
74+
"{ ($.resource.attributes.['service.name'] = \"%s\") && ($.body = \"This is a custom log for validation testing\") }",
75+
context.getServiceName()
76+
);
77+
String addTraceIdFilter = (context.getTraceId() != null ? "&& ($.traceId = \"" + context.getTraceId() + "\") " : "");
78+
String genAILogFilterPattern = String.format(
79+
"{ ($.resource.attributes.['service.name'] = \"%s\") " +
80+
"&& ($.body.output.messages[0].role = \"assistant\") " +
81+
"&& ($.body.input.messages[0].role = \"user\") " +
82+
"&& ($.body.output.messages[1] NOT EXISTS) " +
83+
"&& ($.body.input.messages[1] NOT EXISTS) " +
84+
"%s" +
85+
"}",
86+
context.getServiceName(),
87+
addTraceIdFilter
88+
);
89+
actualLog = this.getActualAwsOtlpLog(Arrays.asList(otlpLogFilterPattern, genAILogFilterPattern));
7290
} else {
7391
String operation = (String) expectedAttributes.get("Operation");
7492
String remoteService = (String) expectedAttributes.get("RemoteService");
@@ -153,9 +171,12 @@ private JsonifyArrayList<Map<String, Object>> getExpectedAttributes() throws Exc
153171

154172
private boolean isAwsOtlpLog(Map<String, Object> expectedAttributes) {
155173
// OTLP SigV4 logs have 'body' as a top-level attribute
156-
return expectedAttributes.containsKey("body") &&
157-
expectedAttributes.containsKey("severityNumber") &&
158-
expectedAttributes.containsKey("severityText");
174+
boolean hasBodyKey = expectedAttributes.keySet().stream()
175+
.anyMatch(key -> key.startsWith("body"));
176+
177+
return expectedAttributes.containsKey("severityNumber") &&
178+
expectedAttributes.containsKey("severityText") &&
179+
hasBodyKey;
159180
}
160181

161182
private Map<String, Object> getActualLog(
@@ -234,25 +255,33 @@ private Map<String, Object> getActualOtelSpanLog(String operation, String remote
234255
return JsonFlattener.flattenAsMap(retrievedLogs.get(0).getMessage());
235256
}
236257

237-
private Map<String, Object> getActualAwsOtlpLog() throws Exception {
238-
String filterPattern= String.format(
239-
"{ ($.resource.attributes.['service.name'] = \"%s\") && ($.body = \"This is a custom log for validation testing\") }",
240-
context.getServiceName()
241-
);
242-
log.info("Filter Pattern for OTLP Log Search: " + filterPattern);
258+
private Map<String, Object> getActualAwsOtlpLog(List<String> filterPatterns) throws Exception {
259+
log.info("Filter patterns {}", filterPatterns);
243260

244-
List<FilteredLogEvent> retrievedLogs =
245-
this.cloudWatchService.filterLogs(
246-
context.getLogGroup(),
247-
filterPattern,
248-
System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5),
249-
10);
261+
List<FilteredLogEvent> retrievedLogs = null;
262+
263+
for (String pattern : filterPatterns) {
264+
log.info("Attempting filter Pattern for OTLP Log Search: {}", pattern);
265+
266+
retrievedLogs = this.cloudWatchService.filterLogs(
267+
context.getLogGroup(),
268+
pattern,
269+
System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5),
270+
10);
271+
272+
if (retrievedLogs != null && !retrievedLogs.isEmpty()) {
273+
log.info("Found logs for filter pattern {}", pattern);
274+
break;
275+
}
276+
}
250277

251278
if (retrievedLogs == null || retrievedLogs.isEmpty()) {
252-
throw new BaseException(ExceptionCode.EMPTY_LIST);
279+
throw new BaseException(ExceptionCode.EMPTY_LIST);
253280
}
254281

255-
return JsonFlattener.flattenAsMap(retrievedLogs.get(0).getMessage());
282+
return new JsonFlattener(retrievedLogs.get(0).getMessage())
283+
.withFlattenMode(FlattenMode.KEEP_ARRAYS)
284+
.flattenAsMap();
256285
}
257286

258287
@Override

‎validator/src/main/java/com/amazon/aoc/validators/CWMetricValidator.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ public void validate() throws Exception {
9191
RetryHelper.retry(
9292
maxRetryCount,
9393
() -> {
94+
String httpPath = validationConfig.getHttpPath();
95+
// Special handling for Genesis path - just check if any metrics exists in namespace
96+
// since ADOT will just capture any OTel Metrics emitted from the instrumentation library used
97+
// and convert them into EMF metrics, it's impossible to create a validation template for this.
98+
if (httpPath != null && httpPath.contains("ai-chat")) {
99+
validateAnyMetricExists();
100+
return;
101+
}
94102
// We will query the Service, RemoteService, and RemoteTarget dimensions to ensure we
95103
// get all metrics from all aggregations, specifically the [RemoteService] aggregation.
96104
List<String> serviceNames =
@@ -210,6 +218,17 @@ private void compareMetricLists(List<Metric> expectedMetricList, List<Metric> ac
210218
matchAny.stream().findAny().get(), actualMetricSnapshot));
211219
}
212220
}
221+
222+
private void validateAnyMetricExists() throws Exception {
223+
// This will grab all metrics from last 3 hours
224+
// See: https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html
225+
List<Metric> allMetricsInNamespace = cloudWatchService.listMetrics(context.getMetricNamespace(), null, null, null);
226+
log.info("Found {} metrics in namespace {}", allMetricsInNamespace.size(), context.getMetricNamespace());
227+
if (allMetricsInNamespace.isEmpty()) {
228+
throw new BaseException(ExceptionCode.EXPECTED_METRIC_NOT_FOUND, "No metrics found in namespace: " + context.getMetricNamespace());
229+
}
230+
log.info("validation is passed for path {}", validationConfig.getHttpPath());
231+
}
213232

214233
private List<Metric> listMetricFromCloudWatch(
215234
CloudWatchService cloudWatchService,

‎validator/src/main/java/com/amazon/aoc/validators/TraceValidator.java‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -147,15 +147,15 @@ private Map<String, Object> getTrace() throws Exception {
147147
validationConfig.getHttpMethod().toUpperCase(),
148148
validationConfig.getHttpPath()));
149149
}
150+
151+
if (validationConfig.getHttpPath() != null && validationConfig.getHttpPath().contains("ai-chat")) {
152+
return this.getTraceById(Collections.singletonList(context.getTraceId()));
153+
}
154+
150155
log.info("Trace Filter: {}", traceFilter);
151156
List<TraceSummary> retrieveTraceLists = xrayService.searchTraces(traceFilter);
152157
List<String> traceIdLists = Collections.singletonList(retrieveTraceLists.get(0).getId());
153-
List<Trace> retrievedTraceList = xrayService.listTraceByIds(traceIdLists);
154-
155-
if (retrievedTraceList == null || retrievedTraceList.isEmpty()) {
156-
throw new BaseException(ExceptionCode.EMPTY_LIST);
157-
}
158-
return this.flattenDocument(retrievedTraceList.get(0).getSegments());
158+
return getTraceById(traceIdLists);
159159
}
160160

161161
private Map<String, Object> flattenDocument(List<Segment> segmentList) {
@@ -190,6 +190,14 @@ private Map<String, Object> flattenDocument(List<Segment> segmentList) {
190190
return JsonFlattener.flattenAsMap(segmentsJson.toString());
191191
}
192192

193+
private Map<String, Object> getTraceById(List<String> traceIdLists) throws Exception {
194+
List<Trace> retrievedTraceList = xrayService.listTraceByIds(traceIdLists);
195+
if (retrievedTraceList == null || retrievedTraceList.isEmpty()) {
196+
throw new BaseException(ExceptionCode.EMPTY_LIST);
197+
}
198+
return this.flattenDocument(retrievedTraceList.get(0).getSegments());
199+
}
200+
193201
// This method will get the stored traces
194202
private Map<String, Object> getStoredTrace() throws Exception {
195203
Map<String, Object> flattenedJsonMapForStoredTraces = null;

‎validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-log.mustache‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
[{
2+
"resource": {
3+
"attributes": {
4+
"aws.local.service": "{{serviceName}}",
5+
"aws.service.type": "gen_ai_agent",
6+
"service.name": "{{serviceName}}"
7+
}
8+
},
9+
"scope": {
10+
"name": "openinference.instrumentation.langchain"
11+
},
12+
"severityNumber": "^[0-9]+$",
13+
"severityText": ".*",
14+
"body": {
15+
"output": {
16+
"messages": [
17+
{
18+
"content": "^.+$",
19+
"role": "assistant"
20+
}
21+
]
22+
},
23+
"input": {
24+
"messages": [
25+
{
26+
"content": "^.+$",
27+
"role": "user"
28+
}
29+
]
30+
}
31+
},
32+
"attributes": {
33+
"event.name": "openinference.instrumentation.langchain"
34+
},
35+
"traceId": "{{traceId}}"
36+
}]

‎validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-metric.mustache‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-
2+
metricName: ANY_VALUE
3+
namespace: {{metricNamespace}}
4+
dimensions: []

‎validator/src/main/resources/expected-data-template/python/ec2/adot-genai/genai-trace.mustache‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
[{
2+
"name": "^{{serviceName}}$",
3+
"trace_id": "^{{traceId}}$",
4+
"http": {
5+
"request": {
6+
"url": "^.*/ai-chat$",
7+
"method": "^POST$"
8+
}
9+
},
10+
"aws": {
11+
"service.type": "^gen_ai_agent$"
12+
},
13+
"annotations": {
14+
"aws.local.service": "^{{serviceName}}$",
15+
"aws.local.operation": "^POST /ai-chat$"
16+
},
17+
"metadata": {
18+
"service.name": "^{{serviceName}}$"
19+
}
20+
}]

‎validator/src/main/resources/validations/python/ec2/adot-genai/log-validation.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
-
2+
validationType: "cw-log"
3+
httpPath: "ai-chat"
4+
httpMethod: "post"
5+
callingType: "http"
6+
expectedLogStructureTemplate: "PYTHON_EC2_ADOT_GENAI_LOG"

0 commit comments

Comments
 (0)