Skip to content

Commit c7e8f78

Browse files
wyTrivailwyTrivail
committed
support multi span check in validator
1 parent 936f361 commit c7e8f78

File tree

12 files changed

+173
-110
lines changed

12 files changed

+173
-110
lines changed

‎terraform/ec2/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ resource "null_resource" "sample-app-validator" {
158158
}
159159

160160
provisioner "local-exec" {
161-
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${aws_instance.emitter.public_ip}'"
161+
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${aws_instance.emitter.public_ip}:80'"
162162
working_dir = "../../"
163163
}
164164
}

‎terraform/ecs/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ resource "aws_ecs_service" "aoc" {
170170

171171
provisioner "local-exec" {
172172
working_dir = "../../"
173-
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${aws_lb.aoc_lb[0].dns_name}'"
173+
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${aws_lb.aoc_lb[0].dns_name}:80'"
174174
}
175175
}
176176

‎terraform/eks/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,7 @@ resource "kubernetes_pod" "aoc_pod" {
297297
resource "null_resource" "callable_sample_app_validator" {
298298
count = var.sample_app_callable ? 1 : 0
299299
provisioner "local-exec" {
300-
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${kubernetes_service.sample_app_service[0].load_balancer_ingress.0.hostname}'"
300+
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${kubernetes_service.sample_app_service[0].load_balancer_ingress.0.hostname}:80'"
301301
working_dir = "../../"
302302
}
303303
}

‎terraform/eks/variables.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ variable "aoc_image_repo" {
3434
}
3535

3636
variable "aoc_version" {
37-
default = "v0.1.11"
37+
default = "v0.1.0-320983060"
3838
}
3939

4040
variable "validation_config" {

‎validator/src/main/java/com/amazon/aoc/helpers/RetryHelper.java‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,24 +35,22 @@ public class RetryHelper {
3535
public static void retry(
3636
int retryCount, int sleepInMilliSeconds, boolean throwExceptionInTheEnd, Retryable retryable)
3737
throws Exception {
38+
Exception exceptionInTheEnd = null;
3839
while (retryCount-- > 0) {
3940
try {
4041
log.info("retry attempt left : {} ", retryCount);
4142
retryable.execute();
4243
return;
4344
} catch (Exception ex) {
45+
exceptionInTheEnd = ex;
4446
log.info("retrying after {} seconds", TimeUnit.MILLISECONDS.toSeconds(sleepInMilliSeconds));
45-
46-
if (retryCount == 0) {
47-
log.error("retries exhausted, possible exception: ", ex);
48-
break;
49-
}
5047
TimeUnit.MILLISECONDS.sleep(sleepInMilliSeconds);
5148
}
5249
}
5350

5451
if (throwExceptionInTheEnd) {
55-
throw new BaseException(ExceptionCode.FAILED_AFTER_RETRY);
52+
log.error("retries exhausted, possible");
53+
throw exceptionInTheEnd;
5654
}
5755
}
5856

‎validator/src/main/java/com/amazon/aoc/validators/MetricValidator.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,15 +58,17 @@ public void validate() throws Exception {
5858
// get expected metrics and remove the to be skipped dimensions
5959
final List<Metric> expectedMetricList = this.getExpectedMetricList(context);
6060
Set<String> skippedDimensionNameList = new HashSet<>();
61-
for(Metric metric: expectedMetricList){
62-
for(Dimension dimension: metric.getDimensions()){
63-
if(dimension.getValue().equals("SKIP")){
61+
for (Metric metric : expectedMetricList) {
62+
for (Dimension dimension : metric.getDimensions()) {
63+
if (dimension.getValue().equals("SKIP")) {
6464
skippedDimensionNameList.add(dimension.getName());
6565
}
6666
}
6767
}
6868
for (Metric metric : expectedMetricList) {
69-
metric.getDimensions().removeIf((dimension) -> skippedDimensionNameList.contains(dimension.getName()));
69+
metric
70+
.getDimensions()
71+
.removeIf((dimension) -> skippedDimensionNameList.contains(dimension.getName()));
7072
}
7173

7274
// get metric from cloudwatch
@@ -80,7 +82,9 @@ public void validate() throws Exception {
8082
// remove the skip dimensions
8183
log.info("dimensions to be skipped in validation: {}", skippedDimensionNameList);
8284
for (Metric metric : metricList) {
83-
metric.getDimensions().removeIf((dimension) -> skippedDimensionNameList.contains(dimension.getName()));
85+
metric
86+
.getDimensions()
87+
.removeIf((dimension) -> skippedDimensionNameList.contains(dimension.getName()));
8488
}
8589

8690
log.info("check if all the expected metrics are found");

‎validator/src/main/java/com/amazon/aoc/validators/TraceValidator.java‎

Lines changed: 112 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -25,108 +25,135 @@
2525
import com.amazon.aoc.models.SampleAppResponse;
2626
import com.amazon.aoc.models.ValidationConfig;
2727
import com.amazon.aoc.services.XRayService;
28+
import com.amazonaws.services.xray.model.Segment;
2829
import com.amazonaws.services.xray.model.Trace;
30+
import com.fasterxml.jackson.databind.ObjectMapper;
2931
import com.github.wnameless.json.flattener.JsonFlattener;
3032
import lombok.extern.log4j.Log4j2;
3133

32-
import java.util.*;
34+
import java.util.Collections;
35+
import java.util.List;
36+
import java.util.Map;
3337
import java.util.concurrent.atomic.AtomicReference;
3438

3539
@Log4j2
3640
public class TraceValidator implements IValidator {
37-
private MustacheHelper mustacheHelper = new MustacheHelper();
38-
private XRayService xrayService;
39-
private ICaller caller;
40-
private Context context;
41-
private FileConfig expectedTrace;
42-
43-
@Override
44-
public void init(Context context, ValidationConfig validationConfig, ICaller caller, FileConfig expectedTrace) throws Exception {
45-
this.xrayService = new XRayService(context.getRegion());
46-
this.caller = caller;
47-
this.context = context;
48-
this.expectedTrace = expectedTrace;
41+
private MustacheHelper mustacheHelper = new MustacheHelper();
42+
private XRayService xrayService;
43+
private ICaller caller;
44+
private Context context;
45+
private FileConfig expectedTrace;
46+
47+
@Override
48+
public void init(
49+
Context context, ValidationConfig validationConfig, ICaller caller, FileConfig expectedTrace)
50+
throws Exception {
51+
this.xrayService = new XRayService(context.getRegion());
52+
this.caller = caller;
53+
this.context = context;
54+
this.expectedTrace = expectedTrace;
55+
}
56+
57+
@Override
58+
public void validate() throws Exception {
59+
// get stored trace
60+
Map<String, Object> storedTrace = this.getStoredTrace();
61+
62+
// create trace id list to retrieve trace from x-ray service
63+
String traceId = (String) storedTrace.get("[0].trace_id");
64+
List<String> traceIdList = Collections.singletonList(traceId);
65+
66+
// get retrieved trace from x-ray service
67+
Map<String, Object> retrievedTrace = this.getRetrievedTrace(traceIdList);
68+
69+
// data model validation of other fields of segment document
70+
for (Map.Entry<String, Object> entry : storedTrace.entrySet()) {
71+
if (!entry.getValue().equals(retrievedTrace.get(entry.getKey()))) {
72+
log.error("data model validation failed");
73+
log.info("mis matched data model field list");
74+
log.info("value of stored trace map: {}", entry.getValue());
75+
log.info("value of retrieved map: {}", retrievedTrace.get(entry.getKey()));
76+
log.info("==========================================");
77+
throw new BaseException(ExceptionCode.DATA_MODEL_NOT_MATCHED);
78+
}
4979
}
50-
51-
@Override
52-
public void validate() throws Exception {
53-
// get stored trace
54-
Map<String, Object> storedTrace = this.getStoredTrace();
55-
56-
// create trace id list to retrieve trace from x-ray service
57-
String traceId = (String) storedTrace.get("trace_id");
58-
List<String> traceIdList = Collections.singletonList(traceId);
59-
60-
// get retrieved trace from x-ray service
61-
Map<String, Object> retrievedTrace = this.getRetrievedTrace(traceIdList);
62-
63-
// validation of trace id
64-
if (!storedTrace.get("trace_id").equals(retrievedTrace.get("trace_id"))) {
65-
log.error("trace id validation failed");
66-
throw new BaseException(ExceptionCode.TRACE_ID_NOT_MATCHED);
67-
}
68-
69-
// data model validation of other fields of segment document
70-
for (Map.Entry<String, Object> entry : storedTrace.entrySet()) {
71-
if (!entry.getValue().equals(retrievedTrace.get(entry.getKey()))) {
72-
log.error("data model validation failed");
73-
log.info("mis matched data model field list");
74-
log.info("value of stored trace map: {}", entry.getValue());
75-
log.info("value of retrieved map: {}",retrievedTrace.get(entry.getKey()));
76-
log.info("==========================================");
77-
throw new BaseException(ExceptionCode.DATA_MODEL_NOT_MATCHED);
78-
}
79-
}
80-
}
81-
82-
// this method will hit get trace from x-ray service and get retrieved trace
83-
private Map<String, Object> getRetrievedTrace(List<String> traceIdList) throws Exception {
84-
Map<String, Object> flattenedJsonMapForRetrievedTrace = null;
85-
AtomicReference<List<Trace>> retrieveTraceListAtomicReference = new AtomicReference<>();
86-
int MAX_RETRY_COUNT = 3;
87-
88-
RetryHelper.retry(MAX_RETRY_COUNT, () -> {
89-
List<Trace> retrieveTraceList = null;
90-
retrieveTraceList = xrayService.listTraceByIds(traceIdList);
91-
retrieveTraceListAtomicReference.set(retrieveTraceList);
92-
93-
if (retrieveTraceList == null || retrieveTraceList.isEmpty()) {
94-
throw new BaseException(ExceptionCode.EMPTY_LIST);
95-
}
80+
}
81+
82+
// this method will hit get trace from x-ray service and get retrieved trace
83+
private Map<String, Object> getRetrievedTrace(List<String> traceIdList) throws Exception {
84+
AtomicReference<List<Trace>> retrieveTraceListAtomicReference = new AtomicReference<>();
85+
RetryHelper.retry(
86+
3,
87+
() -> {
88+
List<Trace> retrieveTraceList = null;
89+
retrieveTraceList = xrayService.listTraceByIds(traceIdList);
90+
retrieveTraceListAtomicReference.set(retrieveTraceList);
91+
92+
if (retrieveTraceList == null || retrieveTraceList.isEmpty()) {
93+
throw new BaseException(ExceptionCode.EMPTY_LIST);
94+
}
9695
});
9796

98-
// flattened JSON object to a map
99-
if (retrieveTraceListAtomicReference.get() != null && !retrieveTraceListAtomicReference.get().isEmpty()) {
100-
try {
101-
flattenedJsonMapForRetrievedTrace = JsonFlattener.flattenAsMap(retrieveTraceListAtomicReference.get().get(0).getSegments().get(0).getDocument());
102-
} catch (Exception e) {
103-
log.error("exception while flattening the retrieved trace: " + e.getMessage());
104-
e.printStackTrace();
105-
}
106-
} else {
107-
log.error("retrieved trace list is empty or null");
108-
throw new BaseException(ExceptionCode.EMPTY_LIST);
97+
// flattened JSON object to a map
98+
Map<String, Object> flattenedJsonMapForRetrievedTrace = null;
99+
if (retrieveTraceListAtomicReference.get() != null
100+
&& !retrieveTraceListAtomicReference.get().isEmpty()) {
101+
try {
102+
List<Segment> segmentList = retrieveTraceListAtomicReference.get().get(0).getSegments();
103+
// have to sort the segments by start_time because
104+
// 1. we can not get span id from xraysdk today,
105+
// 2. the segments come out with different order everytime
106+
segmentList.sort(
107+
(segment1, segment2) -> {
108+
try {
109+
Map<String, Object> map1 =
110+
new ObjectMapper().readValue(segment1.getDocument(), Map.class);
111+
Map<String, Object> map2 =
112+
new ObjectMapper().readValue(segment2.getDocument(), Map.class);
113+
return map1.get("start_time")
114+
.toString()
115+
.compareTo(map2.get("start_time").toString());
116+
} catch (Exception ex) {
117+
log.error(ex);
118+
return 0;
119+
}
120+
});
121+
122+
// build the segment's document as a jsonarray and flatten it for easy comparison
123+
StringBuilder segmentsJson = new StringBuilder("[");
124+
for (Segment segment : segmentList) {
125+
segmentsJson.append(segment.getDocument());
126+
segmentsJson.append(",");
109127
}
110-
111-
return flattenedJsonMapForRetrievedTrace;
128+
segmentsJson.replace(segmentsJson.length() - 1, segmentsJson.length(), "]");
129+
flattenedJsonMapForRetrievedTrace = JsonFlattener.flattenAsMap(segmentsJson.toString());
130+
} catch (Exception e) {
131+
log.error("exception while flattening the retrieved trace: " + e.getMessage());
132+
}
133+
} else {
134+
log.error("retrieved trace list is empty or null");
135+
throw new BaseException(ExceptionCode.EMPTY_LIST);
112136
}
113137

114-
// this method will hit a http endpoints of sample web apps and get stored trace
115-
private Map<String, Object> getStoredTrace() throws Exception {
116-
Map<String, Object> flattenedJsonMapForStoredTraces = null;
138+
return flattenedJsonMapForRetrievedTrace;
139+
}
117140

118-
SampleAppResponse sampleAppResponse = this.caller.callSampleApp();
141+
// this method will hit a http endpoints of sample web apps and get stored trace
142+
private Map<String, Object> getStoredTrace() throws Exception {
143+
Map<String, Object> flattenedJsonMapForStoredTraces = null;
119144

120-
String jsonExpectedTrace = mustacheHelper.render(this.expectedTrace, context);
145+
SampleAppResponse sampleAppResponse = this.caller.callSampleApp();
121146

122-
try {
123-
// flattened JSON object to a map
124-
flattenedJsonMapForStoredTraces = JsonFlattener.flattenAsMap(jsonExpectedTrace);
125-
flattenedJsonMapForStoredTraces.put("trace_id", sampleAppResponse.getTraceId());
126-
} catch (Exception e) {
127-
e.printStackTrace();
128-
}
147+
String jsonExpectedTrace = mustacheHelper.render(this.expectedTrace, context);
129148

130-
return flattenedJsonMapForStoredTraces;
149+
try {
150+
// flattened JSON object to a map
151+
flattenedJsonMapForStoredTraces = JsonFlattener.flattenAsMap(jsonExpectedTrace);
152+
flattenedJsonMapForStoredTraces.put("[0].trace_id", sampleAppResponse.getTraceId());
153+
} catch (Exception e) {
154+
e.printStackTrace();
131155
}
156+
157+
return flattenedJsonMapForStoredTraces;
158+
}
132159
}

‎validator/src/main/resources/expected-data-template/defaultExpectedTrace.mustache‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,36 @@
1-
-
2-
id: {{traceId}}
1+
[
2+
{
3+
"http": {
4+
"request": {
5+
"url": "{{endpoint}}/span0",
6+
"method": "GET"
7+
},
8+
"response": {
9+
"status": 200
10+
}
11+
}
12+
},
13+
{
14+
"http": {
15+
"request": {
16+
"url": "http://localhost:4567/span1",
17+
"method": "GET"
18+
},
19+
"response": {
20+
"status": 200
21+
}
22+
}
23+
},
24+
{
25+
"http": {
26+
"request": {
27+
"url": "http://localhost:4567/span2",
28+
"method": "GET"
29+
},
30+
"response": {
31+
"status": 200
32+
}
33+
}
34+
}
35+
]
36+

‎validator/src/main/resources/expected-data-template/otelSDKexpectedAWSSDKTrace.mustache‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{
1+
[{
22
"name": "/aws-sdk-call",
33
"fault": false,
44
"error": false,
@@ -55,4 +55,4 @@
5555
]
5656
}
5757
]
58-
}
58+
}]

‎validator/src/main/resources/expected-data-template/otelSDKexpectedHTTPTrace.mustache‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{
1+
[{
22
"name": "/outgoing-http-call",
33
"fault": false,
44
"error": false,
@@ -37,4 +37,4 @@
3737
]
3838
}
3939
]
40-
}
40+
}]

0 commit comments

Comments
 (0)