Skip to content

Commit d5fc6e8

Browse files
wyTrivailwyTrivail
authored
Merge pull request #37 from wyTrivail/terraform1
support multi span check in validator
2 parents 936f361 + c683b90 commit d5fc6e8

File tree

14 files changed

+186
-107
lines changed

14 files changed

+186
-107
lines changed

‎terraform/ec2/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ resource "null_resource" "sample-app-validator" {
158158
}
159159

160160
provisioner "local-exec" {
161-
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${aws_instance.emitter.public_ip}'"
161+
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${aws_instance.emitter.public_ip}:80'"
162162
working_dir = "../../"
163163
}
164164
}

‎terraform/ecs/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ resource "aws_ecs_service" "aoc" {
170170

171171
provisioner "local-exec" {
172172
working_dir = "../../"
173-
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${aws_lb.aoc_lb[0].dns_name}'"
173+
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${aws_lb.aoc_lb[0].dns_name}:80'"
174174
}
175175
}
176176

‎terraform/eks/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,7 @@ resource "kubernetes_pod" "aoc_pod" {
297297
resource "null_resource" "callable_sample_app_validator" {
298298
count = var.sample_app_callable ? 1 : 0
299299
provisioner "local-exec" {
300-
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${kubernetes_service.sample_app_service[0].load_balancer_ingress.0.hostname}'"
300+
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --metric-namespace ${module.common.otel_service_namespace}/${module.common.otel_service_name} --endpoint http://${kubernetes_service.sample_app_service[0].load_balancer_ingress.0.hostname}:80'"
301301
working_dir = "../../"
302302
}
303303
}

‎terraform/eks/variables.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ variable "aoc_image_repo" {
3434
}
3535

3636
variable "aoc_version" {
37-
default = "v0.1.11"
37+
default = "v0.1.0-320983060"
3838
}
3939

4040
variable "validation_config" {

‎validator/src/main/java/com/amazon/aoc/callers/HttpCaller.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,15 @@
3232
@Log4j2
3333
public class HttpCaller implements ICaller {
3434
private String url;
35+
private String path;
3536

37+
/**
38+
* construct httpCaller.
39+
* @param endpoint the endpoint to call, for example "http://127.0.0.1:8080"
40+
* @param path the path to call, for example "/test"
41+
*/
3642
public HttpCaller(String endpoint, String path) {
43+
this.path = path;
3744
this.url = endpoint + path;
3845
log.info("validator is trying to hit this {} endpoint", this.url);
3946
}
@@ -73,4 +80,9 @@ public SampleAppResponse callSampleApp() throws Exception {
7380

7481
return sampleAppResponseAtomicReference.get();
7582
}
83+
84+
@Override
85+
public String getCallingPath() {
86+
return path;
87+
}
7688
}

‎validator/src/main/java/com/amazon/aoc/callers/ICaller.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,6 @@
1919

2020
public interface ICaller {
2121
SampleAppResponse callSampleApp() throws Exception;
22+
23+
String getCallingPath();
2224
}

‎validator/src/main/java/com/amazon/aoc/helpers/RetryHelper.java‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,24 +35,22 @@ public class RetryHelper {
3535
public static void retry(
3636
int retryCount, int sleepInMilliSeconds, boolean throwExceptionInTheEnd, Retryable retryable)
3737
throws Exception {
38+
Exception exceptionInTheEnd = null;
3839
while (retryCount-- > 0) {
3940
try {
4041
log.info("retry attempt left : {} ", retryCount);
4142
retryable.execute();
4243
return;
4344
} catch (Exception ex) {
45+
exceptionInTheEnd = ex;
4446
log.info("retrying after {} seconds", TimeUnit.MILLISECONDS.toSeconds(sleepInMilliSeconds));
45-
46-
if (retryCount == 0) {
47-
log.error("retries exhausted, possible exception: ", ex);
48-
break;
49-
}
5047
TimeUnit.MILLISECONDS.sleep(sleepInMilliSeconds);
5148
}
5249
}
5350

5451
if (throwExceptionInTheEnd) {
55-
throw new BaseException(ExceptionCode.FAILED_AFTER_RETRY);
52+
log.error("retries exhausted, possible");
53+
throw exceptionInTheEnd;
5654
}
5755
}
5856

‎validator/src/main/java/com/amazon/aoc/validators/MetricValidator.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,15 +58,17 @@ public void validate() throws Exception {
5858
// get expected metrics and remove the to be skipped dimensions
5959
final List<Metric> expectedMetricList = this.getExpectedMetricList(context);
6060
Set<String> skippedDimensionNameList = new HashSet<>();
61-
for(Metric metric: expectedMetricList){
62-
for(Dimension dimension: metric.getDimensions()){
63-
if(dimension.getValue().equals("SKIP")){
61+
for (Metric metric : expectedMetricList) {
62+
for (Dimension dimension : metric.getDimensions()) {
63+
if (dimension.getValue().equals("SKIP")) {
6464
skippedDimensionNameList.add(dimension.getName());
6565
}
6666
}
6767
}
6868
for (Metric metric : expectedMetricList) {
69-
metric.getDimensions().removeIf((dimension) -> skippedDimensionNameList.contains(dimension.getName()));
69+
metric
70+
.getDimensions()
71+
.removeIf((dimension) -> skippedDimensionNameList.contains(dimension.getName()));
7072
}
7173

7274
// get metric from cloudwatch
@@ -80,7 +82,9 @@ public void validate() throws Exception {
8082
// remove the skip dimensions
8183
log.info("dimensions to be skipped in validation: {}", skippedDimensionNameList);
8284
for (Metric metric : metricList) {
83-
metric.getDimensions().removeIf((dimension) -> skippedDimensionNameList.contains(dimension.getName()));
85+
metric
86+
.getDimensions()
87+
.removeIf((dimension) -> skippedDimensionNameList.contains(dimension.getName()));
8488
}
8589

8690
log.info("check if all the expected metrics are found");

‎validator/src/main/java/com/amazon/aoc/validators/TraceValidator.java‎

Lines changed: 111 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -25,108 +25,137 @@
2525
import com.amazon.aoc.models.SampleAppResponse;
2626
import com.amazon.aoc.models.ValidationConfig;
2727
import com.amazon.aoc.services.XRayService;
28+
import com.amazonaws.services.xray.model.Segment;
2829
import com.amazonaws.services.xray.model.Trace;
30+
import com.fasterxml.jackson.databind.ObjectMapper;
2931
import com.github.wnameless.json.flattener.JsonFlattener;
3032
import lombok.extern.log4j.Log4j2;
3133

32-
import java.util.*;
34+
import java.util.Collections;
35+
import java.util.List;
36+
import java.util.Map;
3337
import java.util.concurrent.atomic.AtomicReference;
3438

3539
@Log4j2
3640
public class TraceValidator implements IValidator {
37-
private MustacheHelper mustacheHelper = new MustacheHelper();
38-
private XRayService xrayService;
39-
private ICaller caller;
40-
private Context context;
41-
private FileConfig expectedTrace;
42-
43-
@Override
44-
public void init(Context context, ValidationConfig validationConfig, ICaller caller, FileConfig expectedTrace) throws Exception {
45-
this.xrayService = new XRayService(context.getRegion());
46-
this.caller = caller;
47-
this.context = context;
48-
this.expectedTrace = expectedTrace;
41+
private MustacheHelper mustacheHelper = new MustacheHelper();
42+
private XRayService xrayService;
43+
private ICaller caller;
44+
private Context context;
45+
private FileConfig expectedTrace;
46+
47+
@Override
48+
public void init(
49+
Context context, ValidationConfig validationConfig, ICaller caller, FileConfig expectedTrace)
50+
throws Exception {
51+
this.xrayService = new XRayService(context.getRegion());
52+
this.caller = caller;
53+
this.context = context;
54+
this.expectedTrace = expectedTrace;
55+
}
56+
57+
@Override
58+
public void validate() throws Exception {
59+
// get stored trace
60+
Map<String, Object> storedTrace = this.getStoredTrace();
61+
62+
// create trace id list to retrieve trace from x-ray service
63+
String traceId = (String) storedTrace.get("[0].trace_id");
64+
List<String> traceIdList = Collections.singletonList(traceId);
65+
66+
// get retrieved trace from x-ray service
67+
Map<String, Object> retrievedTrace = this.getRetrievedTrace(traceIdList);
68+
69+
// data model validation of other fields of segment document
70+
for (Map.Entry<String, Object> entry : storedTrace.entrySet()) {
71+
if (!entry.getValue().equals(retrievedTrace.get(entry.getKey()))) {
72+
log.error("data model validation failed");
73+
log.info("mis matched data model field list");
74+
log.info("value of stored trace map: {}", entry.getValue());
75+
log.info("value of retrieved map: {}", retrievedTrace.get(entry.getKey()));
76+
log.info("==========================================");
77+
throw new BaseException(ExceptionCode.DATA_MODEL_NOT_MATCHED);
78+
}
4979
}
5080

51-
@Override
52-
public void validate() throws Exception {
53-
// get stored trace
54-
Map<String, Object> storedTrace = this.getStoredTrace();
81+
log.info("validation is passed for path {}", caller.getCallingPath());
82+
}
5583

56-
// create trace id list to retrieve trace from x-ray service
57-
String traceId = (String) storedTrace.get("trace_id");
58-
List<String> traceIdList = Collections.singletonList(traceId);
84+
// this method will hit get trace from x-ray service and get retrieved trace
85+
private Map<String, Object> getRetrievedTrace(List<String> traceIdList) throws Exception {
86+
AtomicReference<List<Trace>> retrieveTraceListAtomicReference = new AtomicReference<>();
87+
RetryHelper.retry(
88+
3,
89+
() -> {
90+
List<Trace> retrieveTraceList = null;
91+
retrieveTraceList = xrayService.listTraceByIds(traceIdList);
92+
retrieveTraceListAtomicReference.set(retrieveTraceList);
5993

60-
// get retrieved trace from x-ray service
61-
Map<String, Object> retrievedTrace = this.getRetrievedTrace(traceIdList);
62-
63-
// validation of trace id
64-
if (!storedTrace.get("trace_id").equals(retrievedTrace.get("trace_id"))) {
65-
log.error("trace id validation failed");
66-
throw new BaseException(ExceptionCode.TRACE_ID_NOT_MATCHED);
67-
}
68-
69-
// data model validation of other fields of segment document
70-
for (Map.Entry<String, Object> entry : storedTrace.entrySet()) {
71-
if (!entry.getValue().equals(retrievedTrace.get(entry.getKey()))) {
72-
log.error("data model validation failed");
73-
log.info("mis matched data model field list");
74-
log.info("value of stored trace map: {}", entry.getValue());
75-
log.info("value of retrieved map: {}",retrievedTrace.get(entry.getKey()));
76-
log.info("==========================================");
77-
throw new BaseException(ExceptionCode.DATA_MODEL_NOT_MATCHED);
78-
}
79-
}
80-
}
81-
82-
// this method will hit get trace from x-ray service and get retrieved trace
83-
private Map<String, Object> getRetrievedTrace(List<String> traceIdList) throws Exception {
84-
Map<String, Object> flattenedJsonMapForRetrievedTrace = null;
85-
AtomicReference<List<Trace>> retrieveTraceListAtomicReference = new AtomicReference<>();
86-
int MAX_RETRY_COUNT = 3;
87-
88-
RetryHelper.retry(MAX_RETRY_COUNT, () -> {
89-
List<Trace> retrieveTraceList = null;
90-
retrieveTraceList = xrayService.listTraceByIds(traceIdList);
91-
retrieveTraceListAtomicReference.set(retrieveTraceList);
92-
93-
if (retrieveTraceList == null || retrieveTraceList.isEmpty()) {
94-
throw new BaseException(ExceptionCode.EMPTY_LIST);
95-
}
94+
if (retrieveTraceList == null || retrieveTraceList.isEmpty()) {
95+
throw new BaseException(ExceptionCode.EMPTY_LIST);
96+
}
9697
});
9798

98-
// flattened JSON object to a map
99-
if (retrieveTraceListAtomicReference.get() != null && !retrieveTraceListAtomicReference.get().isEmpty()) {
100-
try {
101-
flattenedJsonMapForRetrievedTrace = JsonFlattener.flattenAsMap(retrieveTraceListAtomicReference.get().get(0).getSegments().get(0).getDocument());
102-
} catch (Exception e) {
103-
log.error("exception while flattening the retrieved trace: " + e.getMessage());
104-
e.printStackTrace();
105-
}
106-
} else {
107-
log.error("retrieved trace list is empty or null");
108-
throw new BaseException(ExceptionCode.EMPTY_LIST);
99+
// flattened JSON object to a map
100+
Map<String, Object> flattenedJsonMapForRetrievedTrace = null;
101+
if (retrieveTraceListAtomicReference.get() != null
102+
&& !retrieveTraceListAtomicReference.get().isEmpty()) {
103+
try {
104+
List<Segment> segmentList = retrieveTraceListAtomicReference.get().get(0).getSegments();
105+
// have to sort the segments by start_time because
106+
// 1. we can not get span id from xraysdk today,
107+
// 2. the segments come out with different order everytime
108+
segmentList.sort(
109+
(segment1, segment2) -> {
110+
try {
111+
Map<String, Object> map1 =
112+
new ObjectMapper().readValue(segment1.getDocument(), Map.class);
113+
Map<String, Object> map2 =
114+
new ObjectMapper().readValue(segment2.getDocument(), Map.class);
115+
return map1.get("start_time")
116+
.toString()
117+
.compareTo(map2.get("start_time").toString());
118+
} catch (Exception ex) {
119+
log.error(ex);
120+
return 0;
121+
}
122+
});
123+
124+
// build the segment's document as a jsonarray and flatten it for easy comparison
125+
StringBuilder segmentsJson = new StringBuilder("[");
126+
for (Segment segment : segmentList) {
127+
segmentsJson.append(segment.getDocument());
128+
segmentsJson.append(",");
109129
}
110-
111-
return flattenedJsonMapForRetrievedTrace;
130+
segmentsJson.replace(segmentsJson.length() - 1, segmentsJson.length(), "]");
131+
flattenedJsonMapForRetrievedTrace = JsonFlattener.flattenAsMap(segmentsJson.toString());
132+
} catch (Exception e) {
133+
log.error("exception while flattening the retrieved trace: " + e.getMessage());
134+
}
135+
} else {
136+
log.error("retrieved trace list is empty or null");
137+
throw new BaseException(ExceptionCode.EMPTY_LIST);
112138
}
113139

114-
// this method will hit a http endpoints of sample web apps and get stored trace
115-
private Map<String, Object> getStoredTrace() throws Exception {
116-
Map<String, Object> flattenedJsonMapForStoredTraces = null;
140+
return flattenedJsonMapForRetrievedTrace;
141+
}
117142

118-
SampleAppResponse sampleAppResponse = this.caller.callSampleApp();
143+
// this method will hit a http endpoints of sample web apps and get stored trace
144+
private Map<String, Object> getStoredTrace() throws Exception {
145+
Map<String, Object> flattenedJsonMapForStoredTraces = null;
119146

120-
String jsonExpectedTrace = mustacheHelper.render(this.expectedTrace, context);
147+
SampleAppResponse sampleAppResponse = this.caller.callSampleApp();
121148

122-
try {
123-
// flattened JSON object to a map
124-
flattenedJsonMapForStoredTraces = JsonFlattener.flattenAsMap(jsonExpectedTrace);
125-
flattenedJsonMapForStoredTraces.put("trace_id", sampleAppResponse.getTraceId());
126-
} catch (Exception e) {
127-
e.printStackTrace();
128-
}
149+
String jsonExpectedTrace = mustacheHelper.render(this.expectedTrace, context);
129150

130-
return flattenedJsonMapForStoredTraces;
151+
try {
152+
// flattened JSON object to a map
153+
flattenedJsonMapForStoredTraces = JsonFlattener.flattenAsMap(jsonExpectedTrace);
154+
flattenedJsonMapForStoredTraces.put("[0].trace_id", sampleAppResponse.getTraceId());
155+
} catch (Exception e) {
156+
e.printStackTrace();
131157
}
158+
159+
return flattenedJsonMapForStoredTraces;
160+
}
132161
}

‎validator/src/main/resources/expected-data-template/defaultExpectedTrace.mustache‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,36 @@
1-
-
2-
id: {{traceId}}
1+
[
2+
{
3+
"http": {
4+
"request": {
5+
"url": "{{endpoint}}/span0",
6+
"method": "GET"
7+
},
8+
"response": {
9+
"status": 200
10+
}
11+
}
12+
},
13+
{
14+
"http": {
15+
"request": {
16+
"url": "http://localhost:4567/span1",
17+
"method": "GET"
18+
},
19+
"response": {
20+
"status": 200
21+
}
22+
}
23+
},
24+
{
25+
"http": {
26+
"request": {
27+
"url": "http://localhost:4567/span2",
28+
"method": "GET"
29+
},
30+
"response": {
31+
"status": 200
32+
}
33+
}
34+
}
35+
]
36+

0 commit comments

Comments
 (0)