Skip to content

Commit 6e7afc9

Browse files
wyTrivailwyTrivail
authored
Merge pull request #34 from wyTrivail/terraform-fix
fix skip logic in validator
2 parents da5b1c1 + 425d182 commit 6e7afc9

File tree

21 files changed

+370
-33
lines changed

21 files changed

+370
-33
lines changed

‎terraform/ec2/amis.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,12 @@ variable "ami_family" {
3636
start_command = "sudo /opt/aws/aws-otel-collector/bin/aws-otel-collector-ctl -c /tmp/ot-default.yml -a start"
3737
connection_type = "ssh"
3838
user_data = ""
39+
soaking_cwagent_config = "../template/cwagent-config/soaking-linux.json.tpl"
40+
soaking_cwagent_config_destination = "/tmp/cwagent-config.json"
41+
cwagent_download_command = "sudo rpm -Uvh https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm"
42+
cwagent_start_command = "sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -c file:/tmp/cwagent-config.json -s"
43+
44+
soaking_cpu_metric_name = "procstat_cpu_usage"
3945
}
4046
windows = {
4147
login_user = "Administrator"

‎terraform/ec2/main.tf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,10 @@ data "template_file" "docker_compose" {
127127
otel_endpoint = "${aws_instance.aoc.private_ip}:55680"
128128
}
129129
}
130+
130131
resource "null_resource" "sample-app-validator" {
132+
# skip this validation if it's a soaking test
133+
count = var.soaking ? 0 : 1
131134
provisioner "file" {
132135
content = data.template_file.docker_compose.rendered
133136
destination = "/tmp/docker-compose.yml"

‎terraform/ec2/soaking.tf‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# ------------------------------------------------------------------------
2+
# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License").
5+
# You may not use this file except in compliance with the License.
6+
# A copy of the License is located at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# or in the "license" file accompanying this file. This file is distributed
11+
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
# express or implied. See the License for the specific language governing
13+
# permissions and limitations under the License.
14+
# -------------------------------------------------------------------------
15+
16+
## install cwagent on the instance to collect metric from otel-collector
17+
data "template_file" "cwagent_config" {
18+
count = var.soaking ? 1 : 0
19+
template = file(local.ami_family["soaking_cwagent_config"])
20+
21+
vars = {
22+
soaking_metric_namespace = var.soaking_metric_namespace
23+
}
24+
}
25+
26+
resource "null_resource" "install_cwagent" {
27+
count = var.soaking ? 1 : 0
28+
29+
// copy cwagent config to the instance
30+
provisioner "file" {
31+
content = data.template_file.cwagent_config[0].rendered
32+
destination = local.ami_family["soaking_cwagent_config_destination"]
33+
34+
connection {
35+
type = local.connection_type
36+
user = local.login_user
37+
private_key = local.connection_type == "ssh" ? data.aws_s3_bucket_object.ssh_private_key.body : null
38+
password = local.connection_type == "winrm" ? rsadecrypt(aws_instance.aoc.password_data, data.aws_s3_bucket_object.ssh_private_key.body) : null
39+
host = aws_instance.aoc.public_ip
40+
}
41+
}
42+
43+
provisioner "remote-exec" {
44+
inline = [
45+
local.ami_family["cwagent_download_command"],
46+
local.ami_family["cwagent_start_command"]
47+
]
48+
49+
connection {
50+
type = local.connection_type
51+
user = local.login_user
52+
private_key = local.connection_type == "ssh" ? data.aws_s3_bucket_object.ssh_private_key.body : null
53+
password = local.connection_type == "winrm" ? rsadecrypt(aws_instance.aoc.password_data, data.aws_s3_bucket_object.ssh_private_key.body) : null
54+
host = aws_instance.aoc.public_ip
55+
}
56+
}
57+
}
58+
59+
## create cloudwatch alarm base on the metrics emitted by cwagent
60+
# wait 2 minute for the metrics to be available on cloudwatch
61+
resource "time_sleep" "wait_2_minutes" {
62+
depends_on = [null_resource.install_cwagent[0]]
63+
64+
create_duration = "120s"
65+
}
66+
# cpu alarm
67+
resource "aws_cloudwatch_metric_alarm" "cpu_alarm" {
68+
depends_on = [time_sleep.wait_2_minutes]
69+
alarm_name = "otel-soaking-cpu-alarm-${module.common.testing_id}"
70+
comparison_operator = "GreaterThanOrEqualToThreshold"
71+
evaluation_periods = 2
72+
threshold = "50"
73+
74+
metric_query {
75+
id = "cpu"
76+
return_data = true
77+
78+
metric {
79+
metric_name = local.ami_family["soaking_cpu_metric_name"]
80+
namespace = var.soaking_metric_namespace
81+
period = 60
82+
stat = "Average"
83+
unit = "Percent"
84+
85+
# use this dimension to identify each test
86+
dimensions = {
87+
InstanceId = aws_instance.aoc.id
88+
}
89+
}
90+
}
91+
}
92+
93+
# soaking alarm pulling
94+
resource "null_resource" "bake_alarms" {
95+
depends_on = [aws_cloudwatch_metric_alarm.cpu_alarm]
96+
count = var.soaking ? 1 : 0
97+
provisioner "local-exec" {
98+
command = "${module.common.validator_path} --args='-c ${var.validation_config} -t ${module.common.testing_id} --region ${var.region} --alarm-names ${aws_cloudwatch_metric_alarm.cpu_alarm.alarm_name}'"
99+
working_dir = "../../"
100+
}
101+
}

‎terraform/ec2/validation.tf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
2+
3+

‎terraform/ec2/variables.tf‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ variable "otconfig_path" {
2121
}
2222

2323
variable "docker_compose_path" {
24-
default = "../template/ec2-docker-compose-config/default_ec2_docker_compose.yml"
24+
default = "../template/ec2-docker-compose-config/default_ec2_docker_compose.yml.tpl"
2525
}
2626

2727
variable "package_s3_bucket" {
@@ -65,3 +65,13 @@ variable "sshkey_s3_private_key" {
6565
variable "sample_app_callable" {
6666
default = true
6767
}
68+
69+
# create soaking alarm if this flag is on
70+
variable "soaking" {
71+
default = false
72+
}
73+
74+
variable "soaking_metric_namespace" {
75+
default = "AWSOtelCollector/SoakTest"
76+
}
77+

‎terraform/template/cwagent-config/soaking-linux.json.tpl‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{
2+
"agent": {
3+
"metrics_collection_interval": 10
4+
},
5+
"metrics": {
6+
"append_dimensions": {
7+
"InstanceId": "$${aws:InstanceId}"
8+
},
9+
"metrics_collected": {
10+
"cpu": {
11+
"measurement": [
12+
"cpu_usage_idle",
13+
"cpu_usage_iowait",
14+
"cpu_usage_user",
15+
"cpu_usage_system"
16+
],
17+
"totalcpu": false
18+
},
19+
"disk": {
20+
"measurement": [
21+
"used_percent",
22+
"inodes_free"
23+
]
24+
},
25+
"diskio": {
26+
"measurement": [
27+
"io_time"
28+
]
29+
},
30+
"mem": {
31+
"measurement": [
32+
"mem_used_percent"
33+
]
34+
},
35+
"statsd": {
36+
},
37+
"swap": {
38+
"measurement": [
39+
"swap_used_percent"
40+
]
41+
},
42+
"procstat": [
43+
{
44+
"measurement": [
45+
"cpu_usage",
46+
"memory_rss"
47+
],
48+
"exe": "aws-otel-collector"
49+
}
50+
]
51+
},
52+
"namespace": "${soaking_metric_namespace}"
53+
}
54+
}

‎terraform/template/ec2-docker-compose-config/default_ec2_docker_compose.yml‎ renamed to ‎terraform/template/ec2-docker-compose-config/default_ec2_docker_compose.yml.tpl‎

File renamed without changes.

‎terraform/testing-suites/soaking-ec2.tfvars‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# enable soaking
2+
soaking = true
3+
4+
# assuming the sample app will generate high volume data by default
5+
sample_app_callable = false
6+
7+
# ask validator to pull the alarms
8+
validation_config="alarm-pulling-validation.yml"
9+
10+
# use amazonlinux2 by default to soak
11+
testing_ami = "amazonlinux2"
12+
13+
aoc_version = "v0.1.13-311011856"

‎validator/src/main/java/com/amazon/aoc/App.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ public class App implements Callable<Integer> {
5858
description = "eg, --ecs-context ecsCluster=xxx --ecs-context ecsTaskArn=xxxx")
5959
private Map<String, String> ecsContexts;
6060

61+
@CommandLine.Option(
62+
names = {"--alarm-names"},
63+
description = "the cloudwatch alarm names")
64+
private List<String> alarmNameList;
65+
6166
public static void main(String[] args) throws Exception {
6267
int exitCode = new CommandLine(new App()).execute(args);
6368
System.exit(exitCode);
@@ -66,9 +71,11 @@ public static void main(String[] args) throws Exception {
6671
@Override
6772
public Integer call() throws Exception {
6873
// build context
69-
Context context = new Context(this.testingId, this.metricNamespace, this.region);
74+
Context context = new Context(this.testingId, this.region);
75+
context.setMetricNamespace(this.metricNamespace);
7076
context.setEndpoint(this.endpoint);
7177
context.setEcsContext(buildECSContext(ecsContexts));
78+
context.setAlarmNameList(alarmNameList);
7279

7380
log.info(context);
7481

‎validator/src/main/java/com/amazon/aoc/exception/ExceptionCode.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ public enum ExceptionCode {
3333
// build validator
3434
VALIDATION_TYPE_NOT_EXISTED(60001, "validation type not existed"),
3535
CALLER_TYPE_NOT_EXISTED(60002, "caller type not existed"),
36+
37+
// alarm validation
38+
ALARM_BAKING(70001, "alarms still need to be baked"),
3639
;
3740
private int code;
3841
private String message;

0 commit comments

Comments
 (0)