Skip to content

Commit 6fe47b7

Browse files
authored
Implement JMX E2E Tests in Terraform Framework (#443)
1 parent f92b85b commit 6fe47b7

File tree

17 files changed

+944
-19
lines changed

17 files changed

+944
-19
lines changed

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,5 @@ terraform.*
1010
**/.terraform/*
1111
/test/**/final_*.yml
1212
coverage.txt
13-
generator/resources/*complete*.json
13+
generator/resources/*complete*.json
14+
terraform/eks/e2e/helm-charts

environment/eksdeploymenttype/eks_deployment_type.go

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,22 @@ import "strings"
88
type EKSDeploymentType string
99

1010
const (
11-
DAEMON EKSDeploymentType = "DAEMON"
12-
REPLICA EKSDeploymentType = "REPLICA"
13-
SIDECAR EKSDeploymentType = "SIDECAR"
11+
DAEMON EKSDeploymentType = "DAEMON"
12+
REPLICA EKSDeploymentType = "REPLICA"
13+
SIDECAR EKSDeploymentType = "SIDECAR"
14+
STATEFUL EKSDeploymentType = "STATEFUL"
1415
)
1516

1617
var (
17-
ecsDeploymentTypes = map[string]EKSDeploymentType{
18-
"DAEMON": DAEMON,
19-
"REPLICA": REPLICA,
20-
"SIDECAR": SIDECAR,
18+
eksDeploymentTypes = map[string]EKSDeploymentType{
19+
"DAEMON": DAEMON,
20+
"REPLICA": REPLICA,
21+
"SIDECAR": SIDECAR,
22+
"STATEFUL": STATEFUL,
2123
}
2224
)
2325

2426
func FromString(str string) (EKSDeploymentType, bool) {
25-
c, ok := ecsDeploymentTypes[strings.ToUpper(str)]
27+
c, ok := eksDeploymentTypes[strings.ToUpper(str)]
2628
return c, ok
2729
}

environment/metadata.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ type MetaData struct {
4747
AmpWorkspaceId string
4848
Region string
4949
K8sVersion string
50+
Destroy bool
5051
HelmChartsBranch string
5152
CloudwatchAgentRepository string
5253
CloudwatchAgentTag string
@@ -87,6 +88,7 @@ type MetaDataStrings struct {
8788
AmpWorkspaceId string
8889
Region string
8990
K8sVersion string
91+
Destroy bool
9092
HelmChartsBranch string
9193
CloudwatchAgentRepository string
9294
CloudwatchAgentTag string
@@ -136,6 +138,7 @@ func registerEKSData(d *MetaDataStrings) {
136138
func registerEKSE2ETestData(dataString *MetaDataStrings) {
137139
flag.StringVar(&(dataString.Region), "region", "", "AWS region")
138140
flag.StringVar(&(dataString.K8sVersion), "k8s_version", "", "Kubernetes version")
141+
flag.BoolVar(&(dataString.Destroy), "destroy", false, "Whether to run in destroy mode (true/false)")
139142
flag.StringVar(&(dataString.HelmChartsBranch), "helm_charts_branch", "", "Helm charts branch")
140143
flag.StringVar(&(dataString.CloudwatchAgentRepository), "cloudwatch_agent_repository", "", "CloudWatch Agent repository")
141144
flag.StringVar(&(dataString.CloudwatchAgentTag), "cloudwatch_agent_tag", "", "CloudWatch Agent tag")
@@ -318,6 +321,7 @@ func GetEnvironmentMetaData() *MetaData {
318321
metaDataStorage.AmpWorkspaceId = registeredMetaDataStrings.AmpWorkspaceId
319322
metaDataStorage.Region = registeredMetaDataStrings.Region
320323
metaDataStorage.K8sVersion = registeredMetaDataStrings.K8sVersion
324+
metaDataStorage.Destroy = registeredMetaDataStrings.Destroy
321325
metaDataStorage.HelmChartsBranch = registeredMetaDataStrings.HelmChartsBranch
322326
metaDataStorage.CloudwatchAgentRepository = registeredMetaDataStrings.CloudwatchAgentRepository
323327
metaDataStorage.CloudwatchAgentTag = registeredMetaDataStrings.CloudwatchAgentTag
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[
2+
{
3+
"nodes": 2
4+
}
5+
]

generator/resources/eks_e2e_test_matrix.json

Lines changed: 0 additions & 5 deletions
This file was deleted.

generator/test_case_generator.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ type matrixRow struct {
3030
CaCertPath string `json:"caCertPath"`
3131
ValuesPerMinute int `json:"values_per_minute"` // Number of metrics to be sent or number of log lines to write
3232
K8sVersion string `json:"k8sVersion"`
33+
Nodes int `json:"nodes"`
34+
DeploymentStrategy string `json:"deploymentStrategy"`
3335
TerraformDir string `json:"terraform_dir"`
3436
UseSSM bool `json:"useSSM"`
3537
ExcludedTests string `json:"excludedTests"`
@@ -247,7 +249,7 @@ var testTypeToTestConfig = map[string][]testConfig{
247249
}
248250

249251
var testTypeToTestConfigE2E = map[string][]testConfig{
250-
"eks_e2e": {
252+
"eks_e2e_jmx": {
251253
{testDir: "../../../test/e2e/jmx"},
252254
},
253255
}

terraform/eks/e2e/main.tf

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,10 @@ resource "aws_eks_node_group" "this" {
3939
min_size = var.nodes
4040
}
4141

42-
ami_type = "AL2_x86_64"
42+
ami_type = var.ami_type
4343
capacity_type = "ON_DEMAND"
4444
disk_size = 20
45-
instance_types = ["t3a.medium"]
45+
instance_types = [var.instance_type]
4646

4747
depends_on = [
4848
aws_iam_role_policy_attachment.node_CloudWatchAgentServerPolicy,
@@ -52,6 +52,15 @@ resource "aws_eks_node_group" "this" {
5252
]
5353
}
5454

55+
resource "aws_security_group_rule" "nodeport_inbound" {
56+
type = "ingress"
57+
from_port = 30080
58+
to_port = 30080
59+
protocol = "tcp"
60+
cidr_blocks = ["0.0.0.0/0"]
61+
security_group_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id
62+
}
63+
5564
resource "aws_iam_role" "node_role" {
5665
name = "${local.cluster_name}-Worker-Role-${module.common.testing_id}"
5766

@@ -109,6 +118,13 @@ resource "null_resource" "helm_charts" {
109118
resource "null_resource" "validator" {
110119
depends_on = [aws_eks_cluster.this, aws_eks_node_group.this, null_resource.helm_charts]
111120

121+
triggers = {
122+
cluster_name = aws_eks_cluster.this.name
123+
region = var.region
124+
test_dir = var.test_dir
125+
eks_deployment_strategy = var.eks_deployment_strategy
126+
}
127+
112128
provisioner "local-exec" {
113129
command = <<-EOT
114130
echo "Validating K8s resources and metrics"
@@ -117,7 +133,7 @@ resource "null_resource" "validator" {
117133
-k8s_version=${var.k8s_version} \
118134
-eksClusterName=${aws_eks_cluster.this.name} \
119135
-computeType=EKS \
120-
-eksDeploymentStrategy=DAEMON \
136+
-eksDeploymentStrategy=${var.eks_deployment_strategy} \
121137
-helm_charts_branch=${var.helm_charts_branch} \
122138
-cloudwatch_agent_repository=${var.cloudwatch_agent_repository} \
123139
-cloudwatch_agent_tag=${var.cloudwatch_agent_tag} \
@@ -134,4 +150,17 @@ resource "null_resource" "validator" {
134150
-sample_app="${var.test_dir}/${var.sample_app}"
135151
EOT
136152
}
153+
154+
provisioner "local-exec" {
155+
when = destroy
156+
command = <<-EOT
157+
echo "Running cleanup for K8s resources"
158+
go test -timeout 30m -v ${self.triggers.test_dir} \
159+
-destroy \
160+
-region=${self.triggers.region} \
161+
-eksClusterName=${self.triggers.cluster_name} \
162+
-computeType=EKS \
163+
-eksDeploymentStrategy=${self.triggers.eks_deployment_strategy}
164+
EOT
165+
}
137166
}

terraform/eks/e2e/variables.tf

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,17 @@ variable "cluster_name" {
1818

1919
variable "nodes" {
2020
type = number
21-
default = 1
21+
default = 2
22+
}
23+
24+
variable "ami_type" {
25+
type = string
26+
default = "AL2_x86_64"
27+
}
28+
29+
variable "instance_type" {
30+
type = string
31+
default = "t3a.medium"
2232
}
2333

2434
variable "helm_charts_branch" {
@@ -95,3 +105,8 @@ variable "sample_app" {
95105
type = string
96106
default = ""
97107
}
108+
109+
variable "eks_deployment_strategy" {
110+
type = string
111+
default = "DAEMON"
112+
}

test/e2e/envutils.go

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package e2e
5+
6+
import (
7+
"fmt"
8+
"os"
9+
"os/exec"
10+
"path/filepath"
11+
"strings"
12+
13+
"github.com/aws/amazon-cloudwatch-agent-test/environment"
14+
"github.com/aws/amazon-cloudwatch-agent-test/util/awsservice"
15+
)
16+
17+
//------------------------------------------------------------------------------
18+
// Environment Setup
19+
//------------------------------------------------------------------------------
20+
21+
func InitializeEnvironment(env *environment.MetaData) error {
22+
if env.Region != "us-west-2" {
23+
if err := awsservice.ConfigureAWSClients(env.Region); err != nil {
24+
return fmt.Errorf("failed to reconfigure AWS clients: %v", err)
25+
}
26+
fmt.Printf("AWS clients reconfigured to use region: %s\n", env.Region)
27+
} else {
28+
fmt.Printf("Using default testing region: us-west-2\n")
29+
}
30+
31+
fmt.Println("Applying K8s resources...")
32+
if err := ApplyResources(env); err != nil {
33+
return fmt.Errorf("failed to apply K8s resources: %v", err)
34+
}
35+
36+
return nil
37+
}
38+
39+
//------------------------------------------------------------------------------
40+
// K8s Resource Management Functions
41+
//------------------------------------------------------------------------------
42+
43+
func ApplyResources(env *environment.MetaData) error {
44+
updateKubeconfig := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName)
45+
output, err := updateKubeconfig.CombinedOutput()
46+
if err != nil {
47+
return fmt.Errorf("failed to update kubeconfig: %w\nOutput: %s", err, output)
48+
}
49+
50+
fmt.Println("Installing Helm release...")
51+
helm := []string{
52+
"helm", "upgrade", "--install", "amazon-cloudwatch-observability",
53+
filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability"),
54+
"--set", fmt.Sprintf("clusterName=%s", env.EKSClusterName),
55+
"--set", fmt.Sprintf("region=%s", env.Region),
56+
"--set", fmt.Sprintf("agent.image.repository=%s", env.CloudwatchAgentRepository),
57+
"--set", fmt.Sprintf("agent.image.tag=%s", env.CloudwatchAgentTag),
58+
"--set", fmt.Sprintf("agent.image.repositoryDomainMap.public=%s", env.CloudwatchAgentRepositoryURL),
59+
"--set", fmt.Sprintf("manager.image.repository=%s", env.CloudwatchAgentOperatorRepository),
60+
"--set", fmt.Sprintf("manager.image.tag=%s", env.CloudwatchAgentOperatorTag),
61+
"--set", fmt.Sprintf("manager.image.repositoryDomainMap.public=%s", env.CloudwatchAgentOperatorRepositoryURL),
62+
"--namespace", "amazon-cloudwatch",
63+
"--create-namespace",
64+
}
65+
66+
if env.AgentConfig != "" {
67+
agentConfigContent, err := os.ReadFile(env.AgentConfig)
68+
if err != nil {
69+
return fmt.Errorf("failed to read agent config file: %w", err)
70+
}
71+
helm = append(helm, "--set-json", fmt.Sprintf("agent.config=%s", string(agentConfigContent)))
72+
}
73+
74+
helmUpgrade := exec.Command(helm[0], helm[1:]...)
75+
helmUpgrade.Stdout = os.Stdout
76+
helmUpgrade.Stderr = os.Stderr
77+
if err := helmUpgrade.Run(); err != nil {
78+
return fmt.Errorf("failed to install Helm release: %w", err)
79+
}
80+
81+
fmt.Println("Waiting for CloudWatch Agent Operator to initialize...")
82+
wait := exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=60s", "deployment/amazon-cloudwatch-observability-controller-manager", "-n", "amazon-cloudwatch")
83+
output, err = wait.CombinedOutput()
84+
if err != nil {
85+
return fmt.Errorf("failed to wait for operator deployment: %w\nOutput: %s", err, output)
86+
}
87+
88+
deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml")
89+
90+
apply := exec.Command("kubectl", "apply", "-f", env.SampleApp)
91+
output, err = apply.CombinedOutput()
92+
if err != nil {
93+
return fmt.Errorf("failed to apply sample app: %w\nOutput: %s", err, output)
94+
}
95+
96+
fmt.Println("Waiting for Sample Application to initialize...")
97+
wait = exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=300s", fmt.Sprintf("deployment/%s", deploymentName), "-n", "test")
98+
output, err = wait.CombinedOutput()
99+
if err != nil {
100+
return fmt.Errorf("failed to wait for deployment %s: %w\nOutput: %s", deploymentName, err, output)
101+
}
102+
103+
return nil
104+
}
105+
106+
func DestroyResources(env *environment.MetaData) error {
107+
updateKubeconfig := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName)
108+
output, err := updateKubeconfig.CombinedOutput()
109+
if err != nil {
110+
return fmt.Errorf("failed to update kubeconfig: %w\nOutput: %s", err, output)
111+
}
112+
113+
var errors []error
114+
115+
fmt.Println("Deleting test namespace...")
116+
deleteCmd := exec.Command("kubectl", "delete", "namespace", "test", "--timeout=60s")
117+
output, err = deleteCmd.CombinedOutput()
118+
119+
// We don't want to consider not finding the namespace to be an error since that's the outcome we want
120+
if err != nil && !strings.Contains(string(output), "not found") {
121+
errors = append(errors, fmt.Errorf("failed to delete test namespace: %w\nOutput: %s", err, output))
122+
}
123+
124+
fmt.Println("Uninstalling Helm release...")
125+
helm := []string{
126+
"helm", "uninstall", "amazon-cloudwatch-observability", "--namespace", "amazon-cloudwatch",
127+
}
128+
129+
helmUninstall := exec.Command(helm[0], helm[1:]...)
130+
helmUninstall.Stdout = os.Stdout
131+
helmUninstall.Stderr = os.Stderr
132+
if err := helmUninstall.Run(); err != nil {
133+
errors = append(errors, fmt.Errorf("failed to uninstall Helm release: %w", err))
134+
}
135+
136+
if len(errors) > 0 {
137+
return fmt.Errorf("cleanup errors: %v", errors)
138+
}
139+
140+
return nil
141+
}

0 commit comments

Comments
 (0)