Skip to content

Commit 6a76946

Browse files
fredjywangfredjywang
authored
Implement end-to-end EKS integration test (#154)
1 parent 5adc837 commit 6a76946

32 files changed

+556
-56
lines changed

‎.github/.codecov.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,5 @@ ignore:
2424
- "config/**/*"
2525
- "pkg/apis/**/*"
2626
- "mocks/**/*"
27-
- "integration/scenarios/**/*"
27+
- "integration/shared/scenarios/**/*"
2828
- "pkg/common/logger.go"

‎.github/workflows/integration-test.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ jobs:
2626
- name: Set up env
2727
run: source ~/.bashrc
2828
- name: Start clean
29-
run: make integration-cleanup
29+
run: make kind-integration-cleanup
3030
- name: Set up cluster
31-
run: make integration-setup
31+
run: make kind-integration-setup
3232
- name: Run tests
33-
run: make integration-run
33+
run: make kind-integration-run
3434
- name: Clean up clusters
35-
run: make integration-cleanup
35+
run: make kind-integration-cleanup

‎CONTRIBUTING.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -173,15 +173,15 @@ You must first push a Docker image containing the changes to a Docker repository
173173
If you are deploying to cluster using kustomize templates from the `config` directory, you will need to override the image URI away from `ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s` in order to use your own docker images.
174174

175175

176-
## Integration testing
176+
## Local integration testing
177177
The end-to-end integration test suite can be run locally to validate controller core functionality. This will provision a local Kind cluster and build and run the AWS Cloud Map MCS Controller for K8s. The test will verify service endpoints sync with AWS Cloud Map. If successful, the suite will then de-provision the local test cluster and delete AWS Cloud Map namespace `aws-cloud-map-mcs-e2e` along with test service and service instance resources:
178178
```sh
179-
make integration-suite
179+
make kind-integration-suite
180180
```
181181

182182
If integration test suite fails for some reason, you can perform a cleanup:
183183
```sh
184-
make integration-cleanup
184+
make kind-integration-cleanup
185185
```
186186

187187
## Build and push docker image to ECR

‎Makefile

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -79,20 +79,36 @@ test-setup: ## setup test environment
7979
test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh
8080
source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR)
8181

82-
integration-suite: ## Provision and run integration tests with cleanup
83-
make integration-setup && \
84-
make integration-run && \
85-
make integration-cleanup
82+
kind-integration-suite: ## Provision and run integration tests with cleanup
83+
make kind-integration-setup && \
84+
make kind-integration-run && \
85+
make kind-integration-cleanup
8686

87-
integration-setup: build kind test-setup ## Setup the integration test using kind clusters
88-
@./integration/scripts/setup-kind.sh
87+
kind-integration-setup: build kind test-setup ## Setup the integration test using kind clusters
88+
@./integration/kind-test/scripts/setup-kind.sh
8989

90-
integration-run: ## Run the integration test controller
91-
@./integration/scripts/run-tests.sh
90+
kind-integration-run: ## Run the integration test controller
91+
@./integration/kind-test/scripts/run-tests.sh
9292

93-
integration-cleanup: kind ## Cleanup integration test resources in Cloud Map and local kind cluster
94-
@./integration/scripts/cleanup-cloudmap.sh
95-
@./integration/scripts/cleanup-kind.sh
93+
kind-integration-cleanup: kind ## Cleanup integration test resources in Cloud Map and local kind cluster
94+
@./integration/kind-test/scripts/cleanup-kind.sh
95+
96+
eks-integration-suite: ## Provision and run EKS integration tests with cleanup
97+
make eks-integration-setup && \
98+
make eks-integration-run && \
99+
make eks-integration-cleanup
100+
101+
eks-integration-setup: build test-setup ## Setup the integration test using EKS clusters
102+
@./integration/eks-test/scripts/eks-setup.sh
103+
104+
eks-integration-run: ## Run the integration test controller
105+
@./integration/eks-test/scripts/eks-run-tests.sh
106+
107+
eks-integration-cleanup: ## Cleanup integration test resources in Cloud Map and EKS cluster
108+
@./integration/eks-test/scripts/eks-cleanup.sh
109+
110+
eks-test:
111+
@./integration/eks-test/scripts/eks-test.sh
96112

97113
##@ Build
98114

‎integration/eks-test/configs/client-hello.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: client-hello
5+
namespace: aws-cloud-map-mcs-eks-e2e
6+
spec:
7+
containers:
8+
- command:
9+
- sleep
10+
- "1d"
11+
image: alpine
12+
name: client-hello

‎integration/eks-test/configs/coredns-clusterrole.yaml

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRole
3+
metadata:
4+
labels:
5+
eks.amazonaws.com/component: coredns
6+
k8s-app: kube-dns
7+
kubernetes.io/bootstrapping: rbac-defaults
8+
name: system:coredns
9+
rules:
10+
- apiGroups:
11+
- ""
12+
resources:
13+
- endpoints
14+
- services
15+
- pods
16+
- namespaces
17+
verbs:
18+
- list
19+
- watch
20+
- apiGroups:
21+
- ""
22+
resources:
23+
- nodes
24+
verbs:
25+
- get
26+
- apiGroups:
27+
- discovery.k8s.io
28+
resources:
29+
- endpointslices
30+
verbs:
31+
- create
32+
- get
33+
- list
34+
- patch
35+
- update
36+
- watch
37+
- apiGroups:
38+
- multicluster.x-k8s.io
39+
resources:
40+
- serviceimports
41+
verbs:
42+
- create
43+
- get
44+
- list
45+
- patch
46+
- update
47+
- watch
48+
- apiGroups:
49+
- multicluster.x-k8s.io
50+
resources:
51+
- serviceexports
52+
verbs:
53+
- create
54+
- get
55+
- list
56+
- patch
57+
- update
58+
- watch

‎integration/eks-test/configs/coredns-configmap.yaml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
apiVersion: v1
2+
data:
3+
Corefile: |
4+
.:53 {
5+
errors
6+
health
7+
multicluster clusterset.local
8+
kubernetes cluster.local in-addr.arpa ip6.arpa {
9+
pods insecure
10+
fallthrough in-addr.arpa ip6.arpa
11+
}
12+
prometheus :9153
13+
forward . /etc/resolv.conf
14+
cache 30
15+
loop
16+
reload
17+
loadbalance
18+
}
19+
kind: ConfigMap
20+
metadata:
21+
annotations:
22+
labels:
23+
eks.amazonaws.com/component: coredns
24+
k8s-app: kube-dns
25+
name: coredns
26+
namespace: kube-system

‎integration/eks-test/configs/coredns-deployment.yaml

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
labels:
5+
eks.amazonaws.com/component: coredns
6+
k8s-app: kube-dns
7+
kubernetes.io/name: CoreDNS
8+
name: coredns
9+
namespace: kube-system
10+
spec:
11+
progressDeadlineSeconds: 600
12+
replicas: 2
13+
revisionHistoryLimit: 10
14+
selector:
15+
matchLabels:
16+
eks.amazonaws.com/component: coredns
17+
k8s-app: kube-dns
18+
strategy:
19+
rollingUpdate:
20+
maxSurge: 25%
21+
maxUnavailable: 1
22+
type: RollingUpdate
23+
template:
24+
metadata:
25+
annotations:
26+
eks.amazonaws.com/compute-type: ec2
27+
creationTimestamp: null
28+
labels:
29+
eks.amazonaws.com/component: coredns
30+
k8s-app: kube-dns
31+
spec:
32+
affinity:
33+
nodeAffinity:
34+
requiredDuringSchedulingIgnoredDuringExecution:
35+
nodeSelectorTerms:
36+
- matchExpressions:
37+
- key: beta.kubernetes.io/os
38+
operator: In
39+
values:
40+
- linux
41+
- key: beta.kubernetes.io/arch
42+
operator: In
43+
values:
44+
- amd64
45+
- arm64
46+
podAntiAffinity:
47+
preferredDuringSchedulingIgnoredDuringExecution:
48+
- podAffinityTerm:
49+
labelSelector:
50+
matchExpressions:
51+
- key: k8s-app
52+
operator: In
53+
values:
54+
- kube-dns
55+
topologyKey: kubernetes.io/hostname
56+
weight: 100
57+
containers:
58+
- args:
59+
- -conf
60+
- /etc/coredns/Corefile
61+
image: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s/coredns-multicluster/coredns:v1.8.6
62+
imagePullPolicy: IfNotPresent
63+
livenessProbe:
64+
failureThreshold: 5
65+
httpGet:
66+
path: /health
67+
port: 8080
68+
scheme: HTTP
69+
initialDelaySeconds: 60
70+
periodSeconds: 10
71+
successThreshold: 1
72+
timeoutSeconds: 5
73+
name: coredns
74+
ports:
75+
- containerPort: 53
76+
name: dns
77+
protocol: UDP
78+
- containerPort: 53
79+
name: dns-tcp
80+
protocol: TCP
81+
- containerPort: 9153
82+
name: metrics
83+
protocol: TCP
84+
readinessProbe:
85+
failureThreshold: 3
86+
httpGet:
87+
path: /health
88+
port: 8080
89+
scheme: HTTP
90+
periodSeconds: 10
91+
successThreshold: 1
92+
timeoutSeconds: 1
93+
resources:
94+
limits:
95+
memory: 170Mi
96+
requests:
97+
cpu: 100m
98+
memory: 70Mi
99+
securityContext:
100+
allowPrivilegeEscalation: false
101+
capabilities:
102+
add:
103+
- NET_BIND_SERVICE
104+
drop:
105+
- all
106+
readOnlyRootFilesystem: true
107+
terminationMessagePath: /dev/termination-log
108+
terminationMessagePolicy: File
109+
volumeMounts:
110+
- mountPath: /etc/coredns
111+
name: config-volume
112+
readOnly: true
113+
- mountPath: /tmp
114+
name: tmp
115+
dnsPolicy: Default
116+
priorityClassName: system-cluster-critical
117+
restartPolicy: Always
118+
schedulerName: default-scheduler
119+
securityContext: {}
120+
serviceAccount: coredns
121+
serviceAccountName: coredns
122+
terminationGracePeriodSeconds: 30
123+
tolerations:
124+
- effect: NoSchedule
125+
key: node-role.kubernetes.io/master
126+
- key: CriticalAddonsOnly
127+
operator: Exists
128+
volumes:
129+
- emptyDir: {}
130+
name: tmp
131+
- configMap:
132+
defaultMode: 420
133+
items:
134+
- key: Corefile
135+
path: Corefile
136+
name: coredns
137+
name: config-volume

‎integration/eks-test/configs/eksctl-cluster.yaml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: eksctl.io/v1alpha5
2+
kind: ClusterConfig
3+
metadata:
4+
name: $CLUSTER_NAME
5+
region: $AWS_REGION
6+
version: "1.22"
7+
vpc:
8+
cidr: $VPC_CIDR
9+
autoAllocateIPv6: false
10+
clusterEndpoints:
11+
publicAccess: true
12+
privateAccess: true
13+
managedNodeGroups:
14+
- name: $NODEGROUP_NAME
15+
instanceType: t3.small
16+
minSize: 1
17+
maxSize: 10
18+
desiredCapacity: 1

‎integration/eks-test/configs/nginx-deployment.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
namespace: aws-cloud-map-mcs-eks-e2e
5+
name: nginx-demo
6+
labels:
7+
app: nginx
8+
spec:
9+
replicas: 3
10+
selector:
11+
matchLabels:
12+
app: nginx
13+
template:
14+
metadata:
15+
labels:
16+
app: nginx
17+
spec:
18+
containers:
19+
- name: nginx
20+
image: nginxdemos/hello:plain-text
21+
ports:
22+
- containerPort: 80

0 commit comments

Comments
 (0)