Headless services local integration test (#194)

Author: fredjywang

Makefile

@@ -94,7 +94,7 @@ kind-integration-setup: build kind ## Setup the integration test using kind clus
 	@./integration/kind-test/scripts/setup-kind.sh
 
 kind-integration-run: ## Run the integration test controller
-	@./integration/kind-test/scripts/run-tests.sh
+	@./integration/kind-test/scripts/run-helper.sh
 
 kind-integration-cleanup: kind  ## Cleanup integration test resources in Cloud Map and local kind cluster
 	@./integration/kind-test/scripts/cleanup-kind.sh

integration/eks-test/scripts/eks-common.sh

@@ -8,6 +8,7 @@ export SCENARIOS='./integration/shared/scenarios'
 export NAMESPACE='aws-cloud-map-mcs-eks-e2e'
 export MCS_NAMESPACE='cloud-map-mcs-system'
 export SERVICE='nginx-hello'
+export SERVICE_TYPE='ClusterSetIP'
 export CLIENT_POD='client-hello'
 export ENDPT_PORT=80
 export SERVICE_PORT=80 # from nginx-service.yaml

integration/eks-test/scripts/eks-run-tests.sh

@@ -11,7 +11,7 @@ if ! endpts=$(./integration/shared/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT
 fi
 
 # Runner to verify expected endpoints are exported to Cloud Map
-go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$endpts"
+go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE "$endpts"
 exit_code=$?
 
 # Check imported endpoints in importing cluster
@@ -47,7 +47,7 @@ if [ "$exit_code" -eq 0 ] ; then
 fi
 
 if [ "$exit_code" -eq 0 ] ; then
-  go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$updated_endpoints"
+  go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT $SERVICE_TYPE "$updated_endpoints" 
   exit_code=$?
 fi
 

integration/eks-test/scripts/eks-setup.sh

@@ -2,19 +2,17 @@
 
 source ./integration/eks-test/scripts/eks-common.sh
 
+# Call helper for service account and controller installation
+./integration/eks-test/scripts/eks-setup-helper.sh $EXPORT_CLS
+./integration/eks-test/scripts/eks-setup-helper.sh $IMPORT_CLS
+
 # Apply ClusterProperties
 $KUBECTL_BIN config use-context $EXPORT_CLS
-make install
 $KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterproperty-1.yaml"
 
 $KUBECTL_BIN config use-context $IMPORT_CLS
-make install
 $KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterproperty-2.yaml"
 
-# Call helper for service account and controller installation
-./integration/eks-test/scripts/eks-setup-helper.sh $EXPORT_CLS
-./integration/eks-test/scripts/eks-setup-helper.sh $IMPORT_CLS
-
 # Installing service
 $KUBECTL_BIN config use-context $EXPORT_CLS
 $KUBECTL_BIN create namespace $NAMESPACE

integration/kind-test/configs/e2e-client-hello.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: client-hello
+  namespace: aws-cloud-map-mcs-e2e
+spec:
+  containers:
+    - command:
+        - sleep
+        - "1d"
+      image: alpine
+      name: client-hello

integration/kind-test/configs/e2e-coredns-deployment.yaml

@@ -0,0 +1,137 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    # eks.amazonaws.com/component: coredns
+    k8s-app: kube-dns
+    kubernetes.io/name: CoreDNS
+  name: coredns
+  namespace: kube-system
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 2
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      # eks.amazonaws.com/component: coredns
+      k8s-app: kube-dns
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      # annotations:
+      #   eks.amazonaws.com/compute-type: ec2
+      creationTimestamp: null
+      labels:
+        # eks.amazonaws.com/component: coredns
+        k8s-app: kube-dns
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: beta.kubernetes.io/os
+                operator: In
+                values:
+                - linux
+              - key: beta.kubernetes.io/arch
+                operator: In
+                values:
+                - amd64
+                - arm64
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: k8s-app
+                  operator: In
+                  values:
+                  - kube-dns
+              topologyKey: kubernetes.io/hostname
+            weight: 100
+      containers:
+      - args:
+        - -conf
+        - /etc/coredns/Corefile
+        image: ghcr.io/aws/aws-cloud-map-mcs-controller-for-k8s/coredns-multicluster/coredns:v1.8.4
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 5
+        name: coredns
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 1
+        resources:
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            memory: 70Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /etc/coredns
+          name: config-volume
+          readOnly: true
+        - mountPath: /tmp
+          name: tmp
+      dnsPolicy: Default
+      priorityClassName: system-cluster-critical
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      serviceAccount: coredns
+      serviceAccountName: coredns
+      terminationGracePeriodSeconds: 30
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      - key: CriticalAddonsOnly
+        operator: Exists
+      volumes:
+      - emptyDir: {}
+        name: tmp
+      - configMap:
+          defaultMode: 420
+          items:
+          - key: Corefile
+            path: Corefile
+          name: coredns
+        name: config-volume

integration/kind-test/configs/e2e-headless-export.yaml

@@ -0,0 +1,5 @@
+kind: ServiceExport
+apiVersion: multicluster.x-k8s.io/v1alpha1
+metadata:
+  namespace: aws-cloud-map-mcs-e2e
+  name: e2e-headless

integration/kind-test/configs/e2e-headless.yaml

@@ -0,0 +1,12 @@
+kind: Service
+apiVersion: v1
+metadata:
+  namespace: aws-cloud-map-mcs-e2e
+  name: e2e-headless
+spec:
+  clusterIP: None
+  selector:
+    app: coredns
+  ports:
+    - port: 8080
+      targetPort: 80

integration/kind-test/scripts/DNS-test.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# Testing service consumption with dnsutils pod
+
+echo "verifying single-cluster service consumption..."
+
+# Helper function to verify DNS results
+checkDNS() {
+    endpt_count=$(echo "$1" | wc -l | xargs)
+
+    if [ "$2" = "Headless" ]; then
+        if [ "$endpt_count" -ne "$3" ]; then
+            echo "ERROR: Found $endpt_count endpoints, expected $3 endpoints"
+            exit 1
+        fi
+    fi
+
+    if [ "$2" = "ClusterSetIP" ]; then
+        if [ "$endpt_count" -ne 1 ]; then
+            echo "ERROR: Found $endpt_count endpoints, expected 1 endpoint"
+            exit 1
+        fi
+    fi
+}
+
+# Add pod
+$KUBECTL_BIN apply -f "$KIND_CONFIGS/e2e-client-hello.yaml"
+$KUBECTL_BIN wait --for=condition=ready pod/$DNS_POD -n $NAMESPACE # wait until pod is deployed
+
+# Install dig if not installed
+$KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- dig -v &>/dev/null
+exit_code=$?
+if [ "$exit_code" -ne 0 ]; then
+    echo "dig not installed, installing..."
+    $KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- apk add --update bind-tools
+fi
+
+# Perform a dig to cluster-local CoreDNS 
+# TODO: parse dig outputs for more precise verification - check specifics IPs?
+echo "performing dig for A/AAAA records..."
+addresses=$($KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local +short)
+exit_code=$?
+echo "$addresses"
+
+if [ "$exit_code" -ne 0 ]; then
+    echo "ERROR: Unable to dig service $SERVICE.$NAMESPACE.svc.clusterset.local"
+    exit $exit_code
+fi
+
+# verify DNS results
+checkDNS "$addresses" "$SERVICE_TYPE" "$1"
+
+echo "performing dig for SRV records..."
+addresses=$($KUBECTL_BIN exec $DNS_POD -n $NAMESPACE -- dig +all +ans $SERVICE.$NAMESPACE.svc.clusterset.local. SRV +short)
+exit_code=$?
+echo "$addresses"
+
+if [ "$exit_code" -ne 0 ]; then
+    echo "ERROR: Unable to dig service $SERVICE.$NAMESPACE.svc.clusterset.local"
+    exit $exit_code
+fi
+
+# verify DNS results
+checkDNS "$addresses" "$SERVICE_TYPE" "$1"
+
+echo "confirmed service consumption"
+exit 0

integration/kind-test/scripts/common.sh

@@ -3,16 +3,19 @@
 export KIND_BIN='./bin/kind'
 export KUBECTL_BIN='kubectl'
 export LOGS='./integration/kind-test/testlog'
-export CONFIGS='./integration/kind-test/configs'
+export KIND_CONFIGS='./integration/kind-test/configs'
+export SHARED_CONFIGS='./integration/shared/configs'
 export SCENARIOS='./integration/shared/scenarios'
 export NAMESPACE='aws-cloud-map-mcs-e2e'
-export SERVICE='e2e-service'
 export ENDPT_PORT=80
 export SERVICE_PORT=8080
+export CLUSTERIP_SERVICE='e2e-service'
+export HEADLESS_SERVICE='e2e-headless'
 export KIND_SHORT='cloud-map-e2e'
 export CLUSTER='kind-cloud-map-e2e'
 export CLUSTERID1='kind-e2e-clusterid-1'
 export CLUSTERSETID1='kind-e2e-clustersetid-1'
-export IMAGE='kindest/node:v1.20.15@sha256:a6ce604504db064c5e25921c6c0fffea64507109a1f2a512b1b562ac37d652f3'
+export DNS_POD='client-hello'
+export IMAGE='kindest/node:v1.21.12@sha256:f316b33dd88f8196379f38feb80545ef3ed44d9197dca1bfd48bcb1583210207'
 export EXPECTED_ENDPOINT_COUNT=5
 export UPDATED_ENDPOINT_COUNT=6