EKS Integration Test Fixes (#187)

* Added managed by label - issue 110

* derived services + unit tests + label

* revisions and delete svc bug

* annotation readded. kind & janitor test changes.

* fix expected test result. cluster-id -> cluster

* working eks integration tests

Author: matthewgoodman13

config/rbac/role.yaml

@@ -23,6 +23,18 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - about.k8s.io
+  resources:
+  - clusterproperties
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - discovery.k8s.io
   resources:

integration/eks-test/configs/coredns-clusterrole.yaml

@@ -23,6 +23,17 @@ rules:
   - nodes
   verbs:
   - get
+- apiGroups:
+  - about.k8s.io
+  resources:
+  - clusterproperties
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - discovery.k8s.io
   resources:

integration/eks-test/configs/e2e-clusterproperty-1.yaml

@@ -0,0 +1,13 @@
+apiVersion: about.k8s.io/v1alpha1
+kind: ClusterProperty
+metadata:
+  name: id.k8s.io
+spec:
+  value: eks-e2e-clusterid-1
+---
+apiVersion: about.k8s.io/v1alpha1
+kind: ClusterProperty
+metadata:
+  name: clusterset.k8s.io
+spec:
+  value: eks-e2e-clustersetid-1

integration/eks-test/configs/e2e-clusterproperty-2.yaml

@@ -0,0 +1,13 @@
+apiVersion: about.k8s.io/v1alpha1
+kind: ClusterProperty
+metadata:
+  name: id.k8s.io
+spec:
+  value: eks-e2e-clusterid-2
+---
+apiVersion: about.k8s.io/v1alpha1
+kind: ClusterProperty
+metadata:
+  name: clusterset.k8s.io
+spec:
+  value: eks-e2e-clustersetid-1

integration/eks-test/scripts/eks-DNS-test.sh

@@ -4,22 +4,34 @@
 
 echo "verifying cross-cluster service consumption..."
 
+# Install curl if not installed
 $KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- curl --version &>/dev/null
 exit_code=$?
-
-# Install curl if not installed
 if [ "$exit_code" -eq 126 ]; then
+    echo "curl not installed, installing..."
     $KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- apk add curl
 fi
 
+# Perform an nslookup to cluster-local CoreDNS 
+echo "performing nslookup..."
+$KUBECTL_BIN exec -it $CLIENT_POD -n $NAMESPACE /bin/sh -- nslookup $SERVICE.$NAMESPACE.svc.clusterset.local
+exit_code=$?
+
+if [ "$exit_code" -ne 0 ]; then
+    echo "ERROR: Unable to nslookup service $SERVICE.$NAMESPACE.svc.clusterset.local"
+    exit $exit_code
+fi
+sleep 5
+
 # Call to DNS server, if unable to reach, importing cluster is not able to properly consume service
-$KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- curl -s $SERVICE.$NAMESPACE.svc.clusterset.local
+echo "performing curl..."
+$KUBECTL_BIN exec -it $CLIENT_POD -n $NAMESPACE /bin/sh -- curl $SERVICE.$NAMESPACE.svc.clusterset.local
 exit_code=$?
 
 if [ "$exit_code" -ne 0 ]; then
+    echo "ERROR: Unable to reach service $SERVICE.$NAMESPACE.svc.clusterset.local"
     exit $exit_code
 fi
 
 echo "confirmed service consumption"
 exit 0
-

integration/eks-test/scripts/eks-cleanup.sh

@@ -39,5 +39,6 @@ $KUBECTL_BIN delete -k "github.com/aws/aws-cloud-map-mcs-controller-for-k8s/conf
 
 echo "EKS clusters cleaned!"
 
-./integration/shared/scripts/cleanup-cloudmap.sh
-
+# ./integration/shared/scripts/cleanup-cloudmap.sh
+go run ./integration/janitor/runner/main.go "$NAMESPACE" "$CLUSTERID1" "$CLUSTERSETID1"
+go run ./integration/janitor/runner/main.go "$NAMESPACE" "$CLUSTERID2" "$CLUSTERSETID1"

integration/eks-test/scripts/eks-common.sh

@@ -13,5 +13,8 @@ export ENDPT_PORT=80
 export SERVICE_PORT=80 # from nginx-service.yaml
 export EXPORT_CLS='cls1'
 export IMPORT_CLS='cls2'
+export CLUSTERID1='eks-e2e-clusterid-1'
+export CLUSTERID2='eks-e2e-clusterid-2'
+export CLUSTERSETID1='eks-e2e-clustersetid-1'
 export EXPECTED_ENDPOINT_COUNT=3
 export UPDATED_ENDPOINT_COUNT=4

integration/eks-test/scripts/eks-run-tests.sh

@@ -11,7 +11,7 @@ if ! endpts=$(./integration/shared/scripts/poll-endpoints.sh "$EXPECTED_ENDPOINT
 fi
 
 # Runner to verify expected endpoints are exported to Cloud Map
-go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT "$endpts"
+go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$endpts"
 exit_code=$?
 
 # Check imported endpoints in importing cluster
@@ -47,7 +47,7 @@ if [ "$exit_code" -eq 0 ] ; then
 fi
 
 if [ "$exit_code" -eq 0 ] ; then
-  go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $ENDPT_PORT $SERVICE_PORT "$updated_endpoints"
+  go run $SCENARIOS/runner/main.go $NAMESPACE $SERVICE $CLUSTERID1 $CLUSTERSETID1 $ENDPT_PORT $SERVICE_PORT "$updated_endpoints"
   exit_code=$?
 fi
 

integration/eks-test/scripts/eks-setup.sh

@@ -2,6 +2,15 @@
 
 source ./integration/eks-test/scripts/eks-common.sh
 
+# Apply ClusterProperties
+$KUBECTL_BIN config use-context $EXPORT_CLS
+make install
+$KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterproperty-1.yaml"
+
+$KUBECTL_BIN config use-context $IMPORT_CLS
+make install
+$KUBECTL_BIN apply -f "$CONFIGS/e2e-clusterproperty-2.yaml"
+
 # Call helper for service account and controller installation
 ./integration/eks-test/scripts/eks-setup-helper.sh $EXPORT_CLS
 ./integration/eks-test/scripts/eks-setup-helper.sh $IMPORT_CLS
@@ -23,4 +32,3 @@ $KUBECTL_BIN apply -f "$CONFIGS/nginx-serviceexport.yaml"
 $KUBECTL_BIN config use-context $IMPORT_CLS
 $KUBECTL_BIN apply -f "$CONFIGS/client-hello.yaml"
 sleep 15
-$KUBECTL_BIN exec $CLIENT_POD -n $NAMESPACE /bin/sh -- apk add curl ## install curl

pkg/controllers/multicluster/cloudmap_controller.go

@@ -31,6 +31,7 @@ type CloudMapReconciler struct {
 
 // +kubebuilder:rbac:groups="",resources=namespaces,verbs=list;watch
 // +kubebuilder:rbac:groups="",resources=services,verbs=create;get;list;watch;update;delete
+// +kubebuilder:rbac:groups=about.k8s.io,resources=clusterproperties,verbs=create;get;list;watch;update;patch;delete
 // +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=list;get;create;watch;update;delete
 // +kubebuilder:rbac:groups=multicluster.x-k8s.io,resources=serviceimports,verbs=create;get;list;watch;update;patch;delete