Add integration test for service export (#69)

Author: astaticvoid

.gitignore

@@ -1,6 +1,7 @@
 vendor/
 bin/
 testbin/
+testlog/
 cover.out
 
 # Files generated by JetBrains IDEs, e.g. IntelliJ IDEA

Makefile

@@ -60,6 +60,16 @@ test: manifests generate generate-mocks fmt vet ## Run tests.
 	test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh
 	source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out -covermode=atomic
 
+integration-setup:  ## Setup the integration test using kind clusters
+	@./integration/scripts/setup-kind.sh
+
+integration-run: ## Run the integration test controller
+	@./integration/scripts/run-tests.sh
+
+integration-cleanup:  ## Cleanup integration test resources in Cloud Map and local kind cluster
+	@./integration/scripts/cleanup-cloudmap.sh
+	@./integration/scripts/cleanup-kind.sh
+
 ##@ Build
 
 build: manifests generate generate-mocks fmt vet ## Build manager binary.

integration/configs/e2e-deployment.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: aws-cloud-map-mcs-e2e
+  name: coredns-deployment
+  labels:
+    app: coredns
+spec:
+  replicas: 5
+  selector:
+    matchLabels:
+      app: coredns
+  template:
+    metadata:
+      labels:
+        app: coredns
+    spec:
+      containers:
+        - name: coredns
+          image: k8s.gcr.io/coredns:1.7.0
+          ports:
+            - containerPort: 80

integration/configs/e2e-export.yaml

@@ -0,0 +1,5 @@
+kind: ServiceExport
+apiVersion: multicluster.x-k8s.io/v1alpha1
+metadata:
+  namespace: aws-cloud-map-mcs-e2e
+  name: e2e-service

integration/configs/e2e-service.yaml

@@ -0,0 +1,11 @@
+kind: Service
+apiVersion: v1
+metadata:
+  namespace: aws-cloud-map-mcs-e2e
+  name: e2e-service
+spec:
+  selector:
+    app: coredns
+  ports:
+    - port: 8080
+      targetPort: 80

integration/janitor/janitor.go

@@ -22,9 +22,7 @@ type cloudMapJanitor struct {
 
 // NewDefaultJanitor returns a new janitor object.
 func NewDefaultJanitor() CloudMapJanitor {
-	awsCfg, err := config.LoadDefaultConfig(context.TODO(),
-		config.WithRegion(os.Getenv("AWS_REGION")),
-	)
+	awsCfg, err := config.LoadDefaultConfig(context.TODO())
 
 	if err != nil {
 		fmt.Printf("unable to configure AWS session: %s", err.Error())

integration/janitor/runner/main.go

@@ -2,14 +2,18 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/integration/janitor"
-)
-
-const (
-	e2eNs = "aws-cloud-map-mcs-e2e"
+	"os"
 )
 
 func main() {
+	if len(os.Args) != 2 {
+		fmt.Println("Expected single namespace name argument")
+		os.Exit(1)
+	}
+
 	j := janitor.NewDefaultJanitor()
-	j.Cleanup(context.TODO(), e2eNs)
+	nsName := os.Args[1]
+	j.Cleanup(context.TODO(), nsName)
 }

integration/scenarios/export_service.go

@@ -0,0 +1,109 @@
+package scenarios
+
+import (
+	"context"
+	"fmt"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/controllers"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	defaultScenarioPollInterval = 10 * time.Second
+	defaultScenarioPollTimeout  = 2 * time.Minute
+)
+
+// ExportServiceScenario defines an integration test against a service export to check creation of namespace, service,
+// and endpoint export.
+type ExportServiceScenario interface {
+	// Run executes the service export integration test scenario, returning any error.
+	Run() error
+}
+
+type exportServiceScenario struct {
+	sdClient    cloudmap.ServiceDiscoveryClient
+	expectedSvc model.Service
+}
+
+func NewExportServiceScenario(cfg *aws.Config, nsName string, svcName string, portStr string, ips string) (ExportServiceScenario, error) {
+	endpts := make([]*model.Endpoint, 0)
+
+	port, parseError := strconv.ParseUint(portStr, 10, 16)
+	if parseError != nil {
+		return nil, parseError
+	}
+
+	for _, ip := range strings.Split(ips, ",") {
+		endpts = append(endpts, &model.Endpoint{
+			Id:         model.EndpointIdFromIPAddress(ip),
+			IP:         ip,
+			Port:       int32(port),
+			Attributes: make(map[string]string, 0),
+		})
+	}
+
+	return &exportServiceScenario{
+		sdClient: cloudmap.NewServiceDiscoveryClientWithCustomCache(cfg,
+			&cloudmap.SdCacheConfig{
+				NsTTL:    time.Second,
+				SvcTTL:   time.Second,
+				EndptTTL: time.Second,
+			}),
+		expectedSvc: model.Service{
+			Namespace: nsName,
+			Name:      svcName,
+			Endpoints: endpts,
+		},
+	}, nil
+}
+
+func (e *exportServiceScenario) Run() error {
+	fmt.Printf("Seeking expected service: %v\n", e.expectedSvc)
+
+	return wait.Poll(defaultScenarioPollInterval, defaultScenarioPollTimeout, func() (done bool, err error) {
+		fmt.Println("Polling service...")
+		cmSvc, err := e.sdClient.GetService(context.TODO(), e.expectedSvc.Namespace, e.expectedSvc.Name)
+		if err != nil {
+			return true, err
+		}
+
+		if cmSvc == nil {
+			fmt.Println("Service not found.")
+			return false, nil
+		}
+
+		fmt.Printf("Found service: %v\n", cmSvc)
+		return e.compareEndpoints(cmSvc.Endpoints), nil
+	})
+}
+
+func (e *exportServiceScenario) compareEndpoints(cmEndpoints []*model.Endpoint) bool {
+	if len(e.expectedSvc.Endpoints) != len(cmEndpoints) {
+		fmt.Println("Endpoints do not match.")
+		return false
+	}
+
+	for _, expected := range e.expectedSvc.Endpoints {
+		match := false
+		for _, actual := range cmEndpoints {
+			// Ignore K8S instance attribute for the purpose of this test.
+			delete(actual.Attributes, controllers.K8sVersionAttr)
+			if expected.Equals(actual) {
+				match = true
+				break
+			}
+		}
+		if !match {
+			fmt.Println("Endpoints do not match.")
+			return false
+		}
+	}
+
+	fmt.Println("Endpoints match.")
+	return true
+}

integration/scenarios/runner/main.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/integration/scenarios"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"os"
+)
+
+func main() {
+	if len(os.Args) != 5 {
+		fmt.Println("Expected namespace, service, endpoint port, and endpoint IP list arguments")
+		os.Exit(1)
+	}
+
+	nsName := os.Args[1]
+	svcName := os.Args[2]
+	port := os.Args[3]
+	ips := os.Args[4]
+
+	testServiceExport(nsName, svcName, port, ips)
+}
+
+func testServiceExport(nsName string, svcName string, port string, ips string) {
+	fmt.Printf("Testing service export integration for namespace %s and service %s\n", nsName, svcName)
+
+	export, err := scenarios.NewExportServiceScenario(getAwsConfig(), nsName, svcName, port, ips)
+	if err != nil {
+		fmt.Printf("Failed to setup service export integration test scenario: %s", err.Error())
+		os.Exit(1)
+	}
+
+	if err := export.Run(); err != nil {
+		fmt.Printf("Service export integration test scenario failed: %s", err.Error())
+		os.Exit(1)
+	}
+}
+
+func getAwsConfig() *aws.Config {
+	awsCfg, err := config.LoadDefaultConfig(context.TODO())
+
+	if err != nil {
+		fmt.Printf("unable to configure AWS session: %s", err.Error())
+		os.Exit(1)
+	}
+
+	return &awsCfg
+}

integration/scripts/cleanup-cloudmap.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# Deletes all AWS Cloud Map resources used for integration test.
+
+set -eo pipefail
+source ./integration/scripts/common.sh
+
+go run ./integration/janitor/runner/main.go "$NAMESPACE"