Add janitor to clean up Cloud Map integration test resources (#57)

Author: astaticvoid

Makefile

@@ -97,6 +97,8 @@ generate-mocks: mockgen
 	$(MOCKGEN) --source pkg/cloudmap/operation_collector.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/operation_collector_mock.go --package cloudmap
 	$(MOCKGEN) --source pkg/cloudmap/api.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/api_mock.go --package cloudmap
 	$(MOCKGEN) --source pkg/cloudmap/aws_facade.go --destination $(MOCKS_DESTINATION)/pkg/cloudmap/aws_facade_mock.go --package cloudmap
+	$(MOCKGEN) --source integration/janitor/api.go --destination $(MOCKS_DESTINATION)/integration/janitor/api_mock.go --package janitor
+	$(MOCKGEN) --source integration/janitor/aws_facade.go --destination $(MOCKS_DESTINATION)/integration/janitor/aws_facade_mock.go --package janitor
 
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen

integration/janitor/api.go

@@ -0,0 +1,40 @@
+package janitor
+
+import (
+	"context"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery"
+)
+
+type ServiceDiscoveryJanitorApi interface {
+	DeleteNamespace(ctx context.Context, namespaceId string) (operationId string, err error)
+	DeleteService(ctx context.Context, serviceId string) error
+	cloudmap.ServiceDiscoveryApi
+}
+
+type serviceDiscoveryJanitorApi struct {
+	cloudmap.ServiceDiscoveryApi
+	janitorFacade SdkJanitorFacade
+}
+
+func NewServiceDiscoveryJanitorApiFromConfig(cfg *aws.Config) ServiceDiscoveryJanitorApi {
+	return &serviceDiscoveryJanitorApi{
+		ServiceDiscoveryApi: cloudmap.NewServiceDiscoveryApiFromConfig(cfg),
+		janitorFacade:       NewSdkJanitorFacadeFromConfig(cfg),
+	}
+}
+
+func (api *serviceDiscoveryJanitorApi) DeleteNamespace(ctx context.Context, nsId string) (opId string, err error) {
+	out, err := api.janitorFacade.DeleteNamespace(ctx, &sd.DeleteNamespaceInput{Id: &nsId})
+	if err != nil {
+		return "", err
+	}
+
+	return aws.ToString(out.OperationId), nil
+}
+
+func (api *serviceDiscoveryJanitorApi) DeleteService(ctx context.Context, svcId string) error {
+	_, err := api.janitorFacade.DeleteService(ctx, &sd.DeleteServiceInput{Id: &svcId})
+	return err
+}

integration/janitor/api_test.go

@@ -0,0 +1,51 @@
+package janitor
+
+import (
+	"context"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestNewServiceDiscoveryJanitorApiFromConfig(t *testing.T) {
+	assert.NotNil(t, NewServiceDiscoveryJanitorApiFromConfig(&aws.Config{}))
+}
+
+func TestServiceDiscoveryJanitorApi_DeleteNamespace_HappyCase(t *testing.T) {
+	mockController := gomock.NewController(t)
+	defer mockController.Finish()
+
+	mocksdk := janitor.NewMockSdkJanitorFacade(mockController)
+	jApi := getJanitorApi(t, mocksdk)
+
+	mocksdk.EXPECT().DeleteNamespace(context.TODO(), &sd.DeleteNamespaceInput{Id: aws.String(test.NsId)}).
+		Return(&sd.DeleteNamespaceOutput{OperationId: aws.String(test.OpId1)}, nil)
+
+	opId, err := jApi.DeleteNamespace(context.TODO(), test.NsId)
+	assert.Nil(t, err, "No error for happy case")
+	assert.Equal(t, test.OpId1, opId)
+}
+
+func TestServiceDiscoveryJanitorApi_DeleteService_HappyCase(t *testing.T) {
+	mockController := gomock.NewController(t)
+	defer mockController.Finish()
+
+	mocksdk := janitor.NewMockSdkJanitorFacade(mockController)
+	jApi := getJanitorApi(t, mocksdk)
+
+	mocksdk.EXPECT().DeleteService(context.TODO(), &sd.DeleteServiceInput{Id: aws.String(test.SvcId)}).
+		Return(&sd.DeleteServiceOutput{}, nil)
+
+	err := jApi.DeleteService(context.TODO(), test.SvcId)
+	assert.Nil(t, err, "No error for happy case")
+}
+
+func getJanitorApi(t *testing.T, sdk *janitor.MockSdkJanitorFacade) ServiceDiscoveryJanitorApi {
+	return &serviceDiscoveryJanitorApi{
+		janitorFacade: sdk,
+	}
+}

integration/janitor/aws_facade.go

@@ -0,0 +1,30 @@
+package janitor
+
+import (
+	"context"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	sd "github.com/aws/aws-sdk-go-v2/service/servicediscovery"
+)
+
+// SdkJanitorFacade extends the minimal surface area of ServiceDiscovery API calls of the client
+// for integration test janitor operations.
+type SdkJanitorFacade interface {
+	// DeleteNamespace provides ServiceDiscovery DeleteNamespace wrapper interface.
+	DeleteNamespace(context.Context, *sd.DeleteNamespaceInput, ...func(*sd.Options)) (*sd.DeleteNamespaceOutput, error)
+
+	// DeleteService provides ServiceDiscovery DeleteService wrapper interface.
+	DeleteService(context.Context, *sd.DeleteServiceInput, ...func(*sd.Options)) (*sd.DeleteServiceOutput, error)
+
+	cloudmap.AwsFacade
+}
+
+type sdkJanitorFacade struct {
+	*sd.Client
+}
+
+// NewSdkJanitorFacadeFromConfig creates a new AWS facade from an AWS client config
+// extended for integration test janitor operations.
+func NewSdkJanitorFacadeFromConfig(cfg *aws.Config) SdkJanitorFacade {
+	return &sdkJanitorFacade{sd.NewFromConfig(*cfg)}
+}

integration/janitor/janitor.go

@@ -0,0 +1,109 @@
+package janitor
+
+import (
+	"context"
+	"fmt"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/cloudmap"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"os"
+)
+
+// CloudMapJanitor handles AWS Cloud Map resource cleanup during integration tests.
+type CloudMapJanitor interface {
+	// Cleanup removes all instances, services and the namespace from AWS Cloud Map for a given namespace name.
+	Cleanup(ctx context.Context, nsName string)
+}
+
+type cloudMapJanitor struct {
+	sdApi ServiceDiscoveryJanitorApi
+	fail  func()
+}
+
+// NewDefaultJanitor returns a new janitor object.
+func NewDefaultJanitor() CloudMapJanitor {
+	awsCfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(os.Getenv("AWS_REGION")),
+	)
+
+	if err != nil {
+		fmt.Printf("unable to configure AWS session: %s", err.Error())
+		os.Exit(1)
+	}
+
+	return &cloudMapJanitor{
+		sdApi: NewServiceDiscoveryJanitorApiFromConfig(&awsCfg),
+		fail:  func() { os.Exit(1) },
+	}
+}
+
+func (j *cloudMapJanitor) Cleanup(ctx context.Context, nsName string) {
+	fmt.Printf("Cleaning up all test resources in Cloud Map for namespace : %s\n", nsName)
+
+	nsList, err := j.sdApi.ListNamespaces(ctx)
+	j.checkOrFail(err, "", "could not find namespace to clean")
+
+	var nsId string
+	for _, ns := range nsList {
+		if ns.Name == nsName {
+			nsId = ns.Id
+		}
+	}
+
+	if nsId == "" {
+		fmt.Println("namespace does not exist in account, nothing to clean")
+		return
+	}
+
+	fmt.Printf("found namespace to clean: %s\n", nsId)
+
+	svcs, err := j.sdApi.ListServices(ctx, nsId)
+	j.checkOrFail(err,
+		fmt.Sprintf("namespace has %d services to clean", len(svcs)),
+		"could not find services to clean")
+
+	for _, svc := range svcs {
+		fmt.Printf("found service to clean: %s\n", svc.Id)
+		j.deregisterInstances(ctx, svc.Id)
+
+		delSvcErr := j.sdApi.DeleteService(ctx, svc.Id)
+		j.checkOrFail(delSvcErr, "service deleted", "could not cleanup service")
+	}
+
+	opId, err := j.sdApi.DeleteNamespace(ctx, nsId)
+	if err == nil {
+		fmt.Println("namespace delete in progress")
+		_, err = j.sdApi.PollNamespaceOperation(ctx, opId)
+	}
+	j.checkOrFail(err, "clean up successful", "could not cleanup namespace")
+}
+
+func (j *cloudMapJanitor) deregisterInstances(ctx context.Context, svcId string) {
+	insts, err := j.sdApi.ListInstances(ctx, svcId)
+	j.checkOrFail(err,
+		fmt.Sprintf("service has %d instances to clean", len(insts)),
+		"could not list instances to cleanup")
+
+	opColl := cloudmap.NewOperationCollector()
+	for _, inst := range insts {
+		instId := aws.ToString(inst.Id)
+		fmt.Printf("found instance to clean: %s\n", instId)
+		opColl.Add(func() (opId string, err error) {
+			return j.sdApi.DeregisterInstance(ctx, svcId, instId)
+		})
+	}
+
+	opErr := cloudmap.NewDeregisterInstancePoller(j.sdApi, svcId, opColl.Collect(), opColl.GetStartTime()).Poll(ctx)
+	j.checkOrFail(opErr, "instances de-registered", "could not cleanup instances")
+}
+
+func (j *cloudMapJanitor) checkOrFail(err error, successMsg string, failMsg string) {
+	if err != nil {
+		fmt.Printf("%s: %s\n", failMsg, err.Error())
+		j.fail()
+	}
+
+	if successMsg != "" {
+		fmt.Println(successMsg)
+	}
+}

integration/janitor/janitor_test.go

@@ -0,0 +1,76 @@
+package janitor
+
+import (
+	"context"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/mocks/integration/janitor"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/pkg/model"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/test"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/servicediscovery/types"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+type testJanitor struct {
+	janitor *cloudMapJanitor
+	mockApi *janitor.MockServiceDiscoveryJanitorApi
+	failed  *bool
+	close   func()
+}
+
+func TestNewDefaultJanitor(t *testing.T) {
+	assert.NotNil(t, NewDefaultJanitor())
+}
+
+func TestCleanupHappyCase(t *testing.T) {
+	tj := getTestJanitor(t)
+	defer tj.close()
+
+	tj.mockApi.EXPECT().ListNamespaces(context.TODO()).
+		Return([]*model.Namespace{{Id: test.NsId, Name: test.NsName}}, nil)
+	tj.mockApi.EXPECT().ListServices(context.TODO(), test.NsId).
+		Return([]*model.Resource{{Id: test.SvcId, Name: test.SvcName}}, nil)
+	tj.mockApi.EXPECT().ListInstances(context.TODO(), test.SvcId).
+		Return([]types.InstanceSummary{{Id: aws.String(test.EndptId1)}}, nil)
+
+	tj.mockApi.EXPECT().DeregisterInstance(context.TODO(), test.SvcId, test.EndptId1).
+		Return(test.OpId1, nil)
+	tj.mockApi.EXPECT().ListOperations(context.TODO(), gomock.Any()).
+		Return(map[string]types.OperationStatus{test.OpId1: types.OperationStatusSuccess}, nil)
+	tj.mockApi.EXPECT().DeleteService(context.TODO(), test.SvcId).
+		Return(nil)
+	tj.mockApi.EXPECT().DeleteNamespace(context.TODO(), test.NsId).
+		Return(test.OpId2, nil)
+	tj.mockApi.EXPECT().PollNamespaceOperation(context.TODO(), test.OpId2).
+		Return(test.NsId, nil)
+
+	tj.janitor.Cleanup(context.TODO(), test.NsName)
+	assert.False(t, *tj.failed)
+}
+
+func TestCleanupNothingToClean(t *testing.T) {
+	tj := getTestJanitor(t)
+	defer tj.close()
+
+	tj.mockApi.EXPECT().ListNamespaces(context.TODO()).
+		Return([]*model.Namespace{}, nil)
+
+	tj.janitor.Cleanup(context.TODO(), test.NsName)
+	assert.False(t, *tj.failed)
+}
+
+func getTestJanitor(t *testing.T) *testJanitor {
+	mockController := gomock.NewController(t)
+	api := janitor.NewMockServiceDiscoveryJanitorApi(mockController)
+	failed := false
+	return &testJanitor{
+		janitor: &cloudMapJanitor{
+			sdApi: api,
+			fail:  func() { failed = true },
+		},
+		mockApi: api,
+		failed:  &failed,
+		close:   func() { mockController.Finish() },
+	}
+}

integration/janitor/runner/main.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"context"
+	"github.com/aws/aws-cloud-map-mcs-controller-for-k8s/integration/janitor"
+)
+
+const (
+	e2eNs = "aws-cloud-map-mcs-e2e"
+)
+
+func main() {
+	j := janitor.NewDefaultJanitor()
+	j.Cleanup(context.TODO(), e2eNs)
+}