Add deletion policy CLI arguments and annotations (#107)

Issue #, if available: aws-controllers-k8s/community#82

Description of changes:
As per the proposal: aws-controllers-k8s/community#1148

Adds support for a new optional CLI argument for the controller binary `--deletion-policy`, supporting `delete` and `retain` as available options. Setting this argument to `delete` leaves the controller as the current default behaviour, whereby it deletes the AWS resource under management before it is removed from the K8s API server. Setting this argument to `retain` modifies the default behaviour, leaving the AWS resources intact (taking no action on them) before removing it from the K8s API server.

Adds support for a new annotation on `Namespace` objects: `{service}.services.k8s.aws/deletion-policy: delete/retain` (where `{service}` is the service alias of the controller eg. `s3`). This annotation overrides the deletion policy behaviour configured for the controller for all resources deployed under the namespace.

Adds support for a new annotation on any ACK resource object: `services.k8s.aws/deletion-policy: delete/retain`. This annotation overrides only the deletion policy behaviour configured for the resource.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: RedbackThomson

apis/core/v1alpha1/annotations.go

@@ -59,4 +59,11 @@ const (
 	// will either use the default behavior	of aws-sdk-go to create endpoints or
 	// aws-endpoint-url if it is set in controller binary flags and environment variables.
 	AnnotationEndpointURL = AnnotationPrefix + "endpoint-url"
+	// AnnotationDeletionPolicy is an annotation whose value is the identifier for the
+	// the deletion policy for the current resource. If this annotation is set
+	// to "delete" the resource manager will delete the AWS resource when the
+	// K8s resource is deleted. If this annotation is set to "retain" the
+	// resource manager will leave the AWS resource intact when the K8s resource
+	// is deleted.
+	AnnotationDeletionPolicy = AnnotationPrefix + "deletion-policy"
 )

apis/core/v1alpha1/deletion_policy.go

@@ -0,0 +1,47 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package v1alpha1
+
+import (
+	"fmt"
+)
+
+// DeletionPolicy represents how the ACK reconciler will handle the deletion of
+// a resource. A DeletionPolicy of "delete" will delete the underlying AWS
+// resource, whereas a DeletionPolicy of "retain" will only delete the K8s
+// object leaving the AWS resource intact.
+type DeletionPolicy string
+
+const (
+	DeletionPolicyDelete DeletionPolicy = "delete"
+	DeletionPolicyRetain DeletionPolicy = "retain"
+)
+
+func (e *DeletionPolicy) String() string {
+	return string(*e)
+}
+
+func (e *DeletionPolicy) Set(v string) error {
+	switch v {
+	case string(DeletionPolicyDelete), string(DeletionPolicyRetain):
+		*e = DeletionPolicy(v)
+		return nil
+	default:
+		return fmt.Errorf("invalid DeletionPolicy value: %s", v)
+	}
+}
+
+func (e *DeletionPolicy) Type() string {
+	return "DeletionPolicy"
+}

pkg/config/config.go

@@ -27,6 +27,7 @@ import (
 	ctrlrt "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
+	ackv1alpha1 "github.com/aws-controllers-k8s/runtime/apis/core/v1alpha1"
 	acktags "github.com/aws-controllers-k8s/runtime/pkg/tags"
 )
 
@@ -43,6 +44,7 @@ const (
 	flagWatchNamespace         = "watch-namespace"
 	flagEnableWebhookServer    = "enable-webhook-server"
 	flagWebhookServerAddr      = "webhook-server-addr"
+	flagDeletionPolicy         = "deletion-policy"
 	envVarAWSRegion            = "AWS_REGION"
 )
 
@@ -74,6 +76,7 @@ type Config struct {
 	WatchNamespace           string
 	EnableWebhookServer      bool
 	WebhookServerAddr        string
+	DeletionPolicy           ackv1alpha1.DeletionPolicy
 }
 
 // BindFlags defines CLI/runtime configuration options
@@ -145,6 +148,10 @@ func (cfg *Config) BindFlags() {
 		"Specific namespace the service controller will watch for object creation from CRD. "+
 			" By default it will listen to all namespaces",
 	)
+	flag.Var(
+		&cfg.DeletionPolicy, flagDeletionPolicy,
+		"The default deletion policy for all resources managed by the controller",
+	)
 }
 
 // SetupLogger initializes the logger used in the service controller
@@ -222,6 +229,10 @@ func (cfg *Config) Validate() error {
 	if cfg.EnableWebhookServer && cfg.WebhookServerAddr == "" {
 		return errors.New("empty webhook server address")
 	}
+
+	if cfg.DeletionPolicy == "" {
+		cfg.DeletionPolicy = ackv1alpha1.DeletionPolicyDelete
+	}
 	return nil
 }
 

pkg/runtime/cache/namespace.go

@@ -14,6 +14,7 @@
 package cache
 
 import (
+	"strings"
 	"sync"
 
 	"github.com/go-logr/logr"
@@ -33,6 +34,8 @@ type namespaceInfo struct {
 	ownerAccountID string
 	// services.k8s.aws/endpoint-url Annotation
 	endpointURL string
+	// {service}.services.k8s.aws/deletion-policy Annotations (keyed by service)
+	deletionPolicies map[string]string
 }
 
 // getDefaultRegion returns the default region value
@@ -59,6 +62,17 @@ func (n *namespaceInfo) getEndpointURL() string {
 	return n.endpointURL
 }
 
+// getDeletionPolicy returns the namespace deletion policy for a given service
+func (n *namespaceInfo) getDeletionPolicy(service string) string {
+	if n == nil {
+		return ""
+	}
+	if val, exists := n.deletionPolicies[strings.ToLower(service)]; exists {
+		return val
+	}
+	return ""
+}
+
 // NamespaceCache is responsible of keeping track of namespaces
 // annotations, and caching those related to the ACK controller.
 type NamespaceCache struct {
@@ -150,6 +164,16 @@ func (c *NamespaceCache) GetEndpointURL(namespace string) (string, bool) {
 	return "", false
 }
 
+// GetDeletionPolicy returns the deletion policy if it exists
+func (c *NamespaceCache) GetDeletionPolicy(namespace string, service string) (string, bool) {
+	info, ok := c.getNamespaceInfo(namespace)
+	if ok {
+		e := info.getDeletionPolicy(service)
+		return e, e != ""
+	}
+	return "", false
+}
+
 // getNamespaceInfo reads a namespace cached annotations and
 // return a given namespace default aws region, owner account id and endpoint url.
 // This function is thread safe.
@@ -177,6 +201,17 @@ func (c *NamespaceCache) setNamespaceInfoFromK8sObject(ns *corev1.Namespace) {
 	if ok {
 		nsInfo.endpointURL = EndpointURL
 	}
+
+	nsInfo.deletionPolicies = map[string]string{}
+	nsDeletionPolicySuffix := "." + ackv1alpha1.AnnotationDeletionPolicy
+	for key, elem := range nsa {
+		if !strings.HasSuffix(key, nsDeletionPolicySuffix) {
+			continue
+		}
+		service := strings.TrimSuffix(key, nsDeletionPolicySuffix)
+		nsInfo.deletionPolicies[service] = elem
+	}
+
 	c.Lock()
 	defer c.Unlock()
 	c.namespaceInfos[ns.ObjectMeta.Name] = nsInfo

pkg/runtime/reconciler.go

@@ -194,10 +194,20 @@ func (r *resourceReconciler) reconcile(
 	res acktypes.AWSResource,
 ) (acktypes.AWSResource, error) {
 	if res.IsBeingDeleted() {
-		// Resolve references before deleting the resource.
-		// Ignore any errors while resolving the references
-		res, _ = rm.ResolveReferences(ctx, r.apiReader, res)
-		return r.deleteResource(ctx, rm, res)
+		// Determine whether we should retain or delete the resource
+		if r.getDeletionPolicy(res) == ackv1alpha1.DeletionPolicyDelete {
+			// Resolve references before deleting the resource.
+			// Ignore any errors while resolving the references
+			res, _ = rm.ResolveReferences(ctx, r.apiReader, res)
+			return r.deleteResource(ctx, rm, res)
+		}
+
+		rlog := ackrtlog.FromContext(ctx)
+		rlog.Info("AWS resource will not be deleted - deletion policy set to retain")
+		if err := r.setResourceUnmanaged(ctx, res); err != nil {
+			return res, err
+		}
+		return r.handleRequeues(ctx, res)
 	}
 	latest, err := r.Sync(ctx, rm, res)
 	if err != nil {
@@ -1015,9 +1025,9 @@ func (r *resourceReconciler) getRoleARN(
 //
 // If the resource has not yet been created, we look for the AWS region
 // in the following order of precedence:
-//  - The resource's `services.k8s.aws/region` annotation, if present
-//  - The resource's Namespace's `services.k8s.aws/region` annotation, if present
-//  - The controller's `--aws-region` CLI flag
+//   - The resource's `services.k8s.aws/region` annotation, if present
+//   - The resource's Namespace's `services.k8s.aws/region` annotation, if present
+//   - The controller's `--aws-region` CLI flag
 func (r *resourceReconciler) getRegion(
 	res acktypes.AWSResource,
 ) ackv1alpha1.AWSRegion {
@@ -1045,6 +1055,35 @@ func (r *resourceReconciler) getRegion(
 	return ackv1alpha1.AWSRegion(r.cfg.Region)
 }
 
+// getDeletionPolicy returns the resource's deletion policy based on the default
+// behaviour or any other overriding annotations.
+//
+// We look for the deletion policy in the annotations based on the following
+// precedence:
+//   - The resource's `services.k8s.aws/deletion-policy` annotation, if present
+//   - The resource's Namespace's `{service}.services.k8s.aws/deletion-policy` annotation, if present
+//   - The controller's `--deletion-policy` CLI flag
+func (r *resourceReconciler) getDeletionPolicy(
+	res acktypes.AWSResource,
+) ackv1alpha1.DeletionPolicy {
+	// look for deletion policy in CR metadata annotations
+	resAnnotations := res.MetaObject().GetAnnotations()
+	deletionPolicy, ok := resAnnotations[ackv1alpha1.AnnotationDeletionPolicy]
+	if ok {
+		return ackv1alpha1.DeletionPolicy(deletionPolicy)
+	}
+
+	// look for default deletion policy in namespace metadata annotations
+	ns := res.MetaObject().GetNamespace()
+	deletionPolicy, ok = r.cache.Namespaces.GetDeletionPolicy(ns, r.sc.GetMetadata().ServiceAlias)
+	if ok {
+		return ackv1alpha1.DeletionPolicy(deletionPolicy)
+	}
+
+	// use controller configuration policy
+	return r.cfg.DeletionPolicy
+}
+
 // getEndpointURL returns the AWS account that owns the supplied resource.
 // We look for the namespace associated endpoint url, if that is set we use it.
 // Otherwise if none of these annotations are set we use the endpoint url specified