Add e2e test for rdsproxy (#115)

Issue #, if available: aws-controllers-k8s/community#1440

Description of changes:

- Add e2e test for rds `dbproxy` resource

Two major issues were observed while testing locally

1. This test needs a complex dependency chain and it will cause bootstrap logic pretty complex. I'm inclined to either hard code it for our rds tests account or we just remove it for now.  The dependency chain is like this :  create one base rds instance -> create one aws secret service for this rds instance -> create an iam role for this secret -> create rds proxy using iam role & secret . Deletion is the opposite way.  Currently we're not able to create a rds instance in bootstrap logic or create a aws secret service secret. 
2. The e2e test lasts pretty long. creation will take around 10 min, deletion will take around 15 min, the total time would be around 30min in total which will greatly increase e2e test time. 

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: brucegucode

apis/v1alpha1/ack-generate-metadata.yaml

@@ -1,7 +1,7 @@
 ack_generate_info:
-  build_date: "2022-08-13T17:21:53Z"
+  build_date: "2022-08-22T20:21:30Z"
   build_hash: fe61d04673fd4d9848d5f726b01e0689a16d3733
-  go_version: go1.18.1
+  go_version: go1.18.2
   version: v0.19.3-1-gfe61d04
 api_directory_checksum: 6a967cc8a62d521d4f4816dbccc48f81d0cb271d
 api_version: v1alpha1

pkg/resource/db_proxy/hooks.go

@@ -0,0 +1,151 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package db_proxy
+
+import (
+	"context"
+
+	ackcompare "github.com/aws-controllers-k8s/runtime/pkg/compare"
+	ackrtlog "github.com/aws-controllers-k8s/runtime/pkg/runtime/log"
+
+	svcapitypes "github.com/aws-controllers-k8s/rds-controller/apis/v1alpha1"
+	svcsdk "github.com/aws/aws-sdk-go/service/rds"
+
+	"github.com/aws-controllers-k8s/rds-controller/pkg/util"
+)
+
+// syncTags keeps the resource's tags in sync
+//
+// NOTE(jaypipes): RDS' Tagging APIs differ from other AWS APIs in the
+// following ways:
+//
+// 1. The names of the tagging API operations are different. Other APIs use the
+//    Tagris `ListTagsForResource`, `TagResource` and `UntagResource` API
+//    calls. RDS uses `ListTagsForResource`, `AddTagsToResource` and
+//    `RemoveTagsFromResource`.
+//
+// 2. Even though the name of the `ListTagsForResource` API call is the same,
+//    the structure of the input and the output are different from other APIs.
+//    For the input, instead of a `ResourceArn` field, RDS names the field
+//    `ResourceName`, but actually expects an ARN, not the proxy
+//    name.  This is the same for the `AddTagsToResource` and
+//    `RemoveTagsFromResource` input shapes. For the output shape, the field is
+//    called `TagList` instead of `Tags` but is otherwise the same struct with
+//    a `Key` and `Value` member field.
+func (rm *resourceManager) syncTags(
+	ctx context.Context,
+	desired *resource,
+	latest *resource,
+) (err error) {
+	rlog := ackrtlog.FromContext(ctx)
+	exit := rlog.Trace("rm.syncTags")
+	defer func() { exit(err) }()
+
+	arn := (*string)(latest.ko.Status.ACKResourceMetadata.ARN)
+
+	toAdd, toDelete := util.ComputeTagsDelta(
+		desired.ko.Spec.Tags, latest.ko.Spec.Tags,
+	)
+
+	if len(toDelete) > 0 {
+		rlog.Debug("removing tags from proxy", "tags", toDelete)
+		_, err = rm.sdkapi.RemoveTagsFromResourceWithContext(
+			ctx,
+			&svcsdk.RemoveTagsFromResourceInput{
+				ResourceName: arn,
+				TagKeys:      toDelete,
+			},
+		)
+		rm.metrics.RecordAPICall("UPDATE", "RemoveTagsFromResource", err)
+		if err != nil {
+			return err
+		}
+	}
+
+	// NOTE(jaypipes): According to the RDS API documentation, adding a tag
+	// with a new value overwrites any existing tag with the same key. So, we
+	// don't need to do anything to "update" a Tag. Simply including it in the
+	// AddTagsToResource call is enough.
+	if len(toAdd) > 0 {
+		rlog.Debug("adding tags to proxy", "tags", toAdd)
+		_, err = rm.sdkapi.AddTagsToResourceWithContext(
+			ctx,
+			&svcsdk.AddTagsToResourceInput{
+				ResourceName: arn,
+				Tags:         sdkTagsFromResourceTags(toAdd),
+			},
+		)
+		rm.metrics.RecordAPICall("UPDATE", "AddTagsToResource", err)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// getTags retrieves the resource's associated tags
+func (rm *resourceManager) getTags(
+	ctx context.Context,
+	resourceARN string,
+) ([]*svcapitypes.Tag, error) {
+	resp, err := rm.sdkapi.ListTagsForResourceWithContext(
+		ctx,
+		&svcsdk.ListTagsForResourceInput{
+			ResourceName: &resourceARN,
+		},
+	)
+	rm.metrics.RecordAPICall("GET", "ListTagsForResource", err)
+	if err != nil {
+		return nil, err
+	}
+	tags := make([]*svcapitypes.Tag, 0, len(resp.TagList))
+	for _, tag := range resp.TagList {
+		tags = append(tags, &svcapitypes.Tag{
+			Key:   tag.Key,
+			Value: tag.Value,
+		})
+	}
+	return tags, nil
+}
+
+// compareTags adds a difference to the delta if the supplied resources have
+// different tag collections
+func compareTags(
+	delta *ackcompare.Delta,
+	a *resource,
+	b *resource,
+) {
+	if len(a.ko.Spec.Tags) != len(b.ko.Spec.Tags) {
+		delta.Add("Spec.Tags", a.ko.Spec.Tags, b.ko.Spec.Tags)
+	} else if len(a.ko.Spec.Tags) > 0 {
+		if !util.EqualTags(a.ko.Spec.Tags, b.ko.Spec.Tags) {
+			delta.Add("Spec.Tags", a.ko.Spec.Tags, b.ko.Spec.Tags)
+		}
+	}
+}
+
+// sdkTagsFromResourceTags transforms a *svcapitypes.Tag array to a *svcsdk.Tag
+// array.
+func sdkTagsFromResourceTags(
+	rTags []*svcapitypes.Tag,
+) []*svcsdk.Tag {
+	tags := make([]*svcsdk.Tag, len(rTags))
+	for i := range rTags {
+		tags[i] = &svcsdk.Tag{
+			Key:   rTags[i].Key,
+			Value: rTags[i].Value,
+		}
+	}
+	return tags
+}

test/e2e/bootstrap_resources.py

@@ -17,13 +17,15 @@
 
 from dataclasses import dataclass
 from acktest.bootstrapping.vpc import VPC
+from acktest.bootstrapping.iam import Role
 from acktest.bootstrapping import Resources
 from e2e import bootstrap_directory
 
 
 @dataclass
 class BootstrapResources(Resources):
     ClusterVPC: VPC
+    RDSProxyRole: Role
 
 _bootstrap_resources = None
 

test/e2e/db_proxy.py

@@ -0,0 +1,139 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+#	 http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+"""Utilities for working with DB proxy resources"""
+
+import datetime
+import time
+import typing
+
+import boto3
+import pytest
+
+DEFAULT_WAIT_UNTIL_TIMEOUT_SECONDS = 60*10
+DEFAULT_WAIT_UNTIL_INTERVAL_SECONDS = 15
+DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS = 60*10
+DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS = 15
+
+ProxyMatchFunc = typing.NewType(
+    'ProxyMatchFunc',
+    typing.Callable[[dict], bool],
+)
+
+class StatusMatcher:
+    def __init__(self, status):
+        self.match_on = status
+
+    def __call__(self, record: dict) -> bool:
+        return 'Status' in record and record['Status'] == self.match_on
+
+
+def status_matches(status: str) -> ProxyMatchFunc:
+    return StatusMatcher(status)
+
+
+def wait_until(
+        db_proxy_id: str,
+        match_fn: ProxyMatchFunc,
+        timeout_seconds: int = DEFAULT_WAIT_UNTIL_TIMEOUT_SECONDS,
+        interval_seconds: int = DEFAULT_WAIT_UNTIL_INTERVAL_SECONDS,
+    ) -> None:
+    """Waits until a DB proxy with a supplied ID is returned from the RDS API
+    and the matching functor returns True.
+
+    Usage:
+        from e2e.db_proxy import wait_until, status_matches
+
+        wait_until(
+            proxy_id,
+            status_matches("available"),
+        )
+
+    Raises:
+        pytest.fail upon timeout
+    """
+    now = datetime.datetime.now()
+    timeout = now + datetime.timedelta(seconds=timeout_seconds)
+
+    while not match_fn(get(db_proxy_id)):
+        if datetime.datetime.now() >= timeout:
+            pytest.fail("failed to match DBProxy before timeout")
+        time.sleep(interval_seconds)
+
+
+def wait_until_deleted(
+        db_proxy_id: str,
+        timeout_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS,
+        interval_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS,
+    ) -> None:
+    """Waits until a DB proxy with a supplied ID is no longer returned from
+    the RDS API.
+
+    Usage:
+        from e2e.db_proxy import wait_until_deleted
+
+        wait_until_deleted(proxy_id)
+
+    Raises:
+        pytest.fail upon timeout or if the DB proxy goes to any other status
+        other than 'deleting'
+    """
+    now = datetime.datetime.now()
+    timeout = now + datetime.timedelta(seconds=timeout_seconds)
+
+    while True:
+        if datetime.datetime.now() >= timeout:
+            pytest.fail(
+                "Timed out waiting for DB proxy to be "
+                "deleted in RDS API"
+            )
+        time.sleep(interval_seconds)
+
+        latest = get(db_proxy_id)
+        if latest is None:
+            break
+
+        if latest['Status'] != "deleting":
+            pytest.fail(
+                "Status is not 'deleting' for DB proxy that was "
+                "deleted. Status is " + latest['Status']
+            )
+
+
+def get(db_proxy_id):
+    """Returns a dict containing the DB proxy record from the RDS API.
+
+    If no such DB proxy exists, returns None.
+    """
+    c = boto3.client('rds')
+    try:
+        resp = c.describe_db_proxies(DBProxyName=db_proxy_id)
+        assert len(resp['DBProxies']) == 1
+        return resp['DBProxies'][0]
+    except c.exceptions.DBProxyNotFoundFault:
+        return None
+
+
+def get_tags(db_proxy_arn):
+    """Returns a dict containing the DB proxy's tag records from the RDS API.
+
+    If no such DB proxy exists, returns None.
+    """
+    c = boto3.client('rds')
+    try:
+        resp = c.list_tags_for_resource(
+            ResourceName=db_proxy_arn,
+        )
+        return resp['TagList']
+    except c.exceptions.DBProxyNotFoundFault:
+        return None

test/e2e/resources/db_proxy.yaml

@@ -0,0 +1,16 @@
+apiVersion: rds.services.k8s.aws/v1alpha1
+kind: DBProxy
+metadata:
+  name: $DB_PROXY_NAME
+spec:
+  name: $DB_PROXY_NAME
+  engineFamily: $DB_PROXY_ENGINE_FAMILY
+  roleARN: $IAM_ROLE_ARN
+  auth:
+  - secretARN: $SECRET_ARN
+    authScheme: SECRETS
+    iamAuth: DISABLED
+    description: $DESCRIPTION
+  vpcSubnetIDs: 
+  - $PUBLIC_SUBNET_1
+  - $PUBLIC_SUBNET_2

test/e2e/service_bootstrap.py

@@ -18,6 +18,7 @@
 from e2e import bootstrap_directory
 from acktest.bootstrapping import Resources, BootstrapFailureException
 from acktest.bootstrapping.vpc import VPC
+from acktest.bootstrapping.iam import Role
 from e2e.bootstrap_resources import BootstrapResources
 
 
@@ -26,6 +27,7 @@ def service_bootstrap() -> Resources:
 
     resources = BootstrapResources(
         ClusterVPC=VPC(name_prefix="cluster-vpc", num_public_subnet=2, num_private_subnet=2),
+        RDSProxyRole=Role("rds-proxy-role", "rds.amazonaws.com", managed_policies=["arn:aws:iam::aws:policy/SecretsManagerReadWrite"])
     )
 
     try:
@@ -38,4 +40,4 @@ def service_bootstrap() -> Resources:
 if __name__ == "__main__":
     config = service_bootstrap()
     # Write config to current directory by default
-    config.serialize(bootstrap_directory)
+    config.serialize(bootstrap_directory)