refinement of e2e tests (#43)

jaypipes · web-flow · commit f87a283093c1 · 2021-11-01T08:49:32.000-07:00
This PR includes two patches that provide enhancements and
simplifications to the e2e testing for the RDS controller.

The first enhancement is to separate the Kubernetes secret creation
and management into a pytest fixture that is importable from the
test modules.

The second enhancement pulls the waiter and getter functionality for
DB clusters out of the test module and into a separate importable module.

This will be important for later patches where I need to test *both* DB
cluster and DB instances in the same file and don't want to copy/paste a
bunch of boilerplate waiter/getter code between test modules.

By submitting this pull request, I confirm that my contribution is made under
the terms of the Apache 2.0 license.
diff --git a/test/e2e/db_cluster.py b/test/e2e/db_cluster.py
@@ -0,0 +1,124 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+#	 http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+"""Utilities for working with DB cluster resources"""
+
+import datetime
+import time
+import typing
+
+import boto3
+import pytest
+
+DEFAULT_WAIT_UNTIL_TIMEOUT_SECONDS = 600
+DEFAULT_WAIT_UNTIL_INTERVAL_SECONDS = 15
+DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS = 600
+DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS = 15
+
+ClusterMatchFunc = typing.NewType(
+    'ClusterMatchFunc',
+    typing.Callable[[dict], bool],
+)
+
+class StatusMatcher:
+    def __init__(self, status):
+        self.match_on = status
+
+    def __call__(self, record: dict) -> bool:
+        return 'Status' in record and record['Status'] == self.match_on
+
+
+def status_matches(status: str) -> ClusterMatchFunc:
+    return StatusMatcher(status)
+
+
+def wait_until(
+        db_cluster_id: str,
+        match_fn: ClusterMatchFunc,
+        timeout_seconds: int = DEFAULT_WAIT_UNTIL_TIMEOUT_SECONDS,
+        interval_seconds: int = DEFAULT_WAIT_UNTIL_INTERVAL_SECONDS,
+    ) -> None:
+    """Waits until a DB cluster with a supplied ID is returned from the RDS API
+    and the matching functor returns True.
+
+    Usage:
+        from e2e.db_cluster import wait_until, status_matches
+
+        wait_until(
+            cluster_id,
+            status_matches("available"),
+        )
+
+    Raises:
+        pytest.fail upon timeout
+    """
+    now = datetime.datetime.now()
+    timeout = now + datetime.timedelta(seconds=timeout_seconds)
+
+    while not match_fn(get(db_cluster_id)):
+        if datetime.datetime.now() >= timeout:
+            pytest.fail("failed to match DBCluster before timeout")
+        time.sleep(interval_seconds)
+
+
+def wait_until_deleted(
+        db_cluster_id: str,
+        timeout_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS,
+        interval_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS,
+    ) -> None:
+    """Waits until a DB cluster with a supplied ID is no longer returned from
+    the RDS API.
+
+    Usage:
+        from e2e.db_cluster import wait_until_deleted
+
+        wait_until_deleted(cluster_id)
+
+    Raises:
+        pytest.fail upon timeout or if the DB cluster goes to any other status
+        other than 'deleting'
+    """
+    now = datetime.datetime.now()
+    timeout = now + datetime.timedelta(seconds=timeout_seconds)
+
+    while True:
+        if datetime.datetime.now() >= timeout:
+            pytest.fail(
+                "Timed out waiting for DB cluster to be "
+                "deleted in RDS API"
+            )
+        time.sleep(interval_seconds)
+
+        latest = get(db_cluster_id)
+        if latest is None:
+            break
+
+        if latest['Status'] != "deleting":
+            pytest.fail(
+                "Status is not 'deleting' for DB cluster that was "
+                "deleted. Status is " + latest['Status']
+            )
+
+
+def get(db_cluster_id):
+    """Returns a dict containing the DB cluster record from the RDS API.
+
+    If no such DB cluster exists, returns None.
+    """
+    c = boto3.client('rds')
+    try:
+        resp = c.describe_db_clusters(DBClusterIdentifier=db_cluster_id)
+        assert len(resp['DBClusters']) == 1
+        return resp['DBClusters'][0]
+    except c.exceptions.DBClusterNotFoundFault:
+        return None
diff --git a/test/e2e/fixtures.py b/test/e2e/fixtures.py
@@ -0,0 +1,59 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+#	 http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+"""Fixtures common to all RDS controller tests"""
+
+import dataclasses
+
+from acktest.k8s import resource as k8s
+import boto3
+import pytest
+
+
+@pytest.fixture(scope="module")
+def rds_client():
+    return boto3.client('rds')
+
+
+@dataclasses.dataclass
+class SecretKeyReference:
+    ns: str
+    name: str
+    key: str
+    val: str
+
+
+@pytest.fixture(scope="module")
+def k8s_secret():
+    """Manages the lifecycle of a Kubernetes Secret for use in tests.
+
+    Usage:
+        from e2e.fixtures import k8s_secret
+
+        class TestThing:
+            def test_thing(self, k8s_secret):
+                secret = k8s_secret(
+                    "default", "mysecret", "mykey", "myval",
+                )
+    """
+    created = []
+    def _k8s_secret(ns, name, key, val):
+        k8s.create_opaque_secret(ns, name, key, val)
+        secret_ref = SecretKeyReference(ns, name, key, val)
+        created.append(secret_ref)
+        return secret_ref
+
+    yield _k8s_secret
+
+    for secret_ref in created:
+        k8s.delete_secret(secret_ref.ns, secret_ref.name)
diff --git a/test/e2e/tests/test_db_cluster.py b/test/e2e/tests/test_db_cluster.py
@@ -14,69 +14,58 @@
 """Integration tests for the RDS API DBCluster resource
 """
 
-import boto3
-import datetime
-import logging
 import time
-from typing import Dict
 
 import pytest
 
 from acktest.k8s import resource as k8s
 from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_rds_resource
 from e2e.replacement_values import REPLACEMENT_VALUES
 from e2e.bootstrap_resources import get_bootstrap_resources
+from e2e.fixtures import k8s_secret
+from e2e import db_cluster
 
 RESOURCE_PLURAL = 'dbclusters'
 
-DELETE_WAIT_INTERVAL_SLEEP_SECONDS = 15
 DELETE_WAIT_AFTER_SECONDS = 120
-DELETE_TIMEOUT_SECONDS = 600
 
-CREATE_INTERVAL_SLEEP_SECONDS = 15
-CREATE_TIMEOUT_SECONDS = 600
 # Time we wait after resource becoming available in RDS and checking the CR's
 # Status has been updated
-CHECK_STATUS_WAIT_SECONDS = 10
+CHECK_STATUS_WAIT_SECONDS = 20
 
 MODIFY_WAIT_AFTER_SECONDS = 20
 
 
-@pytest.fixture(scope="module")
-def rds_client():
-    return boto3.client('rds')
-
-
-@pytest.fixture(scope="module")
-def master_user_pass_secret():
-    ns = "default"
-    name = "dbclustersecrets"
-    key = "master_user_password"
-    secret_val = "secretpass123456"
-    k8s.create_opaque_secret(ns, name, key, secret_val)
-    yield ns, name, key
-    k8s.delete_secret(ns, name)
-
-
 @service_marker
 @pytest.mark.canary
 class TestDBCluster:
+
+    # MUP == Master user password...
+    MUP_NS = "default"
+    MUP_SEC_NAME = "dbclustersecrets"
+    MUP_SEC_KEY = "master_user_password"
+    MUP_SEC_VAL = "secretpass123456"
+
     def test_create_delete_mysql_serverless(
             self,
-            rds_client,
-            master_user_pass_secret,
+            k8s_secret,
     ):
         db_cluster_id = "my-aurora-mysql"
         db_name = "mydb"
-        mup_sec_ns, mup_sec_name, mup_sec_key = master_user_pass_secret
+        secret = k8s_secret(
+            self.MUP_NS,
+            self.MUP_SEC_NAME,
+            self.MUP_SEC_KEY,
+            self.MUP_SEC_VAL,
+        )
 
         replacements = REPLACEMENT_VALUES.copy()
         replacements['COPY_TAGS_TO_SNAPSHOT'] = "False"
         replacements["DB_CLUSTER_ID"] = db_cluster_id
         replacements["DB_NAME"] = db_name
-        replacements["MASTER_USER_PASS_SECRET_NAMESPACE"] = mup_sec_ns
-        replacements["MASTER_USER_PASS_SECRET_NAME"] = mup_sec_name
-        replacements["MASTER_USER_PASS_SECRET_KEY"] = mup_sec_key
+        replacements["MASTER_USER_PASS_SECRET_NAMESPACE"] = secret.ns
+        replacements["MASTER_USER_PASS_SECRET_NAME"] = secret.name
+        replacements["MASTER_USER_PASS_SECRET_KEY"] = secret.key
 
         resource_data = load_rds_resource(
             "db_cluster_mysql_serverless",
@@ -95,23 +84,10 @@ def test_create_delete_mysql_serverless(
         assert 'status' in cr['status']
         assert cr['status']['status'] == 'creating'
 
-        # Let's check that the DB cluster appears in RDS
-        aws_res = rds_client.describe_db_clusters(DBClusterIdentifier=db_cluster_id)
-        assert aws_res is not None
-        assert len(aws_res['DBClusters']) == 1
-        dbc_rec = aws_res['DBClusters'][0]
-
-        now = datetime.datetime.now()
-        timeout = now + datetime.timedelta(seconds=CREATE_TIMEOUT_SECONDS)
-
-        # TODO(jaypipes): Move this into generic AWS-side waiter
-        while dbc_rec['Status'] != "available":
-            if datetime.datetime.now() >= timeout:
-                pytest.fail("failed to find available DBCluster before timeout")
-            time.sleep(CREATE_INTERVAL_SLEEP_SECONDS)
-            aws_res = rds_client.describe_db_clusters(DBClusterIdentifier=db_cluster_id)
-            assert len(aws_res['DBClusters']) == 1
-            dbc_rec = aws_res['DBClusters'][0]
+        db_cluster.wait_until(
+            db_cluster_id,
+            db_cluster.status_matches('available'),
+        )
 
         time.sleep(CHECK_STATUS_WAIT_SECONDS)
 
@@ -132,38 +108,22 @@ def test_create_delete_mysql_serverless(
         # We're now going to modify the CopyTagsToSnapshot field of the DB
         # instance, wait some time and verify that the RDS server-side resource
         # shows the new value of the field.
-        assert dbc_rec['CopyTagsToSnapshot'] == False
+        latest = db_cluster.get(db_cluster_id)
+        assert latest is not None
+        assert latest['CopyTagsToSnapshot'] == False
         updates = {
             "spec": {"copyTagsToSnapshot": True},
         }
         k8s.patch_custom_resource(ref, updates)
         time.sleep(MODIFY_WAIT_AFTER_SECONDS)
 
-        aws_res = rds_client.describe_db_clusters(DBClusterIdentifier=db_cluster_id)
-        assert aws_res is not None
-        assert len(aws_res['DBClusters']) == 1
-        dbc_rec = aws_res['DBClusters'][0]
-        assert dbc_rec['CopyTagsToSnapshot'] == True
+        latest = db_cluster.get(db_cluster_id)
+        assert latest is not None
+        assert latest['CopyTagsToSnapshot'] == True
 
         # Delete the k8s resource on teardown of the module
         k8s.delete_custom_resource(ref)
 
         time.sleep(DELETE_WAIT_AFTER_SECONDS)
 
-        now = datetime.datetime.now()
-        timeout = now + datetime.timedelta(seconds=DELETE_TIMEOUT_SECONDS)
-
-        # DB instance should no longer appear in RDS
-        while True:
-            if datetime.datetime.now() >= timeout:
-                pytest.fail("Timed out waiting for DB cluster to being deleted in RDS API")
-            time.sleep(DELETE_WAIT_INTERVAL_SLEEP_SECONDS)
-
-            try:
-                aws_res = rds_client.describe_db_clusters(DBClusterIdentifier=db_cluster_id)
-                assert len(aws_res['DBClusters']) == 1
-                dbc_rec = aws_res['DBClusters'][0]
-                if dbc_rec['Status'] != "deleting":
-                    pytest.fail("Status is not 'deleting' for DB cluster that was deleted. Status is "+dbc_rec['Status'])
-            except rds_client.exceptions.DBClusterNotFoundFault:
-                break
+        db_cluster.wait_until_deleted(db_cluster_id)
diff --git a/test/e2e/tests/test_db_instance.py b/test/e2e/tests/test_db_instance.py
