Use pytest fixtures in e2e tests (#140)

Updates e2e test cases for the RDS controller to use pytest fixtures.

Issue aws-controllers-k8s/community#1726

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: jaypipes

test/e2e/db_cluster_parameter_group.py

@@ -13,7 +13,46 @@
 
 """Utilities for working with DB cluster parameter group resources"""
 
+import datetime
+import time
+
 import boto3
+import pytest
+
+DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS = 60*10
+DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS = 15
+
+
+def wait_until_deleted(
+        cpg_name: str,
+        timeout_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS,
+        interval_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS,
+    ) -> None:
+    """Waits until a DB cluster param group with a supplied ID is no longer
+    returned from the RDS API.
+
+    Usage:
+        from e2e.db_cluster_parameter_group import wait_until_deleted
+
+        wait_until_deleted(cpg_name)
+
+    Raises:
+        pytest.fail upon timeout
+    """
+    now = datetime.datetime.now()
+    timeout = now + datetime.timedelta(seconds=timeout_seconds)
+
+    while True:
+        if datetime.datetime.now() >= timeout:
+            pytest.fail(
+                "Timed out waiting for DB cluster param group to be "
+                "deleted in RDS API"
+            )
+        time.sleep(interval_seconds)
+
+        latest = get(cpg_name)
+        if latest is None:
+            break
 
 
 def get(db_cluster_parameter_group_name):

test/e2e/db_parameter_group.py

@@ -13,7 +13,47 @@
 
 """Utilities for working with DB parameter group resources"""
 
+import datetime
+import time
+
 import boto3
+import pytest
+
+DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS = 60*10
+DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS = 15
+
+
+def wait_until_deleted(
+        pg_name: str,
+        timeout_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS,
+        interval_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS,
+    ) -> None:
+    """Waits until a DB param group with a supplied ID is no longer returned
+    from the RDS API.
+
+    Usage:
+        from e2e.db_parameter_group import wait_until_deleted
+
+        wait_until_deleted(pg_name)
+
+    Raises:
+        pytest.fail upon timeout or if the DB param group goes to any other
+        status other than 'deleting'
+    """
+    now = datetime.datetime.now()
+    timeout = now + datetime.timedelta(seconds=timeout_seconds)
+
+    while True:
+        if datetime.datetime.now() >= timeout:
+            pytest.fail(
+                "Timed out waiting for DB param group to be "
+                "deleted in RDS API"
+            )
+        time.sleep(interval_seconds)
+
+        latest = get(pg_name)
+        if latest is None:
+            break
 
 
 def get(db_parameter_group_name):
@@ -32,6 +72,7 @@ def get(db_parameter_group_name):
     except c.exceptions.DBParameterGroupNotFoundFault:
         return None
 
+
 def get_parameters(db_parameter_group_name):
     """Returns a dict containing the paramters of a given parameter group
 
@@ -46,6 +87,7 @@ def get_parameters(db_parameter_group_name):
     except c.exceptions.DBParameterGroupNotFoundFault:
         return None
 
+
 def get_tags(db_parameter_group_arn):
     """Returns a dict containing the DB parameter group's tag records from the
     RDS API.

test/e2e/tests/test_db_cluster.py

@@ -19,6 +19,7 @@
 import pytest
 
 from acktest.k8s import resource as k8s
+from acktest.resources import random_suffix_name
 from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_rds_resource
 from e2e.replacement_values import REPLACEMENT_VALUES
 from e2e import condition
@@ -51,55 +52,69 @@
 
 MODIFY_WAIT_AFTER_SECONDS = 20
 
+# MUP == Master user password...
+MUP_NS = "default"
+MUP_SEC_NAME = "dbclustersecrets"
+MUP_SEC_KEY = "master_user_password"
+MUP_SEC_VAL = "secretpass123456"
+
+
+@pytest.fixture
+def aurora_mysql_cluster(k8s_secret):
+    db_cluster_id = random_suffix_name("my-aurora-mysql", 20)
+    db_name = "mydb"
+    secret = k8s_secret(
+        MUP_NS,
+        MUP_SEC_NAME,
+        MUP_SEC_KEY,
+        MUP_SEC_VAL,
+    )
+
+    replacements = REPLACEMENT_VALUES.copy()
+    replacements['COPY_TAGS_TO_SNAPSHOT'] = "False"
+    replacements["DB_CLUSTER_ID"] = db_cluster_id
+    replacements["DB_NAME"] = db_name
+    replacements["MASTER_USER_PASS_SECRET_NAMESPACE"] = secret.ns
+    replacements["MASTER_USER_PASS_SECRET_NAME"] = secret.name
+    replacements["MASTER_USER_PASS_SECRET_KEY"] = secret.key
+
+    resource_data = load_rds_resource(
+        "db_cluster_mysql_serverless",
+        additional_replacements=replacements,
+    )
+
+    ref = k8s.CustomResourceReference(
+        CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL,
+        db_cluster_id, namespace="default",
+    )
+    k8s.create_custom_resource(ref, resource_data)
+    cr = k8s.wait_resource_consumed_by_controller(ref)
+
+    assert cr is not None
+    assert 'status' in cr
+    assert 'status' in cr['status']
+    assert cr['status']['status'] == 'creating'
+    condition.assert_not_synced(ref)
+
+    yield (ref, cr, db_cluster_id)
+
+    # Try to delete, if doesn't already exist
+    try:
+        _, deleted = k8s.delete_custom_resource(ref, 3, 10)
+        assert deleted
+        time.sleep(DELETE_WAIT_AFTER_SECONDS)
+    except:
+        pass
+
+    db_cluster.wait_until_deleted(db_cluster_id)
 
 @service_marker
 @pytest.mark.canary
 class TestDBCluster:
-
-    # MUP == Master user password...
-    MUP_NS = "default"
-    MUP_SEC_NAME = "dbclustersecrets"
-    MUP_SEC_KEY = "master_user_password"
-    MUP_SEC_VAL = "secretpass123456"
-
     def test_crud_mysql_serverless(
-            self,
-            k8s_secret,
+            self, aurora_mysql_cluster,
     ):
-        db_cluster_id = "my-aurora-mysql"
-        db_name = "mydb"
-        secret = k8s_secret(
-            self.MUP_NS,
-            self.MUP_SEC_NAME,
-            self.MUP_SEC_KEY,
-            self.MUP_SEC_VAL,
-        )
-
-        replacements = REPLACEMENT_VALUES.copy()
-        replacements['COPY_TAGS_TO_SNAPSHOT'] = "False"
-        replacements["DB_CLUSTER_ID"] = db_cluster_id
-        replacements["DB_NAME"] = db_name
-        replacements["MASTER_USER_PASS_SECRET_NAMESPACE"] = secret.ns
-        replacements["MASTER_USER_PASS_SECRET_NAME"] = secret.name
-        replacements["MASTER_USER_PASS_SECRET_KEY"] = secret.key
-
-        resource_data = load_rds_resource(
-            "db_cluster_mysql_serverless",
-            additional_replacements=replacements,
-        )
-
-        ref = k8s.CustomResourceReference(
-            CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL,
-            db_cluster_id, namespace="default",
-        )
-        k8s.create_custom_resource(ref, resource_data)
-        cr = k8s.wait_resource_consumed_by_controller(ref)
-
-        assert cr is not None
-        assert 'status' in cr
-        assert 'status' in cr['status']
-        assert cr['status']['status'] == 'creating'
-        condition.assert_not_synced(ref)
+        ref, cr, db_cluster_id = aurora_mysql_cluster
 
         db_cluster.wait_until(
             db_cluster_id,
@@ -168,9 +183,3 @@ def test_crud_mysql_serverless(
             }
         ]
         assert latest_tags == after_update_expected_tags
-
-        k8s.delete_custom_resource(ref)
-
-        time.sleep(DELETE_WAIT_AFTER_SECONDS)
-
-        db_cluster.wait_until_deleted(db_cluster_id)

test/e2e/tests/test_db_cluster_parameter_group.py

@@ -34,15 +34,16 @@
 # take several minutes before the new tag values are available due to caching.
 MODIFY_WAIT_AFTER_SECONDS = 180
 
+RESOURCE_DESC_AURORA_MYSQL57 = "Parameters for Aurora MySQL 5.7-compatible"
+
 
 @pytest.fixture
-def aurora_mysql57_cluster_parameter_group():
-    resource_name = random_suffix_name("aurora-mysql-5-7", 32)
-    resource_desc = "Parameters for Aurora MySQL 5.7-compatible"
+def aurora_mysql57_cluster_param_group():
+    resource_name = random_suffix_name("aurora-mysql-5-7", 24)
 
     replacements = REPLACEMENT_VALUES.copy()
     replacements["DB_CLUSTER_PARAMETER_GROUP_NAME"] = resource_name
-    replacements["DB_CLUSTER_PARAMETER_GROUP_DESC"] = resource_desc
+    replacements["DB_CLUSTER_PARAMETER_GROUP_DESC"] = RESOURCE_DESC_AURORA_MYSQL57
 
     resource_data = load_rds_resource(
         "db_cluster_parameter_group_aurora_mysql5.7",
@@ -61,29 +62,29 @@ def aurora_mysql57_cluster_parameter_group():
     assert cr is not None
     assert k8s.get_resource_exists(ref)
 
-    yield (ref, cr)
+    yield ref, cr, resource_name
 
     # Try to delete, if doesn't already exist
     try:
         _, deleted = k8s.delete_custom_resource(ref, 3, 10)
         assert deleted
-        db_instance.wait_until_deleted(db_instance_id)
+        time.sleep(DELETE_WAIT_AFTER_SECONDS)
     except:
         pass
 
+    db_cluster_parameter_group.wait_until_deleted(resource_name)
+
+
 @service_marker
 @pytest.mark.canary
 class TestDBClusterParameterGroup:
-    def test_create_delete_aurora_mysql5_7(
-        self,
-        aurora_mysql57_cluster_parameter_group,
-    ):
-        (ref, cr) = aurora_mysql57_cluster_parameter_group
-        resource_name = cr["spec"]["name"]
+    def test_crud_aurora_mysql5_7(self, aurora_mysql57_cluster_param_group):
+        ref, cr, resource_name = aurora_mysql57_cluster_param_group
 
         # Let's check that the DB cluster parameter group appears in RDS
         latest = db_cluster_parameter_group.get(resource_name)
         assert latest is not None
+        assert latest['Description'] == RESOURCE_DESC_AURORA_MYSQL57
 
         arn = latest['DBClusterParameterGroupArn']
         expect_tags = [
@@ -157,4 +158,4 @@ def test_create_delete_aurora_mysql5_7(
                 found += 1
                 assert "ParameterValue" in tp, f"No ParameterValue in parameter of name 'aurora_read_replica_read_committed': {tp}"
                 assert tp["ParameterValue"] == "ON", f"Wrong value for parameter of name 'aurora_read_replica_read_committed': {tp}"
-        assert found == 2, f"Did not find parameters with names 'aurora_binlog_read_buffer_size' and 'aurora_read_replica_read_committed': {test_params}"
+        assert found == 2, f"Did not find parameters with names 'aurora_binlog_read_buffer_size' and 'aurora_read_replica_read_committed': {test_params}"