refactor DB instance e2e tests (#44)

Pulls out the waiter and getter functions for DB instance resources into
a test/e2e/db_instance.py file, modeled after the same set of functions
for the DB cluster resource.

Signed-off-by: Jay Pipes <[email protected]>

By submitting this pull request, I confirm that my contribution is made under
the terms of the Apache 2.0 license.

Author: jaypipes

test/e2e/db_cluster.py

@@ -20,9 +20,9 @@
 import boto3
 import pytest
 
-DEFAULT_WAIT_UNTIL_TIMEOUT_SECONDS = 600
+DEFAULT_WAIT_UNTIL_TIMEOUT_SECONDS = 60*10
 DEFAULT_WAIT_UNTIL_INTERVAL_SECONDS = 15
-DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS = 600
+DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS = 60*10
 DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS = 15
 
 ClusterMatchFunc = typing.NewType(

test/e2e/db_instance.py

@@ -0,0 +1,125 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+#	 http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+"""Utilities for working with DB instance resources"""
+
+import datetime
+import time
+import typing
+
+import boto3
+import pytest
+
+DEFAULT_WAIT_UNTIL_TIMEOUT_SECONDS = 60*30
+DEFAULT_WAIT_UNTIL_INTERVAL_SECONDS = 15
+DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS = 60*10
+DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS = 15
+
+InstanceMatchFunc = typing.NewType(
+    'InstanceMatchFunc',
+    typing.Callable[[dict], bool],
+)
+
+class StatusMatcher:
+    def __init__(self, status):
+        self.match_on = status
+
+    def __call__(self, record: dict) -> bool:
+        return ('DBInstanceStatus' in record
+                and record['DBInstanceStatus'] == self.match_on)
+
+
+def status_matches(status: str) -> InstanceMatchFunc:
+    return StatusMatcher(status)
+
+
+def wait_until(
+        db_instance_id: str,
+        match_fn: InstanceMatchFunc,
+        timeout_seconds: int = DEFAULT_WAIT_UNTIL_TIMEOUT_SECONDS,
+        interval_seconds: int = DEFAULT_WAIT_UNTIL_INTERVAL_SECONDS,
+    ) -> None:
+    """Waits until a DB instance with a supplied ID is returned from the RDS API
+    and the matching functor returns True.
+
+    Usage:
+        from e2e.db_instance import wait_until, status_matches
+
+        wait_until(
+            instance_id,
+            status_matches("available"),
+        )
+
+    Raises:
+        pytest.fail upon timeout
+    """
+    now = datetime.datetime.now()
+    timeout = now + datetime.timedelta(seconds=timeout_seconds)
+
+    while not match_fn(get(db_instance_id)):
+        if datetime.datetime.now() >= timeout:
+            pytest.fail("failed to match DBInstance before timeout")
+        time.sleep(interval_seconds)
+
+
+def wait_until_deleted(
+        db_instance_id: str,
+        timeout_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_TIMEOUT_SECONDS,
+        interval_seconds: int = DEFAULT_WAIT_UNTIL_DELETED_INTERVAL_SECONDS,
+    ) -> None:
+    """Waits until a DB instance with a supplied ID is no longer returned from
+    the RDS API.
+
+    Usage:
+        from e2e.db_instance import wait_until_deleted
+
+        wait_until_deleted(instance_id)
+
+    Raises:
+        pytest.fail upon timeout or if the DB instance goes to any other status
+        other than 'deleting'
+    """
+    now = datetime.datetime.now()
+    timeout = now + datetime.timedelta(seconds=timeout_seconds)
+
+    while True:
+        if datetime.datetime.now() >= timeout:
+            pytest.fail(
+                "Timed out waiting for DB instance to be "
+                "deleted in RDS API"
+            )
+        time.sleep(interval_seconds)
+
+        latest = get(db_instance_id)
+        if latest is None:
+            break
+
+        if latest['DBInstanceStatus'] != "deleting":
+            pytest.fail(
+                "Status is not 'deleting' for DB instance that was "
+                "deleted. Status is " + latest['DBInstanceStatus']
+            )
+
+
+def get(db_instance_id):
+    """Returns a dict containing the DB instance record from the RDS API.
+
+    If no such DB instance exists, returns None.
+    """
+    c = boto3.client('rds')
+    try:
+        resp = c.describe_db_instances(DBInstanceIdentifier=db_instance_id)
+        assert len(resp['DBInstances']) == 1
+        return resp['DBInstances'][0]
+    except c.exceptions.DBInstanceNotFoundFault:
+        return None

test/e2e/tests/test_db_cluster.py

@@ -46,7 +46,7 @@ class TestDBCluster:
     MUP_SEC_KEY = "master_user_password"
     MUP_SEC_VAL = "secretpass123456"
 
-    def test_create_delete_mysql_serverless(
+    def test_crud_mysql_serverless(
             self,
             k8s_secret,
     ):
@@ -121,7 +121,6 @@ def test_create_delete_mysql_serverless(
         assert latest is not None
         assert latest['CopyTagsToSnapshot'] == True
 
-        # Delete the k8s resource on teardown of the module
         k8s.delete_custom_resource(ref)
 
         time.sleep(DELETE_WAIT_AFTER_SECONDS)

test/e2e/tests/test_db_instance.py

@@ -14,36 +14,24 @@
 """Integration tests for the RDS API DBInstance resource
 """
 
-import datetime
-import logging
 import time
-from typing import Dict
 
 import pytest
 
 from acktest.k8s import resource as k8s
 from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_rds_resource
 from e2e.replacement_values import REPLACEMENT_VALUES
 from e2e.bootstrap_resources import get_bootstrap_resources
-from e2e.fixtures import rds_client, k8s_secret
+from e2e.fixtures import k8s_secret
+from e2e import db_instance
 
 RESOURCE_PLURAL = 'dbinstances'
 
-DELETE_WAIT_INTERVAL_SLEEP_SECONDS = 15
 DELETE_WAIT_AFTER_SECONDS = 120
-# NOTE(jaypipes): I've seen this take upwards of 5 minutes or more to fully see
-# the DB instance not appear in the DescribeDBInstances call once
-# DeleteDBInstance has been called (even with SkipFinalSnapshot=true)
-DELETE_TIMEOUT_SECONDS = 60*10
-
-CREATE_INTERVAL_SLEEP_SECONDS = 15
-# Time to wait before we get to an expected `available` state.
-# NOTE(jaypipes): I have seen creation of t3-micro PG instances take more than
-# 20 minutes to get to `available`...
-CREATE_TIMEOUT_SECONDS = 60*30
+
 # Time we wait after resource becoming available in RDS and checking the CR's
 # Status has been updated
-CHECK_STATUS_WAIT_SECONDS = 10
+CHECK_STATUS_WAIT_SECONDS = 20
 
 MODIFY_WAIT_AFTER_SECONDS = 20
 
@@ -60,10 +48,9 @@ class TestDBInstance:
 
     def test_crud_postgres13_t3_micro(
             self,
-            rds_client,
             k8s_secret,
     ):
-        db_id = "pg13-t3-micro"
+        db_instance_id = "pg13-t3-micro"
         secret = k8s_secret(
             self.MUP_NS,
             self.MUP_SEC_NAME,
@@ -73,7 +60,7 @@ def test_crud_postgres13_t3_micro(
 
         replacements = REPLACEMENT_VALUES.copy()
         replacements['COPY_TAGS_TO_SNAPSHOT'] = "False"
-        replacements["DB_INSTANCE_ID"] = db_id
+        replacements["DB_INSTANCE_ID"] = db_instance_id
         replacements["MASTER_USER_PASS_SECRET_NAMESPACE"] = secret.ns
         replacements["MASTER_USER_PASS_SECRET_NAME"] = secret.name
         replacements["MASTER_USER_PASS_SECRET_KEY"] = secret.key
@@ -82,12 +69,11 @@ def test_crud_postgres13_t3_micro(
             "db_instance_postgres13_t3_micro",
             additional_replacements=replacements,
         )
-        logging.debug(resource_data)
 
         # Create the k8s resource
         ref = k8s.CustomResourceReference(
             CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL,
-            db_id, namespace="default",
+            db_instance_id, namespace="default",
         )
         k8s.create_custom_resource(ref, resource_data)
         cr = k8s.wait_resource_consumed_by_controller(ref)
@@ -97,23 +83,10 @@ def test_crud_postgres13_t3_micro(
         assert 'dbInstanceStatus' in cr['status']
         assert cr['status']['dbInstanceStatus'] == 'creating'
 
-        # Let's check that the DB instance appears in RDS
-        aws_res = rds_client.describe_db_instances(DBInstanceIdentifier=db_id)
-        assert aws_res is not None
-        assert len(aws_res['DBInstances']) == 1
-        dbi_rec = aws_res['DBInstances'][0]
-
-        now = datetime.datetime.now()
-        timeout = now + datetime.timedelta(seconds=CREATE_TIMEOUT_SECONDS)
-
-        # TODO(jaypipes): Move this into generic AWS-side waiter
-        while dbi_rec['DBInstanceStatus'] != "available":
-            if datetime.datetime.now() >= timeout:
-                pytest.fail("failed to find available DBInstance before timeout")
-            time.sleep(CREATE_INTERVAL_SLEEP_SECONDS)
-            aws_res = rds_client.describe_db_instances(DBInstanceIdentifier=db_id)
-            assert len(aws_res['DBInstances']) == 1
-            dbi_rec = aws_res['DBInstances'][0]
+        db_instance.wait_until(
+            db_instance_id,
+            db_instance.status_matches('available'),
+        )
 
         time.sleep(CHECK_STATUS_WAIT_SECONDS)
 
@@ -134,38 +107,21 @@ def test_crud_postgres13_t3_micro(
         # We're now going to modify the CopyTagsToSnapshot field of the DB
         # instance, wait some time and verify that the RDS server-side resource
         # shows the new value of the field.
-        assert dbi_rec['CopyTagsToSnapshot'] == False
+        latest = db_instance.get(db_instance_id)
+        assert latest is not None
+        assert latest['CopyTagsToSnapshot'] == False
         updates = {
             "spec": {"copyTagsToSnapshot": True},
         }
         k8s.patch_custom_resource(ref, updates)
         time.sleep(MODIFY_WAIT_AFTER_SECONDS)
 
-        aws_res = rds_client.describe_db_instances(DBInstanceIdentifier=db_id)
-        assert aws_res is not None
-        assert len(aws_res['DBInstances']) == 1
-        dbi_rec = aws_res['DBInstances'][0]
-        assert dbi_rec['CopyTagsToSnapshot'] == True
+        latest = db_instance.get(db_instance_id)
+        assert latest is not None
+        assert latest['CopyTagsToSnapshot'] == True
 
-        # Delete the k8s resource on teardown of the module
         k8s.delete_custom_resource(ref)
 
         time.sleep(DELETE_WAIT_AFTER_SECONDS)
 
-        now = datetime.datetime.now()
-        timeout = now + datetime.timedelta(seconds=DELETE_TIMEOUT_SECONDS)
-
-        # DB instance should no longer appear in RDS
-        while True:
-            if datetime.datetime.now() >= timeout:
-                pytest.fail("Timed out waiting for DB instance to being deleted in RDS API")
-            time.sleep(DELETE_WAIT_INTERVAL_SLEEP_SECONDS)
-
-            try:
-                aws_res = rds_client.describe_db_instances(DBInstanceIdentifier=db_id)
-                assert len(aws_res['DBInstances']) == 1
-                dbi_rec = aws_res['DBInstances'][0]
-                if dbi_rec['DBInstanceStatus'] != "deleting":
-                    pytest.fail("DBInstanceStatus is not 'deleting' for DB instance that was deleted. DBInstanceStatus is "+dbi_rec['DBInstanceStatus'])
-            except rds_client.exceptions.DBInstanceNotFoundFault:
-                break
+        db_instance.wait_until_deleted(db_instance_id)