requeue db cluster while not available

After creating a new DBCluster resource, we were not setting
ConditionTypeResourceSynced on the resource nor were we asking to
requeue the resource after some time. This meant that the resource's
Status was not being updated on the Kubernetes side, even after the DB
instance's status transitioned to 'available' on the RDS side.

In this patch, we're adding two generic hook implementations for
sdk_read_many_post_set_output and sdk_create_post_set_output that will
create a ConditionTypeResourceSynced on the resource with a value of
ConditionFalse (because the resource isn't actually synced yet). This
causes the controller to queue the resource up for a new reconciliation
loop, during which the resource's Status will be updated to the latest
observed values.

Issue aws-controllers-k8s/community#923

Author: jaypipes

apis/v1alpha1/ack-generate-metadata.yaml

@@ -1,13 +1,14 @@
 ack_generate_info:
-  build_date: "2021-10-05T23:44:00Z"
-  build_hash: 2b0ac062dd6d305434d252b5da5ed718c801b106
-  go_version: go1.17.1
-  version: v0.14.1
-api_directory_checksum: cd0469d4474d82dbcf9badb97a43cac68ba07044
+  build_date: "2021-09-01T17:48:47Z"
+  build_hash: 6f22b7b568e25b4ee007bb1ab5f9338c16daf172
+  go_version: go1.17 linux/amd64
+  version: v0.13.0
+api_directory_checksum: 387f19255122f7e68716af4d6dbce0e7498cfe37
 api_version: v1alpha1
 aws_sdk_go_version: v1.37.10
 generator_config_info:
-  file_checksum: ec38b79b167ebd86c08109049864acd6a21e5113
+  file_checksum: f5d26ad7ae3f2cdc82f2e08c45f5b0aefafb9fa4
   original_file_name: generator.yaml
 last_modification:
   reason: API generation
+  timestamp: 2021-09-01 17:48:52.072416982 +0000 UTC

apis/v1alpha1/generator.yaml

@@ -66,6 +66,10 @@ resources:
       # resolved.
       custom_method_name: customUpdate
     hooks:
+      sdk_create_post_set_output:
+        template_path: hooks/db_cluster/sdk_create_post_set_output.go.tpl
+      sdk_read_many_post_set_output:
+        template_path: hooks/db_cluster/sdk_read_many_post_set_output.go.tpl
       sdk_update_pre_build_request:
         template_path: hooks/db_cluster/sdk_update_pre_build_request.go.tpl
       sdk_delete_pre_build_request:

generator.yaml

@@ -66,6 +66,10 @@ resources:
       # resolved.
       custom_method_name: customUpdate
     hooks:
+      sdk_create_post_set_output:
+        template_path: hooks/db_cluster/sdk_create_post_set_output.go.tpl
+      sdk_read_many_post_set_output:
+        template_path: hooks/db_cluster/sdk_read_many_post_set_output.go.tpl
       sdk_update_pre_build_request:
         template_path: hooks/db_cluster/sdk_update_pre_build_request.go.tpl
       sdk_delete_pre_build_request:

pkg/resource/db_cluster/hooks.go

@@ -116,6 +116,16 @@ func clusterHasTerminalStatus(r *resource) bool {
 	return false
 }
 
+// clusterAvailable returns true if the supplied DB cluster is in an
+// available status
+func clusterAvailable(r *resource) bool {
+	if r.ko.Status.Status == nil {
+		return false
+	}
+	dbcs := *r.ko.Status.Status
+	return dbcs == StatusAvailable
+}
+
 // clusterCreating returns true if the supplied DB cluster is in the process
 // of being created
 func clusterCreating(r *resource) bool {

pkg/resource/db_cluster/sdk.go

@@ -0,0 +1,9 @@
+	// We expect the DB cluster to be in 'creating' status since we just
+	// issued the call to create it, but I suppose it doesn't hurt to check
+	// here.
+	if clusterCreating(&resource{ko}) {
+		// Setting resource synced condition to false will trigger a requeue of
+		// the resource. No need to return a requeue error here.
+		setSyncedCondition(&resource{ko}, corev1.ConditionFalse, nil, nil)
+		return &resource{ko}, nil
+	}

templates/hooks/db_cluster/sdk_create_post_set_output.go.tpl

@@ -0,0 +1,7 @@
+	if !clusterAvailable(&resource{ko}) {
+		// Setting resource synced condition to false will trigger a requeue of
+		// the resource. No need to return a requeue error here.
+		setSyncedCondition(&resource{ko}, corev1.ConditionFalse, nil, nil)
+		return &resource{ko}, nil
+	}
+

templates/hooks/db_cluster/sdk_read_many_post_set_output.go.tpl

@@ -35,6 +35,9 @@
 
 CREATE_INTERVAL_SLEEP_SECONDS = 15
 CREATE_TIMEOUT_SECONDS = 600
+# Time we wait after resource becoming available in RDS and checking the CR's
+# Status has been updated
+CHECK_STATUS_WAIT_SECONDS = 10
 
 MODIFY_WAIT_AFTER_SECONDS = 20
 
@@ -88,6 +91,9 @@ def test_create_delete_mysql_serverless(
         cr = k8s.wait_resource_consumed_by_controller(ref)
 
         assert cr is not None
+        assert 'status' in cr
+        assert 'status' in cr['status']
+        assert cr['status']['status'] == 'creating'
 
         # Let's check that the DB cluster appears in RDS
         aws_res = rds_client.describe_db_clusters(DBClusterIdentifier=db_cluster_id)
@@ -107,6 +113,22 @@ def test_create_delete_mysql_serverless(
             assert len(aws_res['DBClusters']) == 1
             dbc_rec = aws_res['DBClusters'][0]
 
+        time.sleep(CHECK_STATUS_WAIT_SECONDS)
+
+        # Before we update the DBCluster CR below, let's check to see that the
+        # Status field in the CR has been updated to something other than
+        # 'creating', which is what is set after the initial creation.  The
+        # CR's `Status.Status` should be updated because the CR is requeued on
+        # successful reconciliation loops and subsequent reconciliation loops
+        # call ReadOne and should update the CR's Status with the latest
+        # observed information.
+        # https://github.com/aws-controllers-k8s/community/issues/923
+        cr = k8s.get_resource(ref)
+        assert cr is not None
+        assert 'status' in cr
+        assert 'status' in cr['status']
+        assert cr['status']['status'] != 'creating'
+
         # We're now going to modify the CopyTagsToSnapshot field of the DB
         # instance, wait some time and verify that the RDS server-side resource
         # shows the new value of the field.