Merge pull request kubernetes-sigs#6629 from ykakarap/runtime-sdk_lifecycle-hooks_tracking-hooks

✨ Implements AfterControlPlaneInitialized, AfterControlPlaneUpgrade and AfterClusterUpgrade hooks

Author: k8s-ci-robot

exp/runtime/api/v1alpha1/extensionconfig_types.go

@@ -206,4 +206,9 @@ const (
 	// object wants injection of CAs. It takes the form of a reference to a Secret
 	// as namespace/name.
 	InjectCAFromSecretAnnotation string = "runtime.cluster.x-k8s.io/inject-ca-from-secret"
+
+	// PendingHooksAnnotation is the annotation used to keep a track of pending runtime hooks.
+	// The annotation will be used to track the intent to call a hook as soon as an operation completes;
+	// the intent will be removed as soon as the hook call completes successfully.
+	PendingHooksAnnotation string = "runtime.cluster.x-k8s.io/pending-hooks"
 )

exp/runtime/hooks/api/v1alpha1/lifecyclehooks_types.go

@@ -188,7 +188,7 @@ func init() {
 	catalogBuilder.RegisterHook(AfterControlPlaneInitialized, &runtimecatalog.HookMeta{
 		Tags:        []string{"Lifecycle Hooks"},
 		Summary:     "Called after the Control Plane is available for the first time",
-		Description: "This non-blocking hook is called after the ControlPlane for the Cluster is marked as available for the first time",
+		Description: "This non-blocking hook is called after the ControlPlane for the Cluster reachable for the first time",
 	})
 
 	catalogBuilder.RegisterHook(BeforeClusterUpgrade, &runtimecatalog.HookMeta{

internal/controllers/topology/cluster/desired_state.go

@@ -35,6 +35,7 @@ import (
 	"sigs.k8s.io/cluster-api/feature"
 	"sigs.k8s.io/cluster-api/internal/contract"
 	"sigs.k8s.io/cluster-api/internal/controllers/topology/cluster/scope"
+	"sigs.k8s.io/cluster-api/internal/hooks"
 	tlog "sigs.k8s.io/cluster-api/internal/log"
 	runtimecatalog "sigs.k8s.io/cluster-api/internal/runtime/catalog"
 )
@@ -313,6 +314,37 @@ func (r *Reconciler) computeControlPlaneVersion(ctx context.Context, s *scope.Sc
 		// Nb. We do not return early in the function if the control plane is already at the desired version so as
 		// to know if the control plane is being upgraded. This information
 		// is required when updating the TopologyReconciled condition on the cluster.
+
+		// Call the AfterControlPlaneUpgrade now that the control plane is upgraded.
+		if feature.Gates.Enabled(feature.RuntimeSDK) {
+			// Call the hook only if we are tracking the intent to do so. If it is not tracked it means we don't need to call the
+			// hook because we didn't go through an upgrade or we already called the hook after the upgrade.
+			if hooks.IsPending(runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster) {
+				// Call all the registered extension for the hook.
+				hookRequest := &runtimehooksv1.AfterControlPlaneUpgradeRequest{
+					Cluster:           *s.Current.Cluster,
+					KubernetesVersion: desiredVersion,
+				}
+				hookResponse := &runtimehooksv1.AfterControlPlaneUpgradeResponse{}
+				if err := r.RuntimeClient.CallAllExtensions(ctx, runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster, hookRequest, hookResponse); err != nil {
+					return "", errors.Wrapf(err, "error calling the %s hook", runtimecatalog.HookName(runtimehooksv1.AfterControlPlaneUpgrade))
+				}
+				// Add the response to the tracker so we can later update condition or requeue when required.
+				s.HookResponseTracker.Add(runtimehooksv1.AfterControlPlaneUpgrade, hookResponse)
+
+				// If the extension responds to hold off on starting Machine deployments upgrades,
+				// change the UpgradeTracker accordingly, otherwise the hook call is completed and we
+				// can remove this hook from the list of pending-hooks.
+				if hookResponse.RetryAfterSeconds != 0 {
+					s.UpgradeTracker.MachineDeployments.HoldUpgrades(true)
+				} else {
+					if err := hooks.MarkAsDone(ctx, r.Client, s.Current.Cluster, runtimehooksv1.AfterControlPlaneUpgrade); err != nil {
+						return "", errors.Wrapf(err, "failed to remove the %s hook from pending hooks tracker", runtimecatalog.HookName(runtimehooksv1.AfterControlPlaneUpgrade))
+					}
+				}
+			}
+		}
+
 		return *currentVersion, nil
 	}
 
@@ -349,14 +381,21 @@ func (r *Reconciler) computeControlPlaneVersion(ctx context.Context, s *scope.Sc
 		if err := r.RuntimeClient.CallAllExtensions(ctx, runtimehooksv1.BeforeClusterUpgrade, s.Current.Cluster, hookRequest, hookResponse); err != nil {
 			return "", errors.Wrapf(err, "failed to call %s hook", runtimecatalog.HookName(runtimehooksv1.BeforeClusterUpgrade))
 		}
+		// Add the response to the tracker so we can later update condition or requeue when required.
 		s.HookResponseTracker.Add(runtimehooksv1.BeforeClusterUpgrade, hookResponse)
 		if hookResponse.RetryAfterSeconds != 0 {
 			// Cannot pickup the new version right now. Need to try again later.
 			return *currentVersion, nil
 		}
+
+		// We are picking up the new version here.
+		// Track the intent of calling the AfterControlPlaneUpgrade and the AfterClusterUpgrade hooks once we are done with the upgrade.
+		if err := hooks.MarkAsPending(ctx, r.Client, s.Current.Cluster, runtimehooksv1.AfterControlPlaneUpgrade, runtimehooksv1.AfterClusterUpgrade); err != nil {
+			return "", errors.Wrapf(err, "failed to mark the %s hook as pending", []string{runtimecatalog.HookName(runtimehooksv1.AfterControlPlaneUpgrade), runtimecatalog.HookName(runtimehooksv1.AfterClusterUpgrade)})
+		}
 	}
 
-	// Control plane and machine deployments are stable.
+	// Control plane and machine deployments are stable. All the required hook are called.
 	// Ready to pick up the topology version.
 	return desiredVersion, nil
 }