Fix webhook admission control deadlock during installation (#2179)

This fixes a chicken-and-egg bootstrap issue where the operator would
get stuck during KnativeServing installation.

Problem:
- ValidatingWebhookConfiguration with failurePolicy=Fail intercepts
  Certificate resource creation
- If Certificate resources are created before the webhook pod is ready,
  the API server rejects them
- The activator deployment depends on the routing-serving-certs secret
  (generated from a Certificate resource) at runtime
- Previous stage ordering would check all deployments (including activator)
  before creating Certificate resources, causing a deadlock

Solution:
1. Added CheckWebhookDeployment() function that waits specifically for
   the webhook deployment to be ready before proceeding
2. Reordered reconciliation stages:
   - manifests.Install (creates all deployments)
   - CheckWebhookDeployment (waits for webhook to be ready)
   - InstallWebhookDependentResources (creates Certificate resources)
   - CheckDeployments (checks all deployments including activator)

This ensures:
- Webhook is ready before Certificate creation (avoids admission rejection)
- Certificate resources exist before checking activator (avoids missing secret)
- Clear error message if webhook deployment is missing from manifest

Related functions:
- pkg/reconciler/common/deployments.go: Added CheckWebhookDeployment()
- pkg/reconciler/knativeserving/knativeserving.go: Reordered stages
- pkg/reconciler/common/install.go: Added logging for consistency

Author: linkvt

pkg/reconciler/common/deployments.go

@@ -19,6 +19,7 @@ package common
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	mf "github.com/manifestival/manifestival"
 	appsv1 "k8s.io/api/apps/v1"
@@ -27,6 +28,7 @@ import (
 	"k8s.io/client-go/kubernetes/scheme"
 
 	"knative.dev/operator/pkg/apis/operator/base"
+	"knative.dev/pkg/logging"
 )
 
 type deploymentsNotReadyError struct{}
@@ -43,6 +45,43 @@ func IsDeploymentsNotReadyError(err error) bool {
 	return errors.Is(err, deploymentsNotReadyError{})
 }
 
+// CheckWebhookDeployment checks if the webhook deployment is ready.
+// This is needed before creating webhook-dependent resources (e.g., Certificates)
+// because the ValidatingWebhookConfiguration with failurePolicy=Fail will reject
+// Certificate creation if the webhook pod is not ready.
+func CheckWebhookDeployment(ctx context.Context, manifest *mf.Manifest, instance base.KComponent) error {
+	logger := logging.FromContext(ctx)
+	logger.Debug("Checking webhook deployment")
+	status := instance.GetStatus()
+	webhookDeployment := manifest.Filter(mf.ByKind("Deployment"), mf.ByName("webhook"))
+
+	if len(webhookDeployment.Resources()) == 0 {
+		status.MarkInstallFailed("webhook deployment not found in manifest")
+		return fmt.Errorf("webhook deployment not found in manifest")
+	}
+
+	for _, u := range webhookDeployment.Resources() {
+		resource, err := manifest.Client.Get(&u)
+		if err != nil {
+			status.MarkDeploymentsNotReady([]string{"webhook"})
+			if apierrors.IsNotFound(err) {
+				return deploymentsNotReadyError{}
+			}
+			return err
+		}
+		deployment := &appsv1.Deployment{}
+		if err := scheme.Scheme.Convert(resource, deployment, nil); err != nil {
+			return err
+		}
+		if !isDeploymentAvailable(deployment) {
+			status.MarkDeploymentsNotReady([]string{"webhook"})
+			return deploymentsNotReadyError{}
+		}
+	}
+
+	return nil
+}
+
 // CheckDeployments checks all deployments in the given manifest and updates the given
 // status with the status of the deployments.
 func CheckDeployments(ctx context.Context, manifest *mf.Manifest, instance base.KComponent) error {

pkg/reconciler/common/install.go

@@ -90,6 +90,7 @@ func InstallWebhookConfigs(ctx context.Context, manifest *mf.Manifest, instance
 
 // InstallWebhookDependentResources applies the Webhook dependent resources updates the given status accordingly.
 func InstallWebhookDependentResources(ctx context.Context, manifest *mf.Manifest, instance base.KComponent) error {
+	logging.FromContext(ctx).Debug("Installing webhook dependent resources")
 	status := instance.GetStatus()
 	if err := manifest.Filter(webhookDependentResources).Apply(); err != nil {
 		status.MarkInstallFailed(err.Error())

pkg/reconciler/knativeserving/knativeserving.go

@@ -124,9 +124,10 @@ func (r *Reconciler) ReconcileKind(ctx context.Context, ks *v1beta1.KnativeServi
 		r.appendExtensionManifests,
 		r.transform,
 		manifests.Install,
-		manifests.SetManifestPaths, // setting path right after applying manifests to populate paths
-		common.CheckDeployments,
+		manifests.SetManifestPaths,    // setting path right after applying manifests to populate paths
+		common.CheckWebhookDeployment, // Wait for webhook to be ready before creating Certificate resources
 		common.InstallWebhookDependentResources,
+		common.CheckDeployments,
 		common.MarkStatusSuccess,
 		common.DeleteObsoleteResources(ctx, ks, r.installed),
 	}