chore: Add webhook to prevent GatewayClass from being deleted when referenced by gateway

.golangci.yml

@@ -24,6 +24,8 @@ linters:
     revive:
       rules:
         - name: comment-spacings
+    lll:
+      line-length: 160
   exclusions:
     generated: lax
     rules:

PROJECT

@@ -34,4 +34,12 @@ resources:
   kind: HTTPRoutePolicy
   path: github.com/api7/api7-ingress-controller/api/v1alpha1
   version: v1alpha1
+- external: true
+  group: gateway.networking.k8s.io
+  kind: GatewayClass
+  path: sigs.k8s.io/gateway-api/apis/v1
+  version: v1
+  webhooks:
+    validation: true
+    webhookVersion: v1
 version: "3"

internal/manager/run.go

@@ -20,6 +20,7 @@ import (
 	"github.com/api7/api7-ingress-controller/api/v1alpha1"
 	"github.com/api7/api7-ingress-controller/internal/controller/config"
 	"github.com/api7/api7-ingress-controller/internal/provider/adc"
+	webhookv1 "github.com/api7/api7-ingress-controller/internal/webhook/v1"
 )
 
 var (
@@ -138,6 +139,9 @@ func Run(ctx context.Context, logger logr.Logger) error {
 			return err
 		}
 	}
+	if err = webhookv1.SetupGatewayClassWebhookWithManager(mgr); err != nil {
+		return err
+	}
 
 	// +kubebuilder:scaffold:builder
 

internal/webhook/v1/gatewayclass_webhook.go

@@ -0,0 +1,83 @@
+package v1
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+)
+
+var _gcLog = logf.Log.WithName("gatewayclass-resource")
+
+// SetupGatewayClassWebhookWithManager registers the webhook for GatewayClass in the manager.
+func SetupGatewayClassWebhookWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).For(&gatewayv1.GatewayClass{}).
+		WithValidator(&GatewayClassCustomValidator{
+			Client: mgr.GetClient(),
+			Logger: _gcLog,
+		}).
+		Complete()
+}
+
+// +kubebuilder:webhook:path=/validate-gateway-networking-k8s-io-v1-gatewayclass,mutating=false,failurePolicy=fail,sideEffects=None,groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=create;update;delete,versions=v1,name=vgatewayclass-v1.kb.io,admissionReviewVersions=v1
+
+// GatewayClassCustomValidator struct is responsible for validating the GatewayClass resource
+// when it is created, updated, or deleted.
+//
+// NOTE: The +kubebuilder:object:generate=false marker prevents controller-gen from generating DeepCopy methods,
+// as this struct is used only for temporary operations and does not need to be deeply copied.
+type GatewayClassCustomValidator struct {
+	client.Client
+	logr.Logger
+}
+
+var _ webhook.CustomValidator = &GatewayClassCustomValidator{}
+
+// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type GatewayClass.
+func (v *GatewayClassCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}
+
+// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type GatewayClass.
+func (v *GatewayClassCustomValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) {
+	return nil, nil
+}
+
+// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type GatewayClass.
+func (v *GatewayClassCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) {
+	gc, ok := obj.(*gatewayv1.GatewayClass)
+	if !ok {
+		return nil, fmt.Errorf("expected a GatewayClass object but got %T", obj)
+	}
+	v.Info("Validation for GatewayClass upon deletion", "name", gc.GetName())
+
+	var gatewayList gatewayv1.GatewayList
+	if err := v.List(ctx, &gatewayList); err != nil {
+		v.Error(err, "failed to list gateway for the GatewayClass")
+		return nil, err
+	}
+	var gateways []types.NamespacedName
+	for _, gateway := range gatewayList.Items {
+		if string(gateway.Spec.GatewayClassName) == gc.Name {
+			gateways = append(gateways, types.NamespacedName{
+				Namespace: gateway.GetNamespace(),
+				Name:      gateway.GetName(),
+			})
+		}
+	}
+	if len(gateways) > 0 {
+		err := fmt.Errorf("the GatewayClass is still in using by Gateways: %v", gateways)
+		v.Error(err, "can not delete the GatewayClass")
+		return nil, err
+	}
+
+	return nil, nil
+}

test/e2e/framework/ingress.go

@@ -2,14 +2,26 @@ package framework
 
 import (
 	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	_ "embed"
+	"encoding/base64"
+	"encoding/pem"
 	"text/template"
 	"time"
 
 	"github.com/Masterminds/sprig/v3"
 	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/retry"
 	. "github.com/onsi/gomega" //nolint:staticcheck
+	"github.com/pkg/errors"
+	certificatesv1 "k8s.io/api/certificates/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 var (
@@ -33,12 +45,18 @@ type IngressDeployOpts struct {
 	Namespace      string
 	AdminEnpoint   string
 	StatusAddress  string
+	TLSKey         string
+	TLSCRT         string
+	CaBundle       string
 }
 
 func (f *Framework) DeployIngress(opts IngressDeployOpts) {
+	err := f.setWebhookCertificate(&opts)
+	f.GomegaT.Expect(err).NotTo(HaveOccurred(), "set certificates info for webhook service")
+
 	buf := bytes.NewBuffer(nil)
 
-	err := IngressSpecTpl.Execute(buf, opts)
+	err = IngressSpecTpl.Execute(buf, opts)
 	f.GomegaT.Expect(err).ToNot(HaveOccurred(), "rendering ingress spec")
 
 	kubectlOpts := k8s.NewKubectlOptions("", "", opts.Namespace)
@@ -51,3 +69,114 @@ func (f *Framework) DeployIngress(opts IngressDeployOpts) {
 	f.GomegaT.Expect(err).ToNot(HaveOccurred(), "waiting for controller-manager pod ready")
 	f.WaitControllerManagerLog("All cache synced successfully", 0, time.Minute)
 }
+
+func (f *Framework) setWebhookCertificate(opts *IngressDeployOpts) error {
+	// generate private key
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return errors.Wrap(err, "failed to GenerateKey")
+	}
+	var privateKeyBuf = bytes.NewBuffer(nil)
+	if err = pem.Encode(privateKeyBuf, &pem.Block{
+		Type:    "RSA PRIVATE KEY",
+		Headers: nil,
+		Bytes:   x509.MarshalPKCS1PrivateKey(privateKey),
+	}); err != nil {
+		return errors.Wrap(err, "failed to pem.Encode private key")
+	}
+
+	// prepare CertificateSigningRequest
+	csrBytes, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{
+		Subject: pkix.Name{
+			Organization: []string{"system:nodes"},
+			CommonName:   "system:node:webhook-service." + opts.Namespace,
+		},
+		DNSNames: []string{
+			"webhook-service",
+			"webhook-service." + opts.Namespace,
+			"webhook-service." + opts.Namespace + ".svc",
+		},
+	}, privateKey)
+	if err != nil {
+		return errors.Wrap(err, "failed to CreateCertificateRequest")
+	}
+
+	certificateSigningRequest := &certificatesv1.CertificateSigningRequest{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "webhook-csr",
+		},
+		Spec: certificatesv1.CertificateSigningRequestSpec{
+			Request: pem.EncodeToMemory(&pem.Block{
+				Type:  "CERTIFICATE REQUEST",
+				Bytes: csrBytes,
+			}),
+			SignerName:        "kubernetes.io/kubelet-serving",
+			ExpirationSeconds: nil,
+			Usages: []certificatesv1.KeyUsage{
+				certificatesv1.UsageServerAuth,
+				certificatesv1.UsageDigitalSignature,
+				certificatesv1.UsageKeyEncipherment,
+			},
+		},
+	}
+
+	// try to delete the CertificateSigningRequest before creating
+	err = f.clientset.CertificatesV1().CertificateSigningRequests().
+		Delete(context.Background(), certificateSigningRequest.GetName(), metav1.DeleteOptions{})
+	if err != nil {
+		f.Logf("failed to CertificateSigningRequests().Delete: %v", err)
+	}
+
+	// create CertificateSigningRequest
+	certificateSigningRequest, err = f.clientset.CertificatesV1().CertificateSigningRequests().
+		Create(context.Background(), certificateSigningRequest, metav1.CreateOptions{})
+	if err != nil {
+		return errors.Wrap(err, "failed to CertificateSigningRequests().Create")
+	}
+
+	// to approve the CertificateSigningRequest
+	condition := certificatesv1.CertificateSigningRequestCondition{
+		Type:               certificatesv1.CertificateApproved,
+		Status:             "True",
+		Reason:             "AdminApproval",
+		Message:            "CSR approved by admin",
+		LastUpdateTime:     metav1.Now(),
+		LastTransitionTime: metav1.Now(),
+	}
+	certificateSigningRequest.Status.Conditions = append(certificateSigningRequest.Status.Conditions, condition)
+	certificateSigningRequest, err = f.clientset.CertificatesV1().CertificateSigningRequests().
+		UpdateApproval(context.Background(), certificateSigningRequest.GetName(), certificateSigningRequest, metav1.UpdateOptions{})
+	if err != nil {
+		return errors.Wrap(err, "failed to CertificateSigningRequests().UpdateApproval")
+	}
+
+	// try to get the Certificate issued by K8s
+	certPEM := retry.DoWithRetry(f.GinkgoT, "get approved certificate", 10, time.Second, func() (string, error) {
+		csr, err := f.clientset.CertificatesV1().CertificateSigningRequests().Get(context.Background(), certificateSigningRequest.GetName(), metav1.GetOptions{})
+		if err != nil {
+			return "", err
+		}
+		if csr.Status.Certificate == nil {
+			return "", errors.New("certificate is not signed yet")
+		}
+		return string(csr.Status.Certificate), nil
+	})
+
+	// get client-ca-file as the caBundle for webhook ValidatingWebhookConfiguration
+	var cm corev1.ConfigMap
+	var cmKey = client.ObjectKey{
+		Namespace: "kube-system",
+		Name:      "extension-apiserver-authentication",
+	}
+	err = f.K8sClient.Get(context.Background(), cmKey, &cm)
+	if err != nil {
+		return errors.Wrapf(err, "failed to get ConfigMap: %v", cmKey)
+	}
+
+	// set certificate info
+	opts.TLSKey = "\n" + privateKeyBuf.String()
+	opts.TLSCRT = "\n" + certPEM
+	opts.CaBundle = base64.StdEncoding.EncodeToString([]byte(cm.Data["client-ca-file"]))
+
+	return nil
+}

test/e2e/framework/manifests/ingress.yaml

@@ -340,6 +340,10 @@ data:
     controller_name: {{ .ControllerName | default "apisix.apache.org/api7-ingress-controller" }}
 
     leader_election_id: "api7-ingress-controller-leader"
+  tls.key: |
+    {{- .TLSKey | indent 4 }}
+  tls.crt: |
+    {{- .TLSCRT | indent 4 }}
 ---
 apiVersion: v1
 kind: Service
@@ -398,6 +402,13 @@ spec:
         - name: ingress-config
           mountPath: /app/conf/config.yaml
           subPath: config.yaml
+        - name: ingress-config
+          mountPath: /tmp/k8s-webhook-server/serving-certs
+          readOnly: true
+        ports:
+          - containerPort: 9443
+            name: webhook-server
+            protocol: TCP
         livenessProbe:
           httpGet:
             path: /healthz
@@ -429,3 +440,47 @@ spec:
           name: ingress-config
       serviceAccountName: api7-ingress-controller-manager
       terminationGracePeriodSeconds: 10
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: api7-ingress-controller
+    app.kubernetes.io/managed-by: kustomize
+  name: webhook-service
+  namespace: {{ .Namespace }}
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      targetPort: 9443
+  selector:
+    control-plane: controller-manager
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-webhook-configuration
+webhooks:
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: webhook-service
+        namespace: {{ .Namespace }}
+        path: /validate-gateway-networking-k8s-io-v1-gatewayclass
+      caBundle: {{ .CaBundle }}
+    failurePolicy: Fail
+    name: vgatewayclass-v1.kb.io
+    rules:
+      - apiGroups:
+          - gateway.networking.k8s.io
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+          - DELETE
+        resources:
+          - gatewayclasses
+    sideEffects: None