Merge remote-tracking branch 'origin/release-v2-dev' into fix/pass_more_test

Author: ronething

internal/controller/gateway_controller.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"reflect"
 
@@ -42,7 +43,15 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error {
 				predicate.NewPredicateFuncs(r.checkGatewayClass),
 			),
 		).
-		WithEventFilter(predicate.GenerationChangedPredicate{}).
+		WithEventFilter(
+			predicate.Or(
+				predicate.GenerationChangedPredicate{},
+				predicate.NewPredicateFuncs(func(obj client.Object) bool {
+					_, ok := obj.(*corev1.Secret)
+					return ok
+				}),
+			),
+		).
 		Watches(
 			&gatewayv1.GatewayClass{},
 			handler.EnqueueRequestsFromMapFunc(r.listGatewayForGatewayClass),
@@ -58,6 +67,10 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error {
 			&v1alpha1.GatewayProxy{},
 			handler.EnqueueRequestsFromMapFunc(r.listGatewaysForGatewayProxy),
 		).
+		Watches(
+			&corev1.Secret{},
+			handler.EnqueueRequestsFromMapFunc(r.listGatewaysForSecret),
+		).
 		Complete(r)
 }
 
@@ -295,6 +308,34 @@ func (r *GatewayReconciler) listGatewaysForHTTPRoute(ctx context.Context, obj cl
 	return recs
 }
 
+func (r *GatewayReconciler) listGatewaysForSecret(ctx context.Context, obj client.Object) (requests []reconcile.Request) {
+	secret, ok := obj.(*corev1.Secret)
+	if !ok {
+		r.Log.Error(
+			errors.New("unexpected object type"),
+			"Secret watch predicate received unexpected object type",
+			"expected", FullTypeName(new(corev1.Secret)), "found", FullTypeName(obj),
+		)
+		return nil
+	}
+	var gatewayList gatewayv1.GatewayList
+	if err := r.List(ctx, &gatewayList, client.MatchingFields{
+		indexer.SecretIndexRef: indexer.GenIndexKey(secret.GetNamespace(), secret.GetName()),
+	}); err != nil {
+		r.Log.Error(err, "failed to list gateways")
+		return nil
+	}
+	for _, gateway := range gatewayList.Items {
+		requests = append(requests, reconcile.Request{
+			NamespacedName: types.NamespacedName{
+				Namespace: gateway.GetNamespace(),
+				Name:      gateway.GetName(),
+			},
+		})
+	}
+	return requests
+}
+
 func (r *GatewayReconciler) processInfrastructure(tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) error {
 	rk := provider.ResourceKind{
 		Kind:      gateway.Kind,
@@ -316,12 +357,12 @@ func (r *GatewayReconciler) processListenerConfig(tctx *provider.TranslateContex
 			if ref.Namespace != nil {
 				ns = string(*ref.Namespace)
 			}
-			if ref.Kind != nil && *ref.Kind == gatewayv1.Kind("Secret") {
+			if ref.Kind != nil && *ref.Kind == KindSecret {
 				if err := r.Get(context.Background(), client.ObjectKey{
 					Namespace: ns,
 					Name:      string(ref.Name),
 				}, &secret); err != nil {
-					log.Error(err, "failed to get secret", "namespace", ns, "name", string(ref.Name))
+					log.Error(err, "failed to get secret", "namespace", ns, "name", ref.Name)
 					SetGatewayListenerConditionProgrammed(gateway, string(listener.Name), false, err.Error())
 					SetGatewayListenerConditionResolvedRefs(gateway, string(listener.Name), false, err.Error())
 					break

internal/controller/gatewayclass_congroller.go

@@ -3,18 +3,26 @@ package controller
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/go-logr/logr"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	"github.com/api7/api7-ingress-controller/internal/controller/config"
 )
 
+const (
+	FinalizerGatewayClassProtection = "apisix.apache.org/gc-protection"
+)
+
 // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=get;list;watch;update
 // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses/status,verbs=get;update
 
@@ -23,11 +31,13 @@ type GatewayClassReconciler struct { //nolint:revive
 	client.Client
 	Scheme *runtime.Scheme
 
+	record.EventRecorder
 	Log logr.Logger
 }
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *GatewayClassReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	r.EventRecorder = mgr.GetEventRecorderFor("gatewayclass-controller")
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&gatewayv1.GatewayClass{}).
 		WithEventFilter(predicate.NewPredicateFuncs(r.GatewayClassFilter)).
@@ -41,6 +51,43 @@ func (r *GatewayClassReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	if gc.GetDeletionTimestamp().IsZero() {
+		if !controllerutil.ContainsFinalizer(gc, FinalizerGatewayClassProtection) {
+			controllerutil.AddFinalizer(gc, FinalizerGatewayClassProtection)
+			if err := r.Update(ctx, gc); err != nil {
+				return ctrl.Result{}, err
+			}
+		}
+	} else {
+		if controllerutil.ContainsFinalizer(gc, FinalizerGatewayClassProtection) {
+			var gatewayList gatewayv1.GatewayList
+			if err := r.List(ctx, &gatewayList); err != nil {
+				r.Log.Error(err, "failed to list gateways")
+				return ctrl.Result{}, err
+			}
+			var gateways []types.NamespacedName
+			for _, gateway := range gatewayList.Items {
+				if string(gateway.Spec.GatewayClassName) == gc.GetName() {
+					gateways = append(gateways, types.NamespacedName{
+						Namespace: gateway.GetNamespace(),
+						Name:      gateway.GetName(),
+					})
+				}
+			}
+			if len(gateways) > 0 {
+				r.Eventf(gc, "Warning", "DeletionBlocked", "the GatewayClass is still used by Gateways: %v", gateways)
+				return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+			} else {
+				controllerutil.RemoveFinalizer(gc, FinalizerGatewayClassProtection)
+				if err := r.Update(ctx, gc); err != nil {
+					return ctrl.Result{}, err
+				}
+			}
+		}
+
+		return ctrl.Result{}, nil
+	}
+
 	condition := meta.Condition{
 		Type:               string(gatewayv1.GatewayClassConditionStatusAccepted),
 		Status:             meta.ConditionTrue,

internal/controller/indexer/indexer.go

@@ -27,26 +27,19 @@ const (
 )
 
 func SetupIndexer(mgr ctrl.Manager) error {
-	if err := setupGatewayIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupHTTPRouteIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupIngressIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupConsumerIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupBackendTrafficPolicyIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupIngressClassIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupGatewayProxyIndexer(mgr); err != nil {
-		return err
+	for _, setup := range []func(ctrl.Manager) error{
+		setupGatewayIndexer,
+		setupHTTPRouteIndexer,
+		setupIngressIndexer,
+		setupConsumerIndexer,
+		setupBackendTrafficPolicyIndexer,
+		setupIngressClassIndexer,
+		setupGatewayProxyIndexer,
+		setupGatewaySecretIndex,
+	} {
+		if err := setup(mgr); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -191,6 +184,15 @@ func setupGatewayProxyIndexer(mgr ctrl.Manager) error {
 	return nil
 }
 
+func setupGatewaySecretIndex(mgr ctrl.Manager) error {
+	return mgr.GetFieldIndexer().IndexField(
+		context.Background(),
+		&gatewayv1.Gateway{},
+		SecretIndexRef,
+		GatewaySecretIndexFunc,
+	)
+}
+
 func GatewayProxySecretIndexFunc(rawObj client.Object) []string {
 	gatewayProxy := rawObj.(*v1alpha1.GatewayProxy)
 	secretKeys := make([]string, 0)
@@ -310,6 +312,31 @@ func IngressSecretIndexFunc(rawObj client.Object) []string {
 	return secrets
 }
 
+func GatewaySecretIndexFunc(rawObj client.Object) (keys []string) {
+	gateway := rawObj.(*gatewayv1.Gateway)
+	var m = make(map[string]struct{})
+	for _, listener := range gateway.Spec.Listeners {
+		if listener.TLS == nil || len(listener.TLS.CertificateRefs) == 0 {
+			continue
+		}
+		for _, ref := range listener.TLS.CertificateRefs {
+			if ref.Kind == nil || *ref.Kind != "Secret" {
+				continue
+			}
+			namespace := gateway.GetNamespace()
+			if ref.Namespace != nil {
+				namespace = string(*ref.Namespace)
+			}
+			key := GenIndexKey(namespace, string(ref.Name))
+			if _, ok := m[key]; !ok {
+				m[key] = struct{}{}
+				keys = append(keys, key)
+			}
+		}
+	}
+	return keys
+}
+
 func GenIndexKeyWithGK(group, kind, namespace, name string) string {
 	gvk := schema.GroupKind{
 		Group: group,

internal/controller/utils.go

@@ -3,6 +3,8 @@ package controller
 import (
 	"context"
 	"fmt"
+	"path"
+	"reflect"
 	"strings"
 
 	"github.com/api7/gopkg/pkg/log"
@@ -29,6 +31,7 @@ const (
 	KindIngress      = "Ingress"
 	KindIngressClass = "IngressClass"
 	KindGatewayProxy = "GatewayProxy"
+	KindSecret       = "Secret"
 )
 
 const defaultIngressClassAnnotation = "ingressclass.kubernetes.io/is-default-class"
@@ -839,3 +842,14 @@ func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gatew
 
 	return nil
 }
+
+// FullTypeName returns the fully qualified name of the type of the given value.
+func FullTypeName(a any) string {
+	typeOf := reflect.TypeOf(a)
+	pkgPath := typeOf.PkgPath()
+	name := typeOf.String()
+	if typeOf.Kind() == reflect.Ptr {
+		pkgPath = typeOf.Elem().PkgPath()
+	}
+	return path.Join(path.Dir(pkgPath), name)
+}

test/e2e/gatewayapi/gateway.go

@@ -48,7 +48,7 @@ spec:
 `
 
 	Context("Gateway", func() {
-		var defautlGatewayClass = `
+		var defaultGatewayClass = `
 apiVersion: gateway.networking.k8s.io/v1
 kind: GatewayClass
 metadata:
@@ -57,7 +57,7 @@ spec:
   controllerName: "apisix.apache.org/api7-ingress-controller"
 `
 
-		var defautlGateway = `
+		var defaultGateway = `
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
@@ -101,7 +101,7 @@ spec:
 			time.Sleep(5 * time.Second)
 
 			By("create GatewayClass")
-			err = s.CreateResourceFromStringWithNamespace(defautlGatewayClass, "")
+			err = s.CreateResourceFromStringWithNamespace(defaultGatewayClass, "")
 			Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
 			time.Sleep(5 * time.Second)
 
@@ -112,7 +112,7 @@ spec:
 			Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message")
 
 			By("create Gateway")
-			err = s.CreateResourceFromString(defautlGateway)
+			err = s.CreateResourceFromStringWithNamespace(defaultGateway, s.CurrentNamespace())
 			Expect(err).NotTo(HaveOccurred(), "creating Gateway")
 			time.Sleep(5 * time.Second)
 
@@ -123,7 +123,7 @@ spec:
 			Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the api7-ingress-controller"), "checking Gateway condition message")
 
 			By("create Gateway with not accepted GatewayClass")
-			err = s.CreateResourceFromString(noClassGateway)
+			err = s.CreateResourceFromStringWithNamespace(noClassGateway, s.CurrentNamespace())
 			Expect(err).NotTo(HaveOccurred(), "creating Gateway")
 			time.Sleep(5 * time.Second)
 
@@ -184,7 +184,7 @@ spec:
 			time.Sleep(5 * time.Second)
 
 			By("create Gateway")
-			err = s.CreateResourceFromString(defaultGateway)
+			err = s.CreateResourceFromStringWithNamespace(defaultGateway, s.CurrentNamespace())
 			Expect(err).NotTo(HaveOccurred(), "creating Gateway")
 			time.Sleep(10 * time.Second)
 
@@ -196,7 +196,7 @@ spec:
 		})
 
 		Context("Gateway SSL with and without hostname", func() {
-			It("Check if SSL resource was created", func() {
+			It("Check if SSL resource was created and updated", func() {
 				By("create GatewayProxy")
 				gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey())
 				err := s.CreateResourceFromString(gatewayProxy)
@@ -257,7 +257,7 @@ spec:
 				time.Sleep(5 * time.Second)
 
 				By("create Gateway")
-				err = s.CreateResourceFromString(defaultGateway)
+				err = s.CreateResourceFromStringWithNamespace(defaultGateway, s.CurrentNamespace())
 				Expect(err).NotTo(HaveOccurred(), "creating Gateway")
 				time.Sleep(10 * time.Second)
 
@@ -266,6 +266,18 @@ spec:
 				assert.Len(GinkgoT(), tls, 1, "tls number not expect")
 				assert.Equal(GinkgoT(), Cert, tls[0].Cert, "tls cert not expect")
 				assert.Equal(GinkgoT(), tls[0].Labels["k8s/controller-name"], "apisix.apache.org/api7-ingress-controller")
+
+				By("update secret")
+				err = s.NewKubeTlsSecret(secretName, framework.TestCert, framework.TestKey)
+				Expect(err).NotTo(HaveOccurred(), "update secret")
+				Eventually(func() string {
+					tls, err := s.DefaultDataplaneResource().SSL().List(context.Background())
+					Expect(err).NotTo(HaveOccurred(), "list ssl from dashboard")
+					if len(tls) < 1 {
+						return ""
+					}
+					return tls[0].Cert
+				}).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(framework.TestCert))
 			})
 		})
 	})