chore: Add SecretReconciler to handle Gateway TLS secrets

Introduce a SecretReconciler to watch and reconcile secrets referenced by Gateway TLS configurations. This enables dynamic updates to Gateway SSL certificates when secrets change, improving the controller's ability to handle secret modifications efficiently. The change also includes necessary RBAC updates and test adjustments to ensure proper functionality.

Author: dspo

config/rbac/role.yaml

@@ -10,7 +10,11 @@ rules:
   - events
   verbs:
   - create
+  - get
+  - list
   - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -26,6 +30,7 @@ rules:
   verbs:
   - get
   - list
+  - update
   - watch
 - apiGroups:
   - ""

internal/controller/gateway_controller.go

@@ -5,36 +5,46 @@ import (
 	"fmt"
 	"reflect"
 
-	"github.com/api7/gopkg/pkg/log"
 	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
+	"github.com/api7/gopkg/pkg/log"
+
 	"github.com/api7/api7-ingress-controller/api/v1alpha1"
 	"github.com/api7/api7-ingress-controller/internal/controller/indexer"
 	"github.com/api7/api7-ingress-controller/internal/provider"
 )
 
+// +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;update
+// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update
+
 // GatewayReconciler reconciles a Gateway object.
 type GatewayReconciler struct { //nolint:revive
 	client.Client
 	Scheme *runtime.Scheme
 	Log    logr.Logger
 
 	Provider provider.Provider
+
+	GenericEvent chan event.GenericEvent
 }
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	r.GenericEvent = GatewaySecretChan
 	return ctrl.NewControllerManagedBy(mgr).
 		For(
 			&gatewayv1.Gateway{},
@@ -58,10 +68,24 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error {
 			&v1alpha1.GatewayProxy{},
 			handler.EnqueueRequestsFromMapFunc(r.listGatewaysForGatewayProxy),
 		).
+		WatchesRawSource(
+			source.Channel(
+				r.GenericEvent,
+				handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, object client.Object) []reconcile.Request {
+					switch object.(type) {
+					case *corev1.Secret:
+						return r.listGatewaysForSecret(ctx, object)
+					default:
+						return nil
+					}
+				})),
+		).
 		Complete(r)
 }
 
 func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	r.Log.Info("request Reconcile")
+
 	gateway := new(gatewayv1.Gateway)
 	if err := r.Get(ctx, req.NamespacedName, gateway); err != nil {
 		if client.IgnoreNotFound(err) == nil {
@@ -295,6 +319,36 @@ func (r *GatewayReconciler) listGatewaysForHTTPRoute(ctx context.Context, obj cl
 	return recs
 }
 
+func (r *GatewayReconciler) listGatewaysForSecret(ctx context.Context, obj client.Object) (requests []reconcile.Request) {
+	secret, ok := obj.(*corev1.Secret)
+	if !ok {
+		r.Log.Error(
+			errors.New("unexpected object type"),
+			"Secret watch predicate received unexpected object type",
+			"expected", FullTypeName(new(corev1.Secret)), "found", FullTypeName(obj),
+		)
+		return nil
+	}
+	r.Log.Info("listGatewaysForSecret, secret", "namespace", secret.GetNamespace(), "name", secret.GetName())
+	var gatewayList gatewayv1.GatewayList
+	if err := r.List(ctx, &gatewayList, client.MatchingFields{
+		indexer.SecretIndexRef: indexer.GenIndexKey(secret.GetNamespace(), secret.GetName()),
+	}); err != nil {
+		r.Log.Error(err, "failed to list gateways")
+		return nil
+	}
+	for _, gateway := range gatewayList.Items {
+		requests = append(requests, reconcile.Request{
+			NamespacedName: types.NamespacedName{
+				Namespace: gateway.GetNamespace(),
+				Name:      gateway.GetName(),
+			},
+		})
+	}
+	r.Log.Info("listGatewaysForSecret", "requests", requests)
+	return requests
+}
+
 func (r *GatewayReconciler) processInfrastructure(tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) error {
 	rk := provider.ResourceKind{
 		Kind:      gateway.Kind,
@@ -316,12 +370,12 @@ func (r *GatewayReconciler) processListenerConfig(tctx *provider.TranslateContex
 			if ref.Namespace != nil {
 				ns = string(*ref.Namespace)
 			}
-			if ref.Kind != nil && *ref.Kind == gatewayv1.Kind("Secret") {
+			if ref.Kind != nil && *ref.Kind == "Secret" {
 				if err := r.Get(context.Background(), client.ObjectKey{
 					Namespace: ns,
 					Name:      string(ref.Name),
 				}, &secret); err != nil {
-					log.Error(err, "failed to get secret", "namespace", ns, "name", string(ref.Name))
+					log.Error(err, "failed to get secret", "namespace", ns, "name", ref.Name)
 					SetGatewayListenerConditionProgrammed(gateway, string(listener.Name), false, err.Error())
 					SetGatewayListenerConditionResolvedRefs(gateway, string(listener.Name), false, err.Error())
 					break

internal/controller/indexer/indexer.go

@@ -2,6 +2,7 @@ package indexer
 
 import (
 	"context"
+	"log"
 
 	networkingv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -27,26 +28,19 @@ const (
 )
 
 func SetupIndexer(mgr ctrl.Manager) error {
-	if err := setupGatewayIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupHTTPRouteIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupIngressIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupConsumerIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupBackendTrafficPolicyIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupIngressClassIndexer(mgr); err != nil {
-		return err
-	}
-	if err := setupGatewayProxyIndexer(mgr); err != nil {
-		return err
+	for _, setup := range []func(ctrl.Manager) error{
+		setupGatewayIndexer,
+		setupHTTPRouteIndexer,
+		setupIngressIndexer,
+		setupConsumerIndexer,
+		setupBackendTrafficPolicyIndexer,
+		setupIngressClassIndexer,
+		setupGatewayProxyIndexer,
+		setupGatewaySecretIndex,
+	} {
+		if err := setup(mgr); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -191,6 +185,15 @@ func setupGatewayProxyIndexer(mgr ctrl.Manager) error {
 	return nil
 }
 
+func setupGatewaySecretIndex(mgr ctrl.Manager) error {
+	return mgr.GetFieldIndexer().IndexField(
+		context.Background(),
+		&gatewayv1.Gateway{},
+		SecretIndexRef,
+		GatewaySecretIndexFunc,
+	)
+}
+
 func GatewayProxySecretIndexFunc(rawObj client.Object) []string {
 	gatewayProxy := rawObj.(*v1alpha1.GatewayProxy)
 	secretKeys := make([]string, 0)
@@ -310,6 +313,32 @@ func IngressSecretIndexFunc(rawObj client.Object) []string {
 	return secrets
 }
 
+func GatewaySecretIndexFunc(rawObj client.Object) (keys []string) {
+	gateway := rawObj.(*gatewayv1.Gateway)
+	var m = make(map[string]struct{})
+	for _, listener := range gateway.Spec.Listeners {
+		if listener.TLS == nil || len(listener.TLS.CertificateRefs) == 0 {
+			continue
+		}
+		for _, ref := range listener.TLS.CertificateRefs {
+			if ref.Kind == nil || *ref.Kind != "Secret" {
+				continue
+			}
+			namespace := gateway.GetNamespace()
+			if ref.Namespace != nil {
+				namespace = string(*ref.Namespace)
+			}
+			key := GenIndexKey(namespace, string(ref.Name))
+			if _, ok := m[key]; !ok {
+				m[key] = struct{}{}
+				keys = append(keys, key)
+			}
+		}
+	}
+	log.Printf("GatewaySecretIndexFunc keys: %v", keys)
+	return keys
+}
+
 func GenIndexKeyWithGK(group, kind, namespace, name string) string {
 	gvk := schema.GroupKind{
 		Group: group,

internal/controller/secret_controller.go

@@ -0,0 +1,76 @@
+package controller
+
+import (
+	"context"
+	"time"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/cache"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	"github.com/api7/api7-ingress-controller/internal/controller/indexer"
+)
+
+type SecretReconciler struct {
+	client.Client
+
+	Log     logr.Logger
+	Indexer cache.Indexer
+
+	GenericEvent chan event.GenericEvent
+}
+
+func (r *SecretReconciler) SetupWithManager(mgr manager.Manager) error {
+	r.GenericEvent = GatewaySecretChan
+	return ctrl.NewControllerManagedBy(mgr).
+		Named("CoreV1Secret").
+		WithOptions(controller.Options{
+			CacheSyncTimeout: time.Second,
+			LogConstructor: func(_ *reconcile.Request) logr.Logger {
+				return r.Log
+			},
+		}).
+		For(&corev1.Secret{}). //builder.WithPredicates(r.predicateFuncs()),
+
+		Complete(r)
+}
+
+func (r *SecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	r.GenericEvent <- event.GenericEvent{
+		Object: &corev1.Secret{
+			TypeMeta: metav1.TypeMeta{},
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: req.Namespace,
+				Name:      req.Name,
+			},
+		},
+	}
+	return ctrl.Result{}, nil
+}
+
+func (r *SecretReconciler) predicateFuncs() predicate.Funcs {
+	predicateFuncs := predicate.NewPredicateFuncs(func(object client.Object) bool {
+		if _, ok := object.(*corev1.Secret); !ok {
+			return false
+		}
+		key := indexer.GenIndexKey(object.GetNamespace(), object.GetName())
+		refs, err := r.Indexer.ByIndex("referent", key)
+		if err != nil {
+			r.Log.Error(err, "failed to check whether secret referred", "namespace", object.GetNamespace(), "name", object.GetName())
+			return false
+		}
+		return len(refs) > 0
+	})
+	predicateFuncs.DeleteFunc = func(_ event.DeleteEvent) bool {
+		return true
+	}
+	return predicateFuncs
+}

internal/controller/utils.go

@@ -3,9 +3,10 @@ package controller
 import (
 	"context"
 	"fmt"
+	"path"
+	"reflect"
 	"strings"
 
-	"github.com/api7/gopkg/pkg/log"
 	"github.com/samber/lo"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
@@ -14,9 +15,12 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
+	"github.com/api7/gopkg/pkg/log"
+
 	"github.com/api7/api7-ingress-controller/api/v1alpha1"
 	"github.com/api7/api7-ingress-controller/internal/controller/config"
 	"github.com/api7/api7-ingress-controller/internal/provider"
@@ -31,6 +35,10 @@ const (
 	KindGatewayProxy = "GatewayProxy"
 )
 
+var (
+	GatewaySecretChan = make(chan event.GenericEvent, 100)
+)
+
 const defaultIngressClassAnnotation = "ingressclass.kubernetes.io/is-default-class"
 
 // IsDefaultIngressClass returns whether an IngressClass is the default IngressClass.
@@ -839,3 +847,14 @@ func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gatew
 
 	return nil
 }
+
+// FullTypeName returns the fully qualified name of the type of the given value.
+func FullTypeName(a any) string {
+	typeOf := reflect.TypeOf(a)
+	pkgPath := typeOf.PkgPath()
+	name := typeOf.String()
+	if typeOf.Kind() == reflect.Ptr {
+		pkgPath = typeOf.Elem().PkgPath()
+	}
+	return path.Join(path.Dir(pkgPath), name)
+}

internal/manager/controllers.go

@@ -86,5 +86,9 @@ func setupControllers(ctx context.Context, mgr manager.Manager, pro provider.Pro
 			Log:      ctrl.LoggerFrom(ctx).WithName("controllers").WithName("IngressClass"),
 			Provider: pro,
 		},
+		&controller.SecretReconciler{
+			Client: mgr.GetClient(),
+			Log:    ctrl.LoggerFrom(ctx).WithName("controllers").WithName("Secret"),
+		},
 	}, nil
 }