diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 4f5ccb802..ff3275650 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -5,8 +5,8 @@ import ( "fmt" "reflect" - "github.com/api7/gopkg/pkg/log" "github.com/go-logr/logr" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -14,11 +14,15 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "github.com/api7/gopkg/pkg/log" + "github.com/api7/api7-ingress-controller/api/v1alpha1" "github.com/api7/api7-ingress-controller/internal/controller/indexer" "github.com/api7/api7-ingress-controller/internal/provider" @@ -31,10 +35,13 @@ type GatewayReconciler struct { //nolint:revive Log logr.Logger Provider provider.Provider + + GenericEvent chan event.GenericEvent } // SetupWithManager sets up the controller with the Manager. func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.GenericEvent = GatewaySecretChan return ctrl.NewControllerManagedBy(mgr). For( &gatewayv1.Gateway{}, @@ -58,10 +65,24 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { &v1alpha1.GatewayProxy{}, handler.EnqueueRequestsFromMapFunc(r.listGatewaysForGatewayProxy), ). + WatchesRawSource( + source.Channel( + r.GenericEvent, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, object client.Object) []reconcile.Request { + switch object.(type) { + case *corev1.Secret: + return r.listGatewaysForSecret(ctx, object) + default: + return nil + } + })), + ). Complete(r) } func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.Log.Info("request Reconcile") + gateway := new(gatewayv1.Gateway) if err := r.Get(ctx, req.NamespacedName, gateway); err != nil { if client.IgnoreNotFound(err) == nil { @@ -295,6 +316,36 @@ func (r *GatewayReconciler) listGatewaysForHTTPRoute(ctx context.Context, obj cl return recs } +func (r *GatewayReconciler) listGatewaysForSecret(ctx context.Context, obj client.Object) (requests []reconcile.Request) { + secret, ok := obj.(*corev1.Secret) + if !ok { + r.Log.Error( + errors.New("unexpected object type"), + "Secret watch predicate received unexpected object type", + "expected", FullTypeName(new(corev1.Secret)), "found", FullTypeName(obj), + ) + return nil + } + r.Log.Info("listGatewaysForSecret, secret", "namespace", secret.GetNamespace(), "name", secret.GetName()) + var gatewayList gatewayv1.GatewayList + if err := r.List(ctx, &gatewayList, client.MatchingFields{ + indexer.SecretIndexRef: indexer.GenIndexKey(secret.GetNamespace(), secret.GetName()), + }); err != nil { + r.Log.Error(err, "failed to list gateways") + return nil + } + for _, gateway := range gatewayList.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: gateway.GetNamespace(), + Name: gateway.GetName(), + }, + }) + } + r.Log.Info("listGatewaysForSecret", "requests", requests) + return requests +} + func (r *GatewayReconciler) processInfrastructure(tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) error { rk := provider.ResourceKind{ Kind: gateway.Kind, @@ -316,12 +367,12 @@ func (r *GatewayReconciler) processListenerConfig(tctx *provider.TranslateContex if ref.Namespace != nil { ns = string(*ref.Namespace) } - if ref.Kind != nil && *ref.Kind == gatewayv1.Kind("Secret") { + if ref.Kind != nil && *ref.Kind == "Secret" { if err := r.Get(context.Background(), client.ObjectKey{ Namespace: ns, Name: string(ref.Name), }, &secret); err != nil { - log.Error(err, "failed to get secret", "namespace", ns, "name", string(ref.Name)) + log.Error(err, "failed to get secret", "namespace", ns, "name", ref.Name) SetGatewayListenerConditionProgrammed(gateway, string(listener.Name), false, err.Error()) SetGatewayListenerConditionResolvedRefs(gateway, string(listener.Name), false, err.Error()) break diff --git a/internal/controller/indexer/indexer.go b/internal/controller/indexer/indexer.go index 42ccddc92..0c2c5aaac 100644 --- a/internal/controller/indexer/indexer.go +++ b/internal/controller/indexer/indexer.go @@ -2,6 +2,7 @@ package indexer import ( "context" + "log" networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -27,26 +28,19 @@ const ( ) func SetupIndexer(mgr ctrl.Manager) error { - if err := setupGatewayIndexer(mgr); err != nil { - return err - } - if err := setupHTTPRouteIndexer(mgr); err != nil { - return err - } - if err := setupIngressIndexer(mgr); err != nil { - return err - } - if err := setupConsumerIndexer(mgr); err != nil { - return err - } - if err := setupBackendTrafficPolicyIndexer(mgr); err != nil { - return err - } - if err := setupIngressClassIndexer(mgr); err != nil { - return err - } - if err := setupGatewayProxyIndexer(mgr); err != nil { - return err + for _, setup := range []func(ctrl.Manager) error{ + setupGatewayIndexer, + setupHTTPRouteIndexer, + setupIngressIndexer, + setupConsumerIndexer, + setupBackendTrafficPolicyIndexer, + setupIngressClassIndexer, + setupGatewayProxyIndexer, + setupGatewaySecretIndex, + } { + if err := setup(mgr); err != nil { + return err + } } return nil } @@ -191,6 +185,15 @@ func setupGatewayProxyIndexer(mgr ctrl.Manager) error { return nil } +func setupGatewaySecretIndex(mgr ctrl.Manager) error { + return mgr.GetFieldIndexer().IndexField( + context.Background(), + &gatewayv1.Gateway{}, + SecretIndexRef, + GatewaySecretIndexFunc, + ) +} + func GatewayProxySecretIndexFunc(rawObj client.Object) []string { gatewayProxy := rawObj.(*v1alpha1.GatewayProxy) secretKeys := make([]string, 0) @@ -310,6 +313,32 @@ func IngressSecretIndexFunc(rawObj client.Object) []string { return secrets } +func GatewaySecretIndexFunc(rawObj client.Object) (keys []string) { + gateway := rawObj.(*gatewayv1.Gateway) + var m = make(map[string]struct{}) + for _, listener := range gateway.Spec.Listeners { + if listener.TLS == nil || len(listener.TLS.CertificateRefs) == 0 { + continue + } + for _, ref := range listener.TLS.CertificateRefs { + if ref.Kind == nil || *ref.Kind != "Secret" { + continue + } + namespace := gateway.GetNamespace() + if ref.Namespace != nil { + namespace = string(*ref.Namespace) + } + key := GenIndexKey(namespace, string(ref.Name)) + if _, ok := m[key]; !ok { + m[key] = struct{}{} + keys = append(keys, key) + } + } + } + log.Printf("GatewaySecretIndexFunc keys: %v", keys) + return keys +} + func GenIndexKeyWithGK(group, kind, namespace, name string) string { gvk := schema.GroupKind{ Group: group, diff --git a/internal/controller/secret_controller.go b/internal/controller/secret_controller.go new file mode 100644 index 000000000..37c30f345 --- /dev/null +++ b/internal/controller/secret_controller.go @@ -0,0 +1,55 @@ +package controller + +import ( + "context" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch + +type SecretReconciler struct { + client.Client + + Log logr.Logger + Indexer cache.Indexer + + GenericEvent chan event.GenericEvent +} + +func (r *SecretReconciler) SetupWithManager(mgr manager.Manager) error { + r.GenericEvent = GatewaySecretChan + return ctrl.NewControllerManagedBy(mgr). + Named("CoreV1Secret"). + WithOptions(controller.Options{ + CacheSyncTimeout: time.Second, + LogConstructor: func(_ *reconcile.Request) logr.Logger { + return r.Log + }, + }). + For(&corev1.Secret{}). + Complete(r) +} + +func (r *SecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.GenericEvent <- event.GenericEvent{ + Object: &corev1.Secret{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: req.Namespace, + Name: req.Name, + }, + }, + } + return ctrl.Result{}, nil +} diff --git a/internal/controller/utils.go b/internal/controller/utils.go index b270c5e15..04096e8cf 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -3,9 +3,10 @@ package controller import ( "context" "fmt" + "path" + "reflect" "strings" - "github.com/api7/gopkg/pkg/log" "github.com/samber/lo" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -14,9 +15,12 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "github.com/api7/gopkg/pkg/log" + "github.com/api7/api7-ingress-controller/api/v1alpha1" "github.com/api7/api7-ingress-controller/internal/controller/config" "github.com/api7/api7-ingress-controller/internal/provider" @@ -31,6 +35,10 @@ const ( KindGatewayProxy = "GatewayProxy" ) +var ( + GatewaySecretChan = make(chan event.GenericEvent, 100) +) + const defaultIngressClassAnnotation = "ingressclass.kubernetes.io/is-default-class" // IsDefaultIngressClass returns whether an IngressClass is the default IngressClass. @@ -839,3 +847,14 @@ func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gatew return nil } + +// FullTypeName returns the fully qualified name of the type of the given value. +func FullTypeName(a any) string { + typeOf := reflect.TypeOf(a) + pkgPath := typeOf.PkgPath() + name := typeOf.String() + if typeOf.Kind() == reflect.Ptr { + pkgPath = typeOf.Elem().PkgPath() + } + return path.Join(path.Dir(pkgPath), name) +} diff --git a/internal/manager/controllers.go b/internal/manager/controllers.go index 1bd40f5b9..821954e37 100644 --- a/internal/manager/controllers.go +++ b/internal/manager/controllers.go @@ -86,5 +86,9 @@ func setupControllers(ctx context.Context, mgr manager.Manager, pro provider.Pro Log: ctrl.LoggerFrom(ctx).WithName("controllers").WithName("IngressClass"), Provider: pro, }, + &controller.SecretReconciler{ + Client: mgr.GetClient(), + Log: ctrl.LoggerFrom(ctx).WithName("controllers").WithName("Secret"), + }, }, nil } diff --git a/test/e2e/gatewayapi/gateway.go b/test/e2e/gatewayapi/gateway.go index fc93ac306..4cdaf14f0 100644 --- a/test/e2e/gatewayapi/gateway.go +++ b/test/e2e/gatewayapi/gateway.go @@ -48,7 +48,7 @@ spec: ` Context("Gateway", func() { - var defautlGatewayClass = ` + var defaultGatewayClass = ` apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -57,7 +57,7 @@ spec: controllerName: "apisix.apache.org/api7-ingress-controller" ` - var defautlGateway = ` + var defaultGateway = ` apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: @@ -101,7 +101,7 @@ spec: time.Sleep(5 * time.Second) By("create GatewayClass") - err = s.CreateResourceFromStringWithNamespace(defautlGatewayClass, "") + err = s.CreateResourceFromStringWithNamespace(defaultGatewayClass, "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) @@ -112,7 +112,7 @@ spec: Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message") By("create Gateway") - err = s.CreateResourceFromStringWithNamespace(defautlGateway, s.CurrentNamespace()) + err = s.CreateResourceFromStringWithNamespace(defaultGateway, s.CurrentNamespace()) Expect(err).NotTo(HaveOccurred(), "creating Gateway") time.Sleep(5 * time.Second) @@ -196,7 +196,7 @@ spec: }) Context("Gateway SSL with and without hostname", func() { - It("Check if SSL resource was created", func() { + FIt("Check if SSL resource was created and updated", func() { By("create GatewayProxy") gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) err := s.CreateResourceFromString(gatewayProxy) @@ -266,6 +266,18 @@ spec: assert.Len(GinkgoT(), tls, 1, "tls number not expect") assert.Equal(GinkgoT(), Cert, tls[0].Cert, "tls cert not expect") assert.Equal(GinkgoT(), tls[0].Labels["k8s/controller-name"], "apisix.apache.org/api7-ingress-controller") + + By("update secret") + err = s.NewKubeTlsSecret(secretName, framework.TestCert, framework.TestKey) + Expect(err).NotTo(HaveOccurred(), "update secret") + Eventually(func() string { + tls, err := s.DefaultDataplaneResource().SSL().List(context.Background()) + Expect(err).NotTo(HaveOccurred(), "list ssl from dashboard") + if len(tls) < 1 { + return "" + } + return tls[0].Cert + }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(framework.TestCert)) }) }) })