diff --git a/charts/templates/cluster_role.yaml b/charts/templates/cluster_role.yaml index 7effb5c25..ebe77e254 100644 --- a/charts/templates/cluster_role.yaml +++ b/charts/templates/cluster_role.yaml @@ -170,6 +170,20 @@ rules: - get - list - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - referencegrants + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - referencegrants/status + verbs: + - get - apiGroups: - networking.k8s.io resources: diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 77e9f10b4..332089ae9 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -18,7 +18,6 @@ import ( "fmt" "reflect" - "github.com/api7/gopkg/pkg/log" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,10 +26,14 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/api7/gopkg/pkg/log" "github.com/apache/apisix-ingress-controller/api/v1alpha1" "github.com/apache/apisix-ingress-controller/internal/controller/indexer" @@ -83,6 +86,23 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { &corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.listGatewaysForSecret), ). + Watches(&v1beta1.ReferenceGrant{}, + handler.EnqueueRequestsFromMapFunc(r.listReferenceGrantsForGateway), + builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return referenceGrantHasGatewayFrom(e.Object) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return referenceGrantHasGatewayFrom(e.ObjectOld) || referenceGrantHasGatewayFrom(e.ObjectNew) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return referenceGrantHasGatewayFrom(e.Object) + }, + GenericFunc: func(e event.GenericEvent) bool { + return referenceGrantHasGatewayFrom(e.Object) + }, + }), + ). Complete(r) } @@ -117,7 +137,7 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct msg: acceptedMessage("gateway"), } - // create a translate context + // create a translation context tctx := provider.NewDefaultTranslateContext(ctx) r.processListenerConfig(tctx, gateway) @@ -164,20 +184,25 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } } - ListenerStatuses, err := getListenerStatus(ctx, r.Client, gateway) + var referenceGrantList v1beta1.ReferenceGrantList + if err := r.List(ctx, &referenceGrantList); err != nil { + r.Log.Error(err, "failed to list reference grants") + return ctrl.Result{}, err + } + listenerStatuses, err := getListenerStatus(ctx, r.Client, gateway, referenceGrantList.Items) if err != nil { - log.Error(err, "failed to get listener status", "gateway", gateway.GetName()) + r.Log.Error(err, "failed to get listener status", "gateway", types.NamespacedName{Namespace: gateway.GetNamespace(), Name: gateway.GetName()}) return ctrl.Result{}, err } accepted := SetGatewayConditionAccepted(gateway, acceptStatus.status, acceptStatus.msg) - Programmed := SetGatewayConditionProgrammed(gateway, conditionProgrammedStatus, conditionProgrammedMsg) - if accepted || Programmed || len(addrs) > 0 || len(ListenerStatuses) > 0 { + programmed := SetGatewayConditionProgrammed(gateway, conditionProgrammedStatus, conditionProgrammedMsg) + if accepted || programmed || len(addrs) > 0 || len(listenerStatuses) > 0 { if len(addrs) > 0 { gateway.Status.Addresses = addrs } - if len(ListenerStatuses) > 0 { - gateway.Status.Listeners = ListenerStatuses + if len(listenerStatuses) > 0 { + gateway.Status.Listeners = listenerStatuses } return ctrl.Result{}, r.Status().Update(ctx, gateway) @@ -348,6 +373,56 @@ func (r *GatewayReconciler) listGatewaysForSecret(ctx context.Context, obj clien return requests } +func (r *GatewayReconciler) listReferenceGrantsForGateway(ctx context.Context, obj client.Object) (requests []reconcile.Request) { + grant, ok := obj.(*v1beta1.ReferenceGrant) + if !ok { + r.Log.Error( + errors.New("unexpected object type"), + "ReferenceGrant watch predicate received unexpected object type", + "expected", FullTypeName(new(v1beta1.ReferenceGrant)), "found", FullTypeName(obj), + ) + return nil + } + + var gatewayList gatewayv1.GatewayList + if err := r.List(ctx, &gatewayList); err != nil { + r.Log.Error(err, "failed to list gateways in watch predicate", "ReferenceGrant", grant.GetName()) + return nil + } + + for _, gateway := range gatewayList.Items { + for _, from := range grant.Spec.From { + gw := v1beta1.ReferenceGrantFrom{ + Group: gatewayv1.GroupName, + Kind: KindGateway, + Namespace: v1beta1.Namespace(gateway.GetNamespace()), + } + if from == gw { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: gateway.GetNamespace(), + Name: gateway.GetName(), + }, + }) + } + } + } + return requests +} + +func referenceGrantHasGatewayFrom(obj client.Object) bool { + grant, ok := obj.(*v1beta1.ReferenceGrant) + if !ok { + return false + } + for _, from := range grant.Spec.From { + if from.Kind == KindGateway && string(from.Group) == gatewayv1.GroupName { + return true + } + } + return false +} + func (r *GatewayReconciler) processInfrastructure(tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) error { rk := provider.ResourceKind{ Kind: gateway.Kind, diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 5dd2e7f51..dd356dd6c 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/apache/apisix-ingress-controller/api/v1alpha1" "github.com/apache/apisix-ingress-controller/internal/controller/config" @@ -121,14 +122,9 @@ func IsConditionPresentAndEqual(conditions []metav1.Condition, condition metav1. } func SetGatewayConditionAccepted(gw *gatewayv1.Gateway, status bool, message string) (ok bool) { - conditionStatus := metav1.ConditionTrue - if !status { - conditionStatus = metav1.ConditionFalse - } - condition := metav1.Condition{ Type: string(gatewayv1.GatewayConditionAccepted), - Status: conditionStatus, + Status: ConditionStatus(status), Reason: string(gatewayv1.GatewayReasonAccepted), ObservedGeneration: gw.GetGeneration(), Message: message, @@ -661,6 +657,7 @@ func getListenerStatus( ctx context.Context, mrgc client.Client, gateway *gatewayv1.Gateway, + grants []v1beta1.ReferenceGrant, ) ([]gatewayv1.ListenerStatus, error) { statuses := make(map[gatewayv1.SectionName]gatewayv1.ListenerStatus, len(gateway.Spec.Listeners)) @@ -670,12 +667,35 @@ func getListenerStatus( return nil, err } var ( - reasonResolvedRef = string(gatewayv1.ListenerReasonResolvedRefs) - statusResolvedRef = metav1.ConditionTrue - messageResolvedRef string - - reasonProgrammed = string(gatewayv1.ListenerReasonProgrammed) - statusProgrammed = metav1.ConditionTrue + now = metav1.Now() + conditionProgrammed = metav1.Condition{ + Type: string(gatewayv1.ListenerConditionProgrammed), + Status: metav1.ConditionTrue, + ObservedGeneration: gateway.GetGeneration(), + LastTransitionTime: now, + Reason: string(gatewayv1.ListenerReasonProgrammed), + } + conditionAccepted = metav1.Condition{ + Type: string(gatewayv1.ListenerConditionAccepted), + Status: metav1.ConditionTrue, + ObservedGeneration: gateway.GetGeneration(), + LastTransitionTime: now, + Reason: string(gatewayv1.ListenerReasonAccepted), + } + conditionConflicted = metav1.Condition{ + Type: string(gatewayv1.ListenerConditionConflicted), + Status: metav1.ConditionTrue, + ObservedGeneration: gateway.GetGeneration(), + LastTransitionTime: now, + Reason: string(gatewayv1.ListenerReasonNoConflicts), + } + conditionResolvedRefs = metav1.Condition{ + Type: string(gatewayv1.ListenerConditionResolvedRefs), + Status: metav1.ConditionTrue, + ObservedGeneration: gateway.GetGeneration(), + LastTransitionTime: now, + Reason: string(gatewayv1.ListenerReasonResolvedRefs), + } supportedKinds = []gatewayv1.RouteGroupKind{} ) @@ -683,42 +703,54 @@ func getListenerStatus( if listener.AllowedRoutes == nil || listener.AllowedRoutes.Kinds == nil { supportedKinds = []gatewayv1.RouteGroupKind{ { - Kind: gatewayv1.Kind("HTTPRoute"), + Kind: KindHTTPRoute, }, } } else { for _, kind := range listener.AllowedRoutes.Kinds { if kind.Group != nil && *kind.Group != gatewayv1.GroupName { - reasonResolvedRef = string(gatewayv1.ListenerReasonInvalidRouteKinds) - statusResolvedRef = metav1.ConditionFalse + conditionResolvedRefs.Status = metav1.ConditionFalse + conditionResolvedRefs.Reason = string(gatewayv1.ListenerReasonInvalidRouteKinds) continue } switch kind.Kind { - case gatewayv1.Kind("HTTPRoute"): + case KindHTTPRoute: supportedKinds = append(supportedKinds, kind) default: - reasonResolvedRef = string(gatewayv1.ListenerReasonInvalidRouteKinds) - statusResolvedRef = metav1.ConditionFalse + conditionResolvedRefs.Status = metav1.ConditionFalse + conditionResolvedRefs.Reason = string(gatewayv1.ListenerReasonInvalidRouteKinds) } - } } if listener.TLS != nil { // TODO: support TLS var ( - secret corev1.Secret - resolved = true + secret corev1.Secret ) for _, ref := range listener.TLS.CertificateRefs { if ref.Group != nil && *ref.Group != corev1.GroupName { - resolved = false - messageResolvedRef = fmt.Sprintf(`Invalid Group, expect "", got "%s"`, *ref.Group) + conditionResolvedRefs.Status = metav1.ConditionFalse + conditionResolvedRefs.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) + conditionResolvedRefs.Message = fmt.Sprintf(`Invalid Group, expect "", got "%s"`, *ref.Group) + conditionProgrammed.Status = metav1.ConditionFalse + conditionProgrammed.Reason = string(gatewayv1.ListenerReasonInvalid) + break + } + if ref.Kind != nil && *ref.Kind != KindSecret { + conditionResolvedRefs.Status = metav1.ConditionFalse + conditionResolvedRefs.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) + conditionResolvedRefs.Message = fmt.Sprintf(`Invalid Kind, expect "Secret", got "%s"`, *ref.Kind) + conditionProgrammed.Status = metav1.ConditionFalse + conditionProgrammed.Reason = string(gatewayv1.ListenerReasonInvalid) break } - if ref.Kind != nil && *ref.Kind != "Secret" { - resolved = false - messageResolvedRef = fmt.Sprintf(`Invalid Kind, expect "Secret", got "%s"`, *ref.Kind) + if ok := checkReferenceGrantBetweenGatewayAndSecret(gateway.Namespace, ref, grants); !ok { + conditionResolvedRefs.Status = metav1.ConditionFalse + conditionResolvedRefs.Reason = string(gatewayv1.ListenerReasonRefNotPermitted) + conditionResolvedRefs.Message = "certificateRefs cross namespaces is not permitted" + conditionProgrammed.Status = metav1.ConditionFalse + conditionProgrammed.Reason = string(gatewayv1.ListenerReasonInvalid) break } ns := gateway.Namespace @@ -726,59 +758,32 @@ func getListenerStatus( ns = string(*ref.Namespace) } if err := mrgc.Get(ctx, client.ObjectKey{Namespace: ns, Name: string(ref.Name)}, &secret); err != nil { - resolved = false - messageResolvedRef = err.Error() + conditionResolvedRefs.Status = metav1.ConditionFalse + conditionResolvedRefs.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) + conditionResolvedRefs.Message = err.Error() + conditionProgrammed.Status = metav1.ConditionFalse + conditionProgrammed.Reason = string(gatewayv1.ListenerReasonInvalid) break } - if reason, ok := isTLSSecretValid(&secret); !ok { - resolved = false - messageResolvedRef = fmt.Sprintf("Malformed Secret referenced: %s", reason) + if cause, ok := isTLSSecretValid(&secret); !ok { + conditionResolvedRefs.Status = metav1.ConditionFalse + conditionResolvedRefs.Reason = string(gatewayv1.ListenerReasonInvalidCertificateRef) + conditionResolvedRefs.Message = fmt.Sprintf("Malformed Secret referenced: %s", cause) + conditionProgrammed.Status = metav1.ConditionFalse + conditionProgrammed.Reason = string(gatewayv1.ListenerReasonInvalid) break } } - if !resolved { - reasonResolvedRef = string(gatewayv1.ListenerReasonInvalidCertificateRef) - statusResolvedRef = metav1.ConditionFalse - reasonProgrammed = string(gatewayv1.ListenerReasonInvalid) - statusProgrammed = metav1.ConditionFalse - } - } - - conditions := []metav1.Condition{ - { - Type: string(gatewayv1.ListenerConditionProgrammed), - Status: statusProgrammed, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Now(), - Reason: reasonProgrammed, - }, - { - Type: string(gatewayv1.ListenerConditionAccepted), - Status: metav1.ConditionTrue, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(gatewayv1.ListenerReasonAccepted), - }, - { - Type: string(gatewayv1.ListenerConditionConflicted), - Status: metav1.ConditionTrue, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(gatewayv1.ListenerReasonNoConflicts), - }, - { - Type: string(gatewayv1.ListenerConditionResolvedRefs), - Status: statusResolvedRef, - ObservedGeneration: gateway.Generation, - LastTransitionTime: metav1.Now(), - Reason: reasonResolvedRef, - Message: messageResolvedRef, - }, } status := gatewayv1.ListenerStatus{ - Name: listener.Name, - Conditions: conditions, + Name: listener.Name, + Conditions: []metav1.Condition{ + conditionProgrammed, + conditionAccepted, + conditionConflicted, + conditionResolvedRefs, + }, SupportedKinds: supportedKinds, AttachedRoutes: attachedRoutes, } @@ -788,7 +793,7 @@ func getListenerStatus( if gateway.Status.Listeners[i].AttachedRoutes != attachedRoutes { changed = true } - for _, condition := range conditions { + for _, condition := range status.Conditions { if !IsConditionPresentAndEqual(gateway.Status.Listeners[i].Conditions, condition) { changed = true break @@ -1089,3 +1094,30 @@ func isTLSSecretValid(secret *corev1.Secret) (string, bool) { } return "", true } + +func checkReferenceGrantBetweenGatewayAndSecret(gwNamespace string, certRef gatewayv1.SecretObjectReference, grants []v1beta1.ReferenceGrant) bool { + // if not cross namespaces + if certRef.Namespace == nil || string(*certRef.Namespace) == gwNamespace { + return true + } + + for _, grant := range grants { + if grant.Namespace == string(*certRef.Namespace) { + for _, from := range grant.Spec.From { + gw := v1beta1.ReferenceGrantFrom{ + Group: gatewayv1.GroupName, + Kind: KindGateway, + Namespace: v1beta1.Namespace(gwNamespace), + } + if from == gw { + for _, to := range grant.Spec.To { + if to.Group == corev1.GroupName && to.Kind == KindSecret && (to.Name == nil || *to.Name == certRef.Name) { + return true + } + } + } + } + } + } + return false +} diff --git a/internal/manager/run.go b/internal/manager/run.go index 769cd4f83..c8156ba91 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -19,7 +19,9 @@ import ( "time" "github.com/go-logr/logr" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" @@ -29,6 +31,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/apache/apisix-ingress-controller/api/v1alpha1" "github.com/apache/apisix-ingress-controller/internal/controller/config" @@ -48,6 +51,9 @@ func init() { if err := v1alpha1.AddToScheme(scheme); err != nil { panic(err) } + if err := v1beta1.Install(scheme); err != nil { + panic(err) + } // +kubebuilder:scaffold:scheme } @@ -177,6 +183,14 @@ func Run(ctx context.Context, logger logr.Logger) error { setupLog.Error(err, "unable to set up controllers") return err } + if _, err = mgr.GetRESTMapper().KindsFor(schema.GroupVersionResource{ + Group: v1beta1.GroupVersion.Group, + Version: v1beta1.GroupVersion.Version, + Resource: "referencegrants", + }); err != nil { + logger.Error(err, "CRD ReferenceGrants is not installed") + return errors.Wrap(err, "CRD ReferenceGrants is not installed") + } for _, c := range controllers { if err := c.SetupWithManager(mgr); err != nil { return err diff --git a/test/conformance/conformance_test.go b/test/conformance/conformance_test.go index bd6362fbe..c664f1fed 100644 --- a/test/conformance/conformance_test.go +++ b/test/conformance/conformance_test.go @@ -25,9 +25,6 @@ var skippedTestsForSSL = []string{ } var skippedTestsForTraditionalRoutes = []string{ - // TODO: Support ReferenceGrant resource - tests.GatewaySecretInvalidReferenceGrant.ShortName, - tests.GatewaySecretMissingReferenceGrant.ShortName, tests.HTTPRouteInvalidCrossNamespaceBackendRef.ShortName, tests.HTTPRouteInvalidReferenceGrant.ShortName, tests.HTTPRoutePartiallyInvalidViaInvalidReferenceGrant.ShortName, diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index fc63585f6..4aeabb893 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -229,6 +229,20 @@ rules: verbs: - get - update +- apiGroups: + - gateway.networking.k8s.io + resources: + - referencegrants + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - referencegrants/status + verbs: + - get - apiGroups: - networking.k8s.io resources: