From e23c5a16c2c230f078fca7accf254f5c0191be17 Mon Sep 17 00:00:00 2001 From: ashing Date: Mon, 14 Apr 2025 12:12:26 +0800 Subject: [PATCH 01/16] feat: process gateway proxy ref secret Signed-off-by: ashing --- internal/controller/gateway_controller.go | 32 ++++++ internal/controller/ingress_controller.go | 116 ++++++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 47e42fa3c..542e11947 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -285,6 +285,38 @@ func (r *GatewayReconciler) processInfrastructure(tctx *provider.TranslateContex } else { log.Info("found GatewayProxy for Gateway", "gateway", gateway.Name, "gatewayproxy", gatewayProxy.Name) tctx.GatewayProxy = gatewayProxy + + // Process provider secrets if provider exists + if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane { + if gatewayProxy.Spec.Provider.ControlPlane != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { + + secretRef := gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef + secret := &corev1.Secret{} + if err := r.Get(context.Background(), client.ObjectKey{ + Namespace: ns, + Name: secretRef.Name, + }, secret); err != nil { + log.Error(err, "failed to get secret for GatewayProxy provider", + "namespace", ns, + "name", secretRef.Name) + return err + } + + log.Info("found secret for GatewayProxy provider", + "gateway", gateway.Name, + "gatewayproxy", gatewayProxy.Name, + "secret", secretRef.Name) + + tctx.Secrets[types.NamespacedName{ + Namespace: ns, + Name: secretRef.Name, + }] = secret + } + } } } diff --git a/internal/controller/ingress_controller.go b/internal/controller/ingress_controller.go index 2e1636eed..4f0276b37 100644 --- a/internal/controller/ingress_controller.go +++ b/internal/controller/ingress_controller.go @@ -2,9 +2,11 @@ package controller import ( "context" + "errors" "fmt" "reflect" + "github.com/api7/api7-ingress-controller/api/v1alpha1" "github.com/api7/api7-ingress-controller/internal/controller/config" "github.com/api7/api7-ingress-controller/internal/controller/indexer" "github.com/api7/api7-ingress-controller/internal/provider" @@ -95,6 +97,12 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // create a translate context tctx := provider.NewDefaultTranslateContext() + // process IngressClass parameters if they reference GatewayProxy + if err := r.processIngressClassParameters(ctx, tctx, ingress); err != nil { + r.Log.Error(err, "failed to process IngressClass parameters", "ingress", ingress.Name) + return ctrl.Result{}, err + } + // process TLS configuration if err := r.processTLS(ctx, tctx, ingress); err != nil { r.Log.Error(err, "failed to process TLS configuration", "ingress", ingress.Name) @@ -122,6 +130,46 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } +// getIngressClass get the ingress class for the ingress +func (r *IngressReconciler) getIngressClass(obj client.Object) (*networkingv1.IngressClass, error) { + ingress := obj.(*networkingv1.Ingress) + + if ingress.Spec.IngressClassName == nil { + // handle the case where IngressClassName is not specified + // find all ingress classes and check if any of them is marked as default + ingressClassList := &networkingv1.IngressClassList{} + if err := r.List(context.Background(), ingressClassList, client.MatchingFields{ + indexer.IngressClass: config.GetControllerName(), + }); err != nil { + r.Log.Error(err, "failed to list ingress classes") + return nil, err + } + + // find the ingress class that is marked as default + for _, ic := range ingressClassList.Items { + if IsDefaultIngressClass(&ic) && matchesController(ic.Spec.Controller) { + log.Debugw("match the default ingress class") + return &ic, nil + } + } + + log.Debugw("no default ingress class found") + return nil, errors.New("no default ingress class found") + } + + // if it does not match, check if the ingress class is controlled by us + ingressClass := networkingv1.IngressClass{} + if err := r.Client.Get(context.Background(), client.ObjectKey{Name: *ingress.Spec.IngressClassName}, &ingressClass); err != nil { + return nil, err + } + + if matchesController(ingressClass.Spec.Controller) { + return &ingressClass, nil + } + + return nil, errors.New("ingress class is not controlled by us") +} + // checkIngressClass check if the ingress uses the ingress class that we control func (r *IngressReconciler) checkIngressClass(obj client.Object) bool { ingress := obj.(*networkingv1.Ingress) @@ -469,3 +517,71 @@ func (r *IngressReconciler) updateStatus(ctx context.Context, ingress *networkin return nil } + +// processIngressClassParameters processes the IngressClass parameters that reference GatewayProxy +func (r *IngressReconciler) processIngressClassParameters(ctx context.Context, tctx *provider.TranslateContext, ingress *networkingv1.Ingress) error { + ingressClass, err := r.getIngressClass(ingress) + if err != nil { + r.Log.Error(err, "failed to get IngressClass", "name", ingress.Spec.IngressClassName) + return err + } + + if ingressClass.Spec.Parameters == nil { + return nil + } + + parameters := ingressClass.Spec.Parameters + // check if the parameters reference GatewayProxy + if parameters.APIGroup != nil && *parameters.APIGroup == v1alpha1.GroupVersion.Group && parameters.Kind == "GatewayProxy" { + ns := ingress.GetNamespace() + if parameters.Namespace != nil { + ns = *parameters.Namespace + } + + gatewayProxy := &v1alpha1.GatewayProxy{} + if err := r.Get(ctx, client.ObjectKey{ + Namespace: ns, + Name: parameters.Name, + }, gatewayProxy); err != nil { + r.Log.Error(err, "failed to get GatewayProxy", "namespace", ns, "name", parameters.Name) + return err + } + + r.Log.Info("found GatewayProxy for IngressClass", "ingressClass", ingressClass.Name, "gatewayproxy", gatewayProxy.Name) + tctx.GatewayProxy = gatewayProxy + + // check if the provider field references a secret + if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane { + if gatewayProxy.Spec.Provider.ControlPlane != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { + + secretRef := gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef + secret := &corev1.Secret{} + if err := r.Get(ctx, client.ObjectKey{ + Namespace: ns, + Name: secretRef.Name, + }, secret); err != nil { + r.Log.Error(err, "failed to get secret for GatewayProxy provider", + "namespace", ns, + "name", secretRef.Name) + return err + } + + r.Log.Info("found secret for GatewayProxy provider", + "ingressClass", ingressClass.Name, + "gatewayproxy", gatewayProxy.Name, + "secret", secretRef.Name) + + tctx.Secrets[types.NamespacedName{ + Namespace: ns, + Name: secretRef.Name, + }] = secret + } + } + } + + return nil +} From e59fb85057133e8b9001cdb5013a5d778b93b31b Mon Sep 17 00:00:00 2001 From: ashing Date: Mon, 14 Apr 2025 17:12:09 +0800 Subject: [PATCH 02/16] fix: r Signed-off-by: ashing --- internal/provider/adc/adc.go | 124 +++++++++++++++++++++++++++++++--- internal/provider/provider.go | 1 + 2 files changed, 116 insertions(+), 9 deletions(-) diff --git a/internal/provider/adc/adc.go b/internal/provider/adc/adc.go index 285de1a1a..8c0066696 100644 --- a/internal/provider/adc/adc.go +++ b/internal/provider/adc/adc.go @@ -13,7 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - types "github.com/api7/api7-ingress-controller/api/adc" + adctypes "github.com/api7/api7-ingress-controller/api/adc" "github.com/api7/api7-ingress-controller/api/v1alpha1" "github.com/api7/api7-ingress-controller/internal/controller/config" "github.com/api7/api7-ingress-controller/internal/controller/label" @@ -22,19 +22,32 @@ import ( "github.com/api7/gopkg/pkg/log" ) +type ResourceKind struct { + Kind string + Namespace string + Name string +} + +type adcConfig struct { + ServerAddr string + Token string +} + type adcClient struct { translator *translator.Translator ServerAddr string Token string GatewayGroup string + configs map[ResourceKind][]adcConfig } type Task struct { Name string - Resources types.Resources + Resources adctypes.Resources Labels map[string]string ResourceTypes []string + configs []adcConfig } func New() (provider.Provider, error) { @@ -43,9 +56,32 @@ func New() (provider.Provider, error) { translator: &translator.Translator{}, ServerAddr: gc.ControlPlane.Endpoints[0], Token: gc.ControlPlane.AdminKey, + configs: make(map[ResourceKind][]adcConfig), }, nil } +func (d *adcClient) getConfigs(rk ResourceKind) []adcConfig { + return d.configs[rk] +} + +func (d *adcClient) updateGatewayConfigs(rk ResourceKind, tctx *provider.TranslateContext) ([]adcConfig, error) { + // get gateway proxy from tctx + return nil, nil +} + +func (d *adcClient) updateIngressConfigs(rk ResourceKind, tctx *provider.TranslateContext) ([]adcConfig, error) { + // get gateway proxy from tctx + return nil, nil +} + +func (d *adcClient) updateHTTPRouteConfigs(rk ResourceKind, tctx *provider.TranslateContext) ([]adcConfig, error) { + return nil, nil +} + +func (d *adcClient) updateConsumerConfigs(rk ResourceKind, tctx *provider.TranslateContext) ([]adcConfig, error) { + return nil, nil +} + func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, obj client.Object) error { log.Debugw("updating object", zap.Any("object", obj)) var ( @@ -54,22 +90,55 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, err error ) + var configs []adcConfig + + rk := ResourceKind{ + Kind: obj.GetObjectKind().GroupVersionKind().Kind, + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + switch t := obj.(type) { case *gatewayv1.HTTPRoute: result, err = d.translator.TranslateHTTPRoute(tctx, t.DeepCopy()) + if err != nil { + return err + } resourceTypes = append(resourceTypes, "service") + configs, err = d.updateHTTPRouteConfigs(rk, tctx) + if err != nil { + return err + } case *gatewayv1.Gateway: result, err = d.translator.TranslateGateway(tctx, t.DeepCopy()) + if err != nil { + return err + } resourceTypes = append(resourceTypes, "global_rule", "ssl", "plugin_metadata") + configs, err = d.updateGatewayConfigs(rk, tctx) + if err != nil { + return err + } case *networkingv1.Ingress: result, err = d.translator.TranslateIngress(tctx, t.DeepCopy()) + if err != nil { + return err + } resourceTypes = append(resourceTypes, "service", "ssl") + configs, err = d.updateIngressConfigs(rk, tctx) + if err != nil { + return err + } case *v1alpha1.Consumer: result, err = d.translator.TranslateConsumerV1alpha1(tctx, t.DeepCopy()) + if err != nil { + return err + } resourceTypes = append(resourceTypes, "consumer") - } - if err != nil { - return err + configs, err = d.updateConsumerConfigs(rk, tctx) + if err != nil { + return err + } } if result == nil { return nil @@ -78,7 +147,7 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, return d.sync(Task{ Name: obj.GetName(), Labels: label.GenLabel(obj), - Resources: types.Resources{ + Resources: adctypes.Resources{ GlobalRules: result.GlobalRules, PluginMetadata: result.PluginMetadata, Services: result.Services, @@ -86,6 +155,7 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, Consumers: result.Consumers, }, ResourceTypes: resourceTypes, + configs: configs, }) } @@ -108,10 +178,19 @@ func (d *adcClient) Delete(ctx context.Context, obj client.Object) error { labels = label.GenLabel(obj) } + rk := ResourceKind{ + Kind: obj.GetObjectKind().GroupVersionKind().Kind, + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + + configs := d.getConfigs(rk) + return d.sync(Task{ Name: obj.GetName(), Labels: labels, ResourceTypes: resourceTypes, + configs: configs, }) } @@ -150,12 +229,39 @@ func (d *adcClient) sync(task Task) error { args = append(args, "--include-resource-type", t) } + // todo: use adc config + // for _, config := range task.configs { + // if err := d.execADC(config, args); err != nil { + // return err + // } + // } + if err := d.execADC(adcConfig{ + ServerAddr: d.ServerAddr, + Token: d.Token, + }, args); err != nil { + return err + } + + return nil +} + +func (d *adcClient) execADC(config adcConfig, args []string) error { + // todo: use adc config + serverAddr := d.ServerAddr + if config.ServerAddr != "" { + serverAddr = config.ServerAddr + } + token := d.Token + if config.Token != "" { + token = config.Token + } + adcEnv := []string{ "ADC_EXPERIMENTAL_FEATURE_FLAGS=remote-state-file,parallel-backend-request", "ADC_RUNNING_MODE=ingress", "ADC_BACKEND=api7ee", - "ADC_SERVER=" + d.ServerAddr, - "ADC_TOKEN=" + d.Token, + "ADC_SERVER=" + serverAddr, + "ADC_TOKEN=" + token, } var stdout, stderr bytes.Buffer @@ -167,7 +273,7 @@ func (d *adcClient) sync(task Task) error { log.Debug("running adc command", zap.String("command", cmd.String()), zap.Strings("env", adcEnv)) - var result types.SyncResult + var result adctypes.SyncResult if err := cmd.Run(); err != nil { stderrStr := stderr.String() stdoutStr := stdout.String() diff --git a/internal/provider/provider.go b/internal/provider/provider.go index a57675f27..f2555eabe 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -22,6 +22,7 @@ type TranslateContext struct { GatewayTLSConfig []gatewayv1.GatewayTLSConfig GatewayProxy *v1alpha1.GatewayProxy Credentials []v1alpha1.Credential + Gateways []gatewayv1.Gateway EndpointSlices map[types.NamespacedName][]discoveryv1.EndpointSlice Secrets map[types.NamespacedName]*corev1.Secret PluginConfigs map[types.NamespacedName]*v1alpha1.PluginConfig From 5cf5ec73a2a91fb4d48ac2a7a4be48eb4c3ecd16 Mon Sep 17 00:00:00 2001 From: ashing Date: Mon, 14 Apr 2025 23:43:46 +0800 Subject: [PATCH 03/16] fix: r Signed-off-by: ashing --- internal/controller/consumer_controller.go | 27 +++++ internal/controller/gateway_controller.go | 55 +--------- internal/controller/httproute_controller.go | 6 ++ internal/controller/ingress_controller.go | 2 +- internal/controller/utils.go | 64 ++++++++++++ internal/provider/adc/adc.go | 108 +++++++++++++------- internal/provider/adc/translator/gateway.go | 10 +- internal/provider/provider.go | 2 +- 8 files changed, 176 insertions(+), 98 deletions(-) diff --git a/internal/controller/consumer_controller.go b/internal/controller/consumer_controller.go index 1d5054aca..bcf72fb70 100644 --- a/internal/controller/consumer_controller.go +++ b/internal/controller/consumer_controller.go @@ -138,6 +138,17 @@ func (r *ConsumerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c var statusErr error tctx := provider.NewDefaultTranslateContext() + gateway, err := r.getGateway(ctx, consumer) + if err != nil { + r.Log.Error(err, "failed to get gateway", "consumer", consumer) + statusErr = err + } + + if err := ProcessGatewayProxy(r.Client, tctx, gateway); err != nil { + r.Log.Error(err, "failed to process gateway proxy", "gateway", gateway) + statusErr = err + } + if err := r.processSpec(ctx, tctx, consumer); err != nil { r.Log.Error(err, "failed to process consumer spec", "consumer", consumer) statusErr = err @@ -201,6 +212,22 @@ func (r *ConsumerReconciler) updateStatus(ctx context.Context, consumer *v1alpha return nil } +func (r *ConsumerReconciler) getGateway(ctx context.Context, consumer *v1alpha1.Consumer) (*gatewayv1.Gateway, error) { + ns := consumer.GetNamespace() + if consumer.Spec.GatewayRef.Namespace != nil { + ns = *consumer.Spec.GatewayRef.Namespace + } + gateway := &gatewayv1.Gateway{} + if err := r.Get(ctx, client.ObjectKey{ + Name: consumer.Spec.GatewayRef.Name, + Namespace: ns, + }, gateway); err != nil { + r.Log.Error(err, "failed to get gateway", "gateway", consumer.Spec.GatewayRef.Name) + return nil, err + } + return gateway, nil +} + func (r *ConsumerReconciler) checkGatewayRef(object client.Object) bool { consumer, ok := object.(*v1alpha1.Consumer) if !ok { diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 542e11947..014e2b303 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -267,60 +267,7 @@ func (r *GatewayReconciler) listGatewaysForHTTPRoute(_ context.Context, obj clie } func (r *GatewayReconciler) processInfrastructure(tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) error { - infra := gateway.Spec.Infrastructure - if infra == nil || infra.ParametersRef == nil { - return nil - } - - ns := gateway.GetNamespace() - paramRef := infra.ParametersRef - if string(paramRef.Group) == v1alpha1.GroupVersion.Group && string(paramRef.Kind) == "GatewayProxy" { - gatewayProxy := &v1alpha1.GatewayProxy{} - if err := r.Get(context.Background(), client.ObjectKey{ - Namespace: ns, - Name: paramRef.Name, - }, gatewayProxy); err != nil { - log.Error(err, "failed to get GatewayProxy", "namespace", ns, "name", paramRef.Name) - return err - } else { - log.Info("found GatewayProxy for Gateway", "gateway", gateway.Name, "gatewayproxy", gatewayProxy.Name) - tctx.GatewayProxy = gatewayProxy - - // Process provider secrets if provider exists - if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane { - if gatewayProxy.Spec.Provider.ControlPlane != nil && - gatewayProxy.Spec.Provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey && - gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey != nil && - gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && - gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { - - secretRef := gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef - secret := &corev1.Secret{} - if err := r.Get(context.Background(), client.ObjectKey{ - Namespace: ns, - Name: secretRef.Name, - }, secret); err != nil { - log.Error(err, "failed to get secret for GatewayProxy provider", - "namespace", ns, - "name", secretRef.Name) - return err - } - - log.Info("found secret for GatewayProxy provider", - "gateway", gateway.Name, - "gatewayproxy", gatewayProxy.Name, - "secret", secretRef.Name) - - tctx.Secrets[types.NamespacedName{ - Namespace: ns, - Name: secretRef.Name, - }] = secret - } - } - } - } - - return nil + return ProcessGatewayProxy(r.Client, tctx, gateway) } func (r *GatewayReconciler) processListenerConfig(tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) { diff --git a/internal/controller/httproute_controller.go b/internal/controller/httproute_controller.go index afe21a884..a4ccf0060 100644 --- a/internal/controller/httproute_controller.go +++ b/internal/controller/httproute_controller.go @@ -114,6 +114,12 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( tctx := provider.NewDefaultTranslateContext() + for _, gateway := range gateways { + if err := ProcessGatewayProxy(r.Client, tctx, gateway.Gateway); err != nil { + return ctrl.Result{}, err + } + } + if err := r.processHTTPRoute(tctx, hr); err != nil { acceptStatus.status = false acceptStatus.msg = err.Error() diff --git a/internal/controller/ingress_controller.go b/internal/controller/ingress_controller.go index 4f0276b37..5067befc2 100644 --- a/internal/controller/ingress_controller.go +++ b/internal/controller/ingress_controller.go @@ -548,7 +548,7 @@ func (r *IngressReconciler) processIngressClassParameters(ctx context.Context, t } r.Log.Info("found GatewayProxy for IngressClass", "ingressClass", ingressClass.Name, "gatewayproxy", gatewayProxy.Name) - tctx.GatewayProxy = gatewayProxy + tctx.GatewayProxies = append(tctx.GatewayProxies, *gatewayProxy) // check if the provider field references a secret if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane { diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 9282117ea..6938be2da 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -5,12 +5,16 @@ import ( "fmt" "strings" + "github.com/api7/api7-ingress-controller/api/v1alpha1" "github.com/api7/api7-ingress-controller/internal/controller/config" + "github.com/api7/api7-ingress-controller/internal/provider" + "github.com/api7/gopkg/pkg/log" "github.com/samber/lo" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -759,3 +763,63 @@ func SplitMetaNamespaceKey(key string) (namespace, name string, err error) { return "", "", fmt.Errorf("unexpected key format: %q", key) } + +func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) error { + if gateway == nil { + return nil + } + infra := gateway.Spec.Infrastructure + if infra == nil || infra.ParametersRef == nil { + return nil + } + + ns := gateway.GetNamespace() + paramRef := infra.ParametersRef + if string(paramRef.Group) == v1alpha1.GroupVersion.Group && string(paramRef.Kind) == "GatewayProxy" { + gatewayProxy := &v1alpha1.GatewayProxy{} + if err := r.Get(context.Background(), client.ObjectKey{ + Namespace: ns, + Name: paramRef.Name, + }, gatewayProxy); err != nil { + log.Error(err, "failed to get GatewayProxy", "namespace", ns, "name", paramRef.Name) + return err + } else { + log.Info("found GatewayProxy for Gateway", "gateway", gateway.Name, "gatewayproxy", gatewayProxy.Name) + tctx.GatewayProxies = append(tctx.GatewayProxies, *gatewayProxy) + + // Process provider secrets if provider exists + if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane { + if gatewayProxy.Spec.Provider.ControlPlane != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { + + secretRef := gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef + secret := &corev1.Secret{} + if err := r.Get(context.Background(), client.ObjectKey{ + Namespace: ns, + Name: secretRef.Name, + }, secret); err != nil { + log.Error(err, "failed to get secret for GatewayProxy provider", + "namespace", ns, + "name", secretRef.Name) + return err + } + + log.Info("found secret for GatewayProxy provider", + "gateway", gateway.Name, + "gatewayproxy", gatewayProxy.Name, + "secret", secretRef.Name) + + tctx.Secrets[types.NamespacedName{ + Namespace: ns, + Name: secretRef.Name, + }] = secret + } + } + } + } + + return nil +} diff --git a/internal/provider/adc/adc.go b/internal/provider/adc/adc.go index 8c0066696..547883435 100644 --- a/internal/provider/adc/adc.go +++ b/internal/provider/adc/adc.go @@ -20,6 +20,7 @@ import ( "github.com/api7/api7-ingress-controller/internal/provider" "github.com/api7/api7-ingress-controller/internal/provider/adc/translator" "github.com/api7/gopkg/pkg/log" + "k8s.io/apimachinery/pkg/types" ) type ResourceKind struct { @@ -64,22 +65,64 @@ func (d *adcClient) getConfigs(rk ResourceKind) []adcConfig { return d.configs[rk] } -func (d *adcClient) updateGatewayConfigs(rk ResourceKind, tctx *provider.TranslateContext) ([]adcConfig, error) { - // get gateway proxy from tctx - return nil, nil -} +func (d *adcClient) getConfigsForGatewayProxy(rk ResourceKind, tctx *provider.TranslateContext, gatewayProxy *v1alpha1.GatewayProxy) (*adcConfig, error) { + if gatewayProxy == nil || gatewayProxy.Spec.Provider == nil { + return nil, nil + } -func (d *adcClient) updateIngressConfigs(rk ResourceKind, tctx *provider.TranslateContext) ([]adcConfig, error) { - // get gateway proxy from tctx - return nil, nil -} + provider := gatewayProxy.Spec.Provider + if provider.Type != v1alpha1.ProviderTypeControlPlane || provider.ControlPlane == nil { + return nil, nil + } + + endpoints := provider.ControlPlane.Endpoints + if len(endpoints) == 0 { + return nil, errors.New("no endpoints found") + } + + endpoint := endpoints[0] + config := adcConfig{ + ServerAddr: endpoint, + } + + if provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey && provider.ControlPlane.Auth.AdminKey != nil { + if provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { + secretRef := provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef + secret, ok := tctx.Secrets[types.NamespacedName{ + Namespace: rk.Namespace, + Name: secretRef.Name, + }] + if ok { + if token, ok := secret.Data[secretRef.Key]; ok { + config.Token = string(token) + } + } + } else if provider.ControlPlane.Auth.AdminKey.Value != "" { + config.Token = provider.ControlPlane.Auth.AdminKey.Value + } + } -func (d *adcClient) updateHTTPRouteConfigs(rk ResourceKind, tctx *provider.TranslateContext) ([]adcConfig, error) { - return nil, nil + if config.Token == "" { + return nil, errors.New("no token found") + } + + return &config, nil } -func (d *adcClient) updateConsumerConfigs(rk ResourceKind, tctx *provider.TranslateContext) ([]adcConfig, error) { - return nil, nil +func (d *adcClient) updateConfigs(rk ResourceKind, tctx *provider.TranslateContext) error { + var configs []adcConfig + for _, gatewayProxy := range tctx.GatewayProxies { + config, err := d.getConfigsForGatewayProxy(rk, tctx, &gatewayProxy) + if err != nil { + return err + } + if config != nil { + configs = append(configs, *config) + } + } + + d.configs[rk] = configs + return nil } func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, obj client.Object) error { @@ -90,60 +133,47 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, err error ) - var configs []adcConfig - - rk := ResourceKind{ - Kind: obj.GetObjectKind().GroupVersionKind().Kind, - Namespace: obj.GetNamespace(), - Name: obj.GetName(), - } - switch t := obj.(type) { case *gatewayv1.HTTPRoute: result, err = d.translator.TranslateHTTPRoute(tctx, t.DeepCopy()) - if err != nil { - return err - } resourceTypes = append(resourceTypes, "service") - configs, err = d.updateHTTPRouteConfigs(rk, tctx) - if err != nil { - return err - } case *gatewayv1.Gateway: result, err = d.translator.TranslateGateway(tctx, t.DeepCopy()) if err != nil { return err } resourceTypes = append(resourceTypes, "global_rule", "ssl", "plugin_metadata") - configs, err = d.updateGatewayConfigs(rk, tctx) - if err != nil { - return err - } case *networkingv1.Ingress: result, err = d.translator.TranslateIngress(tctx, t.DeepCopy()) if err != nil { return err } resourceTypes = append(resourceTypes, "service", "ssl") - configs, err = d.updateIngressConfigs(rk, tctx) - if err != nil { - return err - } case *v1alpha1.Consumer: result, err = d.translator.TranslateConsumerV1alpha1(tctx, t.DeepCopy()) if err != nil { return err } resourceTypes = append(resourceTypes, "consumer") - configs, err = d.updateConsumerConfigs(rk, tctx) - if err != nil { - return err - } + } + if err != nil { + return err } if result == nil { return nil } + // update adc configs + rk := ResourceKind{ + Kind: obj.GetObjectKind().GroupVersionKind().Kind, + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + if err := d.updateConfigs(rk, tctx); err != nil { + return err + } + configs := d.getConfigs(rk) + return d.sync(Task{ Name: obj.GetName(), Labels: label.GenLabel(obj), diff --git a/internal/provider/adc/translator/gateway.go b/internal/provider/adc/translator/gateway.go index 64835c7a2..cb325bd4e 100644 --- a/internal/provider/adc/translator/gateway.go +++ b/internal/provider/adc/translator/gateway.go @@ -32,14 +32,18 @@ func (t *Translator) TranslateGateway(tctx *provider.TranslateContext, obj *gate result.SSL = append(result.SSL, ssl...) } } - if tctx.GatewayProxy != nil { + var gatewayProxy *v1alpha1.GatewayProxy + if len(tctx.GatewayProxies) > 0 { + gatewayProxy = &tctx.GatewayProxies[0] + } + if gatewayProxy != nil { var ( globalRules = adctypes.Plugins{} pluginMetadata = adctypes.Plugins{} ) // apply plugins from GatewayProxy to global rules - t.fillPluginsFromGatewayProxy(globalRules, tctx.GatewayProxy) - t.fillPluginMetadataFromGatewayProxy(pluginMetadata, tctx.GatewayProxy) + t.fillPluginsFromGatewayProxy(globalRules, gatewayProxy) + t.fillPluginMetadataFromGatewayProxy(pluginMetadata, gatewayProxy) result.GlobalRules = globalRules result.PluginMetadata = pluginMetadata } diff --git a/internal/provider/provider.go b/internal/provider/provider.go index f2555eabe..65799d11f 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -20,7 +20,7 @@ type Provider interface { type TranslateContext struct { BackendRefs []gatewayv1.BackendRef GatewayTLSConfig []gatewayv1.GatewayTLSConfig - GatewayProxy *v1alpha1.GatewayProxy + GatewayProxies []v1alpha1.GatewayProxy Credentials []v1alpha1.Credential Gateways []gatewayv1.Gateway EndpointSlices map[types.NamespacedName][]discoveryv1.EndpointSlice From 3b06ad3c0541b1d54ee1ffa85e4a6fafb84f65f2 Mon Sep 17 00:00:00 2001 From: ashing Date: Mon, 14 Apr 2025 23:54:32 +0800 Subject: [PATCH 04/16] fix: r Signed-off-by: ashing --- internal/controller/httproute_controller.go | 3 ++- internal/provider/adc/adc.go | 16 ++++------------ internal/provider/provider.go | 1 - 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/internal/controller/httproute_controller.go b/internal/controller/httproute_controller.go index a4ccf0060..638083adc 100644 --- a/internal/controller/httproute_controller.go +++ b/internal/controller/httproute_controller.go @@ -116,7 +116,8 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( for _, gateway := range gateways { if err := ProcessGatewayProxy(r.Client, tctx, gateway.Gateway); err != nil { - return ctrl.Result{}, err + acceptStatus.status = false + acceptStatus.msg = err.Error() } } diff --git a/internal/provider/adc/adc.go b/internal/provider/adc/adc.go index 547883435..7904cbe3b 100644 --- a/internal/provider/adc/adc.go +++ b/internal/provider/adc/adc.go @@ -65,7 +65,7 @@ func (d *adcClient) getConfigs(rk ResourceKind) []adcConfig { return d.configs[rk] } -func (d *adcClient) getConfigsForGatewayProxy(rk ResourceKind, tctx *provider.TranslateContext, gatewayProxy *v1alpha1.GatewayProxy) (*adcConfig, error) { +func (d *adcClient) getConfigsForGatewayProxy(tctx *provider.TranslateContext, gatewayProxy *v1alpha1.GatewayProxy) (*adcConfig, error) { if gatewayProxy == nil || gatewayProxy.Spec.Provider == nil { return nil, nil } @@ -89,7 +89,8 @@ func (d *adcClient) getConfigsForGatewayProxy(rk ResourceKind, tctx *provider.Tr if provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { secretRef := provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef secret, ok := tctx.Secrets[types.NamespacedName{ - Namespace: rk.Namespace, + // we should use gateway proxy namespace + Namespace: gatewayProxy.GetNamespace(), Name: secretRef.Name, }] if ok { @@ -112,7 +113,7 @@ func (d *adcClient) getConfigsForGatewayProxy(rk ResourceKind, tctx *provider.Tr func (d *adcClient) updateConfigs(rk ResourceKind, tctx *provider.TranslateContext) error { var configs []adcConfig for _, gatewayProxy := range tctx.GatewayProxies { - config, err := d.getConfigsForGatewayProxy(rk, tctx, &gatewayProxy) + config, err := d.getConfigsForGatewayProxy(tctx, &gatewayProxy) if err != nil { return err } @@ -139,21 +140,12 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, resourceTypes = append(resourceTypes, "service") case *gatewayv1.Gateway: result, err = d.translator.TranslateGateway(tctx, t.DeepCopy()) - if err != nil { - return err - } resourceTypes = append(resourceTypes, "global_rule", "ssl", "plugin_metadata") case *networkingv1.Ingress: result, err = d.translator.TranslateIngress(tctx, t.DeepCopy()) - if err != nil { - return err - } resourceTypes = append(resourceTypes, "service", "ssl") case *v1alpha1.Consumer: result, err = d.translator.TranslateConsumerV1alpha1(tctx, t.DeepCopy()) - if err != nil { - return err - } resourceTypes = append(resourceTypes, "consumer") } if err != nil { diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 65799d11f..5235d2c7f 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -22,7 +22,6 @@ type TranslateContext struct { GatewayTLSConfig []gatewayv1.GatewayTLSConfig GatewayProxies []v1alpha1.GatewayProxy Credentials []v1alpha1.Credential - Gateways []gatewayv1.Gateway EndpointSlices map[types.NamespacedName][]discoveryv1.EndpointSlice Secrets map[types.NamespacedName]*corev1.Secret PluginConfigs map[types.NamespacedName]*v1alpha1.PluginConfig From 63075e11f3405baedfea28b42da75cf8f7d1270f Mon Sep 17 00:00:00 2001 From: ashing Date: Tue, 15 Apr 2025 10:34:25 +0800 Subject: [PATCH 05/16] fix: r Signed-off-by: ashing --- internal/provider/adc/adc.go | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/internal/provider/adc/adc.go b/internal/provider/adc/adc.go index 7904cbe3b..5660850fa 100644 --- a/internal/provider/adc/adc.go +++ b/internal/provider/adc/adc.go @@ -7,6 +7,7 @@ import ( "errors" "os" "os/exec" + "sync" "go.uber.org/zap" networkingv1 "k8s.io/api/networking/v1" @@ -41,6 +42,7 @@ type adcClient struct { Token string GatewayGroup string configs map[ResourceKind][]adcConfig + sync.Mutex } type Task struct { @@ -61,10 +63,6 @@ func New() (provider.Provider, error) { }, nil } -func (d *adcClient) getConfigs(rk ResourceKind) []adcConfig { - return d.configs[rk] -} - func (d *adcClient) getConfigsForGatewayProxy(tctx *provider.TranslateContext, gatewayProxy *v1alpha1.GatewayProxy) (*adcConfig, error) { if gatewayProxy == nil || gatewayProxy.Spec.Provider == nil { return nil, nil @@ -110,7 +108,22 @@ func (d *adcClient) getConfigsForGatewayProxy(tctx *provider.TranslateContext, g return &config, nil } +func (d *adcClient) deleteConfigs(rk ResourceKind) { + d.Lock() + defer d.Unlock() + delete(d.configs, rk) +} + +func (d *adcClient) getConfigs(rk ResourceKind) []adcConfig { + d.Lock() + defer d.Unlock() + return d.configs[rk] +} + func (d *adcClient) updateConfigs(rk ResourceKind, tctx *provider.TranslateContext) error { + d.Lock() + defer d.Unlock() + var configs []adcConfig for _, gatewayProxy := range tctx.GatewayProxies { config, err := d.getConfigsForGatewayProxy(tctx, &gatewayProxy) @@ -208,12 +221,18 @@ func (d *adcClient) Delete(ctx context.Context, obj client.Object) error { configs := d.getConfigs(rk) - return d.sync(Task{ + err := d.sync(Task{ Name: obj.GetName(), Labels: labels, ResourceTypes: resourceTypes, configs: configs, }) + if err != nil { + return err + } + + d.deleteConfigs(rk) + return nil } func (d *adcClient) sync(task Task) error { From 1774afbc18804597cde88bfc6c0b70a5574fe818 Mon Sep 17 00:00:00 2001 From: ashing Date: Tue, 15 Apr 2025 18:15:54 +0800 Subject: [PATCH 06/16] fix: r Signed-off-by: ashing --- api/v1alpha1/gatewayproxy_types.go | 4 ++ config/samples/config.yaml | 6 +- internal/controller/ingress_controller.go | 2 + internal/provider/adc/adc.go | 27 +++++--- test/e2e/ingress/ingress.go | 84 +++++++++++++++++++++++ 5 files changed, 109 insertions(+), 14 deletions(-) diff --git a/api/v1alpha1/gatewayproxy_types.go b/api/v1alpha1/gatewayproxy_types.go index cb0404213..45f534af6 100644 --- a/api/v1alpha1/gatewayproxy_types.go +++ b/api/v1alpha1/gatewayproxy_types.go @@ -113,6 +113,10 @@ type ControlPlaneProvider struct { // +kubebuilder:validation:MinItems=1 Endpoints []string `json:"endpoints"` + // TlsVerify specifies whether to verify the TLS certificate of the control plane + // +optional + TlsVerify *bool `json:"tlsVerify,omitempty"` + // Auth specifies the authentication configuration // +kubebuilder:validation:Required Auth ControlPlaneAuth `json:"auth"` diff --git a/config/samples/config.yaml b/config/samples/config.yaml index 3bfe2f3ab..a158ecbf4 100644 --- a/config/samples/config.yaml +++ b/config/samples/config.yaml @@ -1,4 +1,4 @@ -log_level: "info" # The log level of the API7 Ingress Controller. +log_level: "debug" # The log level of the API7 Ingress Controller. # the default value is "info". controller_name: gateway.api7.io/api7-ingress-controller # The controller name of the API7 Ingress Controller, @@ -12,9 +12,9 @@ ingress_status_address: [] # The status address of the ingress. gateway_configs: # The configuration of the API7 Gateway. - name: api7 # The name of the Gateway in the Gateway API. control_plane: - admin_key: "${ADMIN_KEY}" # The admin key of the control plane. + admin_key: "a7adm-qqEncKkktQFWrSdF8bGpDzjcRDbVFYgRXSBSPCzpVZZkc7pags-70be7645fdf94c3c8e4d656c2334bd8b" # The admin key of the control plane. endpoints: - - ${ENDPOINT} # The endpoint of the control plane. + - https://172.18.0.6:7443 # The endpoint of the control plane. tls_verify: false addresses: # record the status address of the gateway-api gateway - "172.18.0.4" # The LB IP of the gateway service. diff --git a/internal/controller/ingress_controller.go b/internal/controller/ingress_controller.go index 5067befc2..4cf6d8294 100644 --- a/internal/controller/ingress_controller.go +++ b/internal/controller/ingress_controller.go @@ -461,6 +461,8 @@ func (r *IngressReconciler) processBackendService(ctx context.Context, tctx *pro func (r *IngressReconciler) updateStatus(ctx context.Context, ingress *networkingv1.Ingress) error { var loadBalancerStatus networkingv1.IngressLoadBalancerStatus + // todo: remove using default config, use the StatusAddress And PublishService in the gateway proxy + // 1. use the IngressStatusAddress in the config statusAddresses := config.GetIngressStatusAddress() if len(statusAddresses) > 0 { diff --git a/internal/provider/adc/adc.go b/internal/provider/adc/adc.go index 5660850fa..2b423a68c 100644 --- a/internal/provider/adc/adc.go +++ b/internal/provider/adc/adc.go @@ -270,17 +270,22 @@ func (d *adcClient) sync(task Task) error { args = append(args, "--include-resource-type", t) } - // todo: use adc config - // for _, config := range task.configs { - // if err := d.execADC(config, args); err != nil { - // return err - // } - // } - if err := d.execADC(adcConfig{ - ServerAddr: d.ServerAddr, - Token: d.Token, - }, args); err != nil { - return err + if len(task.configs) > 0 { + log.Debugw("syncing resources with multiple configs", zap.Any("configs", task.configs)) + for _, config := range task.configs { + if err := d.execADC(config, args); err != nil { + return err + } + } + } else { + // todo: remove using default config + log.Debugw("syncing resources with default config") + if err := d.execADC(adcConfig{ + ServerAddr: d.ServerAddr, + Token: d.Token, + }, args); err != nil { + return err + } } return nil diff --git a/test/e2e/ingress/ingress.go b/test/e2e/ingress/ingress.go index c5ebb119d..916dc5fef 100644 --- a/test/e2e/ingress/ingress.go +++ b/test/e2e/ingress/ingress.go @@ -188,4 +188,88 @@ spec: Status(200) }) }) + + Context("IngressClass with GatewayProxy", func() { + gatewayProxyYaml := ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config + namespace: default +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" +` + + var ingressClassWithProxy = ` +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: api7-with-proxy + annotations: + ingressclass.kubernetes.io/is-default-class: "true" +spec: + controller: "gateway.api7.io/api7-ingress-controller" + parameters: + apiGroup: "gateway.apisix.io" + kind: "GatewayProxy" + name: "api7-proxy-config" + namespace: "default" + scope: "Namespace" +` + + var testIngress = ` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: api7-ingress-with-proxy +spec: + ingressClassName: api7-with-proxy + rules: + - host: proxy.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: httpbin-service-e2e-test + port: + number: 80 +` + + It("Test IngressClass with GatewayProxy", func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) + + By("create GatewayProxy") + err := s.CreateResourceFromStringWithNamespace(gatewayProxy, "default") + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create IngressClass with GatewayProxy reference") + err = s.CreateResourceFromStringWithNamespace(ingressClassWithProxy, "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass with GatewayProxy") + time.Sleep(5 * time.Second) + + By("create Ingress with GatewayProxy IngressClass") + err = s.CreateResourceFromString(testIngress) + Expect(err).NotTo(HaveOccurred(), "creating Ingress with GatewayProxy IngressClass") + time.Sleep(5 * time.Second) + + By("verify HTTP request") + s.NewAPISIXClient(). + GET("/get"). + WithHost("proxy.example.com"). + Expect(). + Status(200) + }) + }) }) From 8fbfd08dc808d76ca42f8af7f09b1ba4c6963bce Mon Sep 17 00:00:00 2001 From: ashing Date: Tue, 15 Apr 2025 18:37:53 +0800 Subject: [PATCH 07/16] fix: r Signed-off-by: ashing --- test/e2e/gatewayapi/gatewayproxy.go | 73 +++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/test/e2e/gatewayapi/gatewayproxy.go b/test/e2e/gatewayapi/gatewayproxy.go index 37bb39dd2..723a0704a 100644 --- a/test/e2e/gatewayapi/gatewayproxy.go +++ b/test/e2e/gatewayapi/gatewayproxy.go @@ -314,4 +314,77 @@ spec: Body().Contains(`{"error_msg":"404 Route Not Found"}`) }) }) + + var ( + gatewayProxyWithInvalidProviderType = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config +spec: + provider: + type: "InvalidType" +` + gatewayProxyWithMissingControlPlane = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config +spec: + provider: + type: "ControlPlane" +` + gatewayProxyWithValidProvider = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config +spec: + provider: + type: "ControlPlane" + controlPlane: + endpoints: + - "http://localhost:9180" + auth: + type: "AdminKey" + adminKey: + value: "test-key" +` + ) + + Context("Test GatewayProxy Provider Validation", func() { + AfterEach(func() { + By("Clean up GatewayProxy resources") + _ = s.DeleteResourceFromString(gatewayProxyWithInvalidProviderType) + _ = s.DeleteResourceFromString(gatewayProxyWithMissingControlPlane) + _ = s.DeleteResourceFromString(gatewayProxyWithValidProvider) + }) + + It("Should reject invalid provider type", func() { + By("Create GatewayProxy with invalid provider type") + err := s.CreateResourceFromString(gatewayProxyWithInvalidProviderType) + Expect(err).To(HaveOccurred(), "creating GatewayProxy with invalid provider type") + Expect(err.Error()).To(ContainSubstring("Invalid value")) + }) + + It("Should reject missing controlPlane configuration", func() { + By("Create GatewayProxy with missing controlPlane") + err := s.CreateResourceFromString(gatewayProxyWithMissingControlPlane) + Expect(err).To(HaveOccurred(), "creating GatewayProxy with missing controlPlane") + Expect(err.Error()).To(ContainSubstring("controlPlane must be specified when type is ControlPlane")) + }) + + It("Should accept valid provider configuration", func() { + By("Create GatewayProxy with valid provider") + err := s.CreateResourceFromString(gatewayProxyWithValidProvider) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy with valid provider") + + Eventually(func() string { + gpYaml, err := s.GetResourceYaml("GatewayProxy", "api7-proxy-config") + Expect(err).NotTo(HaveOccurred(), "getting GatewayProxy yaml") + return gpYaml + }).WithTimeout(8*time.Second).ProbeEvery(2*time.Second). + Should(ContainSubstring(`"type":"ControlPlane"`), "checking GatewayProxy is applied") + }) + }) }) From b277d60eac890a8af5b9e202c0ac070443e51e89 Mon Sep 17 00:00:00 2001 From: ashing Date: Tue, 15 Apr 2025 18:42:01 +0800 Subject: [PATCH 08/16] fix: r Signed-off-by: ashing --- api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ config/crd/bases/gateway.apisix.io_gatewayproxies.yaml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 04ff10e12..fed035bb6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -281,6 +281,11 @@ func (in *ControlPlaneProvider) DeepCopyInto(out *ControlPlaneProvider) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.TlsVerify != nil { + in, out := &in.TlsVerify, &out.TlsVerify + *out = new(bool) + **out = **in + } in.Auth.DeepCopyInto(&out.Auth) } diff --git a/config/crd/bases/gateway.apisix.io_gatewayproxies.yaml b/config/crd/bases/gateway.apisix.io_gatewayproxies.yaml index ccf228253..c0dbc3ee3 100644 --- a/config/crd/bases/gateway.apisix.io_gatewayproxies.yaml +++ b/config/crd/bases/gateway.apisix.io_gatewayproxies.yaml @@ -108,6 +108,10 @@ spec: type: string minItems: 1 type: array + tlsVerify: + description: TlsVerify specifies whether to verify the TLS + certificate of the control plane + type: boolean required: - auth - endpoints From 030a2615c773ad76a8e9a51b169aa672abcec336 Mon Sep 17 00:00:00 2001 From: ashing Date: Tue, 15 Apr 2025 18:53:30 +0800 Subject: [PATCH 09/16] fix: r Signed-off-by: ashing --- test/e2e/ingress/ingress.go | 96 +++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/test/e2e/ingress/ingress.go b/test/e2e/ingress/ingress.go index 916dc5fef..e4520005d 100644 --- a/test/e2e/ingress/ingress.go +++ b/test/e2e/ingress/ingress.go @@ -208,6 +208,27 @@ spec: value: "%s" ` + gatewayProxyWithSecretYaml := ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config-with-secret + namespace: default +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + valueFrom: + secretKeyRef: + name: admin-secret + key: admin-key +` + var ingressClassWithProxy = ` apiVersion: networking.k8s.io/v1 kind: IngressClass @@ -225,6 +246,21 @@ spec: scope: "Namespace" ` + var ingressClassWithProxySecret = ` +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: api7-with-proxy-secret +spec: + controller: "gateway.api7.io/api7-ingress-controller" + parameters: + apiGroup: "gateway.apisix.io" + kind: "GatewayProxy" + name: "api7-proxy-config-with-secret" + namespace: "default" + scope: "Namespace" +` + var testIngress = ` apiVersion: networking.k8s.io/v1 kind: Ingress @@ -245,6 +281,26 @@ spec: number: 80 ` + var testIngressWithSecret = ` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: api7-ingress-with-proxy-secret +spec: + ingressClassName: api7-with-proxy-secret + rules: + - host: proxy-secret.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: httpbin-service-e2e-test + port: + number: 80 +` + It("Test IngressClass with GatewayProxy", func() { By("create GatewayProxy") gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) @@ -271,5 +327,45 @@ spec: Expect(). Status(200) }) + + It("Test IngressClass with GatewayProxy using Secret", func() { + By("create admin key secret") + adminSecret := fmt.Sprintf(` +apiVersion: v1 +kind: Secret +metadata: + name: admin-secret + namespace: default +type: Opaque +stringData: + admin-key: %s +`, s.AdminKey()) + err := s.CreateResourceFromStringWithNamespace(adminSecret, "default") + Expect(err).NotTo(HaveOccurred(), "creating admin secret") + time.Sleep(5 * time.Second) + + By("create GatewayProxy with Secret reference") + gatewayProxy := fmt.Sprintf(gatewayProxyWithSecretYaml, framework.DashboardTLSEndpoint) + err = s.CreateResourceFromStringWithNamespace(gatewayProxy, "default") + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy with Secret") + time.Sleep(5 * time.Second) + + By("create IngressClass with GatewayProxy reference") + err = s.CreateResourceFromStringWithNamespace(ingressClassWithProxySecret, "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass with GatewayProxy") + time.Sleep(5 * time.Second) + + By("create Ingress with GatewayProxy IngressClass") + err = s.CreateResourceFromString(testIngressWithSecret) + Expect(err).NotTo(HaveOccurred(), "creating Ingress with GatewayProxy IngressClass") + time.Sleep(5 * time.Second) + + By("verify HTTP request") + s.NewAPISIXClient(). + GET("/get"). + WithHost("proxy-secret.example.com"). + Expect(). + Status(200) + }) }) }) From 252728578bc7488c1a047093837a808fa4265185 Mon Sep 17 00:00:00 2001 From: ashing Date: Tue, 15 Apr 2025 23:36:39 +0800 Subject: [PATCH 10/16] fix: r Signed-off-by: ashing --- test/e2e/gatewayapi/gatewayproxy.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/e2e/gatewayapi/gatewayproxy.go b/test/e2e/gatewayapi/gatewayproxy.go index 723a0704a..e6160aca5 100644 --- a/test/e2e/gatewayapi/gatewayproxy.go +++ b/test/e2e/gatewayapi/gatewayproxy.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/api7/api7-ingress-controller/test/e2e/framework" "github.com/api7/api7-ingress-controller/test/e2e/scaffold" ) @@ -60,6 +61,15 @@ kind: GatewayProxy metadata: name: api7-proxy-config spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" plugins: - name: response-rewrite enabled: true @@ -178,7 +188,7 @@ spec: Expect(gcYaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message") By("Create GatewayProxy with enabled plugin") - err = s.CreateResourceFromString(gatewayProxyWithEnabledPlugin) + err = s.CreateResourceFromString(fmt.Sprintf(gatewayProxyWithEnabledPlugin, framework.DashboardTLSEndpoint, s.AdminKey())) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy with enabled plugin") time.Sleep(5 * time.Second) @@ -196,7 +206,7 @@ spec: AfterEach(func() { By("Clean up resources") - _ = s.DeleteResourceFromString(gatewayProxyWithEnabledPlugin) + _ = s.DeleteResourceFromString(fmt.Sprintf(gatewayProxyWithEnabledPlugin, framework.DashboardTLSEndpoint, s.AdminKey())) _ = s.DeleteResourceFromString(fmt.Sprintf(httpRouteForTest, "api7")) _ = s.DeleteResourceFromString(fmt.Sprintf(gatewayWithProxy, gatewayClassName)) }) From 6bdf26e018bf32efd6eb6ac4e8cabcde344df05f Mon Sep 17 00:00:00 2001 From: ashing Date: Wed, 16 Apr 2025 10:32:21 +0800 Subject: [PATCH 11/16] fix: http route Signed-off-by: ashing --- internal/provider/adc/adc.go | 89 +++++++++++++++++++++++++++++------- 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/internal/provider/adc/adc.go b/internal/provider/adc/adc.go index 2b423a68c..35a97d043 100644 --- a/internal/provider/adc/adc.go +++ b/internal/provider/adc/adc.go @@ -7,6 +7,7 @@ import ( "errors" "os" "os/exec" + "slices" "sync" "go.uber.org/zap" @@ -147,36 +148,50 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, err error ) + rk := ResourceKind{ + Kind: obj.GetObjectKind().GroupVersionKind().Kind, + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } switch t := obj.(type) { case *gatewayv1.HTTPRoute: - result, err = d.translator.TranslateHTTPRoute(tctx, t.DeepCopy()) + if result, err = d.translator.TranslateHTTPRoute(tctx, t.DeepCopy()); err != nil { + return err + } resourceTypes = append(resourceTypes, "service") + // delete http route if old configs diff with new configs exist + if err = d.DeleteHTTPRoute(ctx, tctx, t.DeepCopy()); err != nil { + return err + } case *gatewayv1.Gateway: - result, err = d.translator.TranslateGateway(tctx, t.DeepCopy()) + if result, err = d.translator.TranslateGateway(tctx, t.DeepCopy()); err != nil { + return err + } resourceTypes = append(resourceTypes, "global_rule", "ssl", "plugin_metadata") + if err = d.updateConfigs(rk, tctx); err != nil { + return err + } case *networkingv1.Ingress: - result, err = d.translator.TranslateIngress(tctx, t.DeepCopy()) + if result, err = d.translator.TranslateIngress(tctx, t.DeepCopy()); err != nil { + return err + } resourceTypes = append(resourceTypes, "service", "ssl") + if err = d.updateConfigs(rk, tctx); err != nil { + return err + } case *v1alpha1.Consumer: - result, err = d.translator.TranslateConsumerV1alpha1(tctx, t.DeepCopy()) + if result, err = d.translator.TranslateConsumerV1alpha1(tctx, t.DeepCopy()); err != nil { + return err + } resourceTypes = append(resourceTypes, "consumer") - } - if err != nil { - return err + if err = d.updateConfigs(rk, tctx); err != nil { + return err + } } if result == nil { return nil } - // update adc configs - rk := ResourceKind{ - Kind: obj.GetObjectKind().GroupVersionKind().Kind, - Namespace: obj.GetNamespace(), - Name: obj.GetName(), - } - if err := d.updateConfigs(rk, tctx); err != nil { - return err - } configs := d.getConfigs(rk) return d.sync(Task{ @@ -194,6 +209,48 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, }) } +func (d *adcClient) DeleteHTTPRoute(ctx context.Context, tctx *provider.TranslateContext, obj client.Object) error { + // diff adc configs + rk := ResourceKind{ + Kind: obj.GetObjectKind().GroupVersionKind().Kind, + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + + oldConfigs := d.getConfigs(rk) + err := d.updateConfigs(rk, tctx) + if err != nil { + return err + } + newConfigs := d.getConfigs(rk) + + // diff old configs and new configs + var deleteConfigs []adcConfig + for _, config := range oldConfigs { + if !slices.ContainsFunc(newConfigs, func(c adcConfig) bool { + return c.ServerAddr == config.ServerAddr && c.Token == config.Token + }) { + deleteConfigs = append(deleteConfigs, config) + } + } + + if len(deleteConfigs) > 0 { + log.Debugw("http route delete configs", zap.Any("configs", deleteConfigs)) + // sync old delete + err = d.sync(Task{ + Name: obj.GetName(), + Labels: label.GenLabel(obj), + ResourceTypes: []string{"service"}, + configs: deleteConfigs, + }) + if err != nil { + return err + } + } + + return nil +} + func (d *adcClient) Delete(ctx context.Context, obj client.Object) error { log.Debugw("deleting object", zap.Any("object", obj)) From ee796abfc9d3350f8c9ae47bcfa49c6598535b2f Mon Sep 17 00:00:00 2001 From: ashing Date: Wed, 16 Apr 2025 10:44:29 +0800 Subject: [PATCH 12/16] fix: r Signed-off-by: ashing --- internal/provider/adc/adc.go | 94 ++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 48 deletions(-) diff --git a/internal/provider/adc/adc.go b/internal/provider/adc/adc.go index 35a97d043..2d94616b6 100644 --- a/internal/provider/adc/adc.go +++ b/internal/provider/adc/adc.go @@ -153,46 +153,34 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, Namespace: obj.GetNamespace(), Name: obj.GetName(), } + switch t := obj.(type) { case *gatewayv1.HTTPRoute: - if result, err = d.translator.TranslateHTTPRoute(tctx, t.DeepCopy()); err != nil { - return err - } + result, err = d.handleHTTPRoute(tctx, t.DeepCopy()) resourceTypes = append(resourceTypes, "service") - // delete http route if old configs diff with new configs exist - if err = d.DeleteHTTPRoute(ctx, tctx, t.DeepCopy()); err != nil { - return err - } case *gatewayv1.Gateway: - if result, err = d.translator.TranslateGateway(tctx, t.DeepCopy()); err != nil { - return err - } + result, err = d.translator.TranslateGateway(tctx, t.DeepCopy()) resourceTypes = append(resourceTypes, "global_rule", "ssl", "plugin_metadata") - if err = d.updateConfigs(rk, tctx); err != nil { - return err - } case *networkingv1.Ingress: - if result, err = d.translator.TranslateIngress(tctx, t.DeepCopy()); err != nil { - return err - } + result, err = d.translator.TranslateIngress(tctx, t.DeepCopy()) resourceTypes = append(resourceTypes, "service", "ssl") - if err = d.updateConfigs(rk, tctx); err != nil { - return err - } case *v1alpha1.Consumer: - if result, err = d.translator.TranslateConsumerV1alpha1(tctx, t.DeepCopy()); err != nil { - return err - } + result, err = d.translator.TranslateConsumerV1alpha1(tctx, t.DeepCopy()) resourceTypes = append(resourceTypes, "consumer") - if err = d.updateConfigs(rk, tctx); err != nil { - return err - } + } + if err != nil { + return err } if result == nil { return nil } - configs := d.getConfigs(rk) + // Update configs for non-HTTPRoute types + if _, ok := obj.(*gatewayv1.HTTPRoute); !ok { + if err := d.updateConfigs(rk, tctx); err != nil { + return err + } + } return d.sync(Task{ Name: obj.GetName(), @@ -205,12 +193,24 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, Consumers: result.Consumers, }, ResourceTypes: resourceTypes, - configs: configs, + configs: d.getConfigs(rk), }) } -func (d *adcClient) DeleteHTTPRoute(ctx context.Context, tctx *provider.TranslateContext, obj client.Object) error { - // diff adc configs +func (d *adcClient) handleHTTPRoute(tctx *provider.TranslateContext, route *gatewayv1.HTTPRoute) (*translator.TranslateResult, error) { + result, err := d.translator.TranslateHTTPRoute(tctx, route) + if err != nil { + return nil, err + } + + if err := d.deleteHTTPRoute(tctx, route); err != nil { + return nil, err + } + + return result, nil +} + +func (d *adcClient) deleteHTTPRoute(tctx *provider.TranslateContext, obj client.Object) error { rk := ResourceKind{ Kind: obj.GetObjectKind().GroupVersionKind().Kind, Namespace: obj.GetNamespace(), @@ -218,13 +218,26 @@ func (d *adcClient) DeleteHTTPRoute(ctx context.Context, tctx *provider.Translat } oldConfigs := d.getConfigs(rk) - err := d.updateConfigs(rk, tctx) - if err != nil { + if err := d.updateConfigs(rk, tctx); err != nil { return err } newConfigs := d.getConfigs(rk) - // diff old configs and new configs + deleteConfigs := d.findConfigsToDelete(oldConfigs, newConfigs) + if len(deleteConfigs) == 0 { + return nil + } + + log.Debugw("http route delete configs", zap.Any("configs", deleteConfigs)) + return d.sync(Task{ + Name: obj.GetName(), + Labels: label.GenLabel(obj), + ResourceTypes: []string{"service"}, + configs: deleteConfigs, + }) +} + +func (d *adcClient) findConfigsToDelete(oldConfigs, newConfigs []adcConfig) []adcConfig { var deleteConfigs []adcConfig for _, config := range oldConfigs { if !slices.ContainsFunc(newConfigs, func(c adcConfig) bool { @@ -233,22 +246,7 @@ func (d *adcClient) DeleteHTTPRoute(ctx context.Context, tctx *provider.Translat deleteConfigs = append(deleteConfigs, config) } } - - if len(deleteConfigs) > 0 { - log.Debugw("http route delete configs", zap.Any("configs", deleteConfigs)) - // sync old delete - err = d.sync(Task{ - Name: obj.GetName(), - Labels: label.GenLabel(obj), - ResourceTypes: []string{"service"}, - configs: deleteConfigs, - }) - if err != nil { - return err - } - } - - return nil + return deleteConfigs } func (d *adcClient) Delete(ctx context.Context, obj client.Object) error { From 82c8869f70792d73a89a8774b64090935d072842 Mon Sep 17 00:00:00 2001 From: ashing Date: Wed, 16 Apr 2025 11:13:00 +0800 Subject: [PATCH 13/16] chore: r Signed-off-by: ashing --- config/samples/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/samples/config.yaml b/config/samples/config.yaml index a158ecbf4..3a392bf9f 100644 --- a/config/samples/config.yaml +++ b/config/samples/config.yaml @@ -12,9 +12,9 @@ ingress_status_address: [] # The status address of the ingress. gateway_configs: # The configuration of the API7 Gateway. - name: api7 # The name of the Gateway in the Gateway API. control_plane: - admin_key: "a7adm-qqEncKkktQFWrSdF8bGpDzjcRDbVFYgRXSBSPCzpVZZkc7pags-70be7645fdf94c3c8e4d656c2334bd8b" # The admin key of the control plane. + admin_key: "${ADMIN_KEY}" # The admin key of the control plane. endpoints: - - https://172.18.0.6:7443 # The endpoint of the control plane. + - ${ENDPOINT} # The endpoint of the control plane. tls_verify: false addresses: # record the status address of the gateway-api gateway - "172.18.0.4" # The LB IP of the gateway service. From d627153d4bd314c8e96aa8b7395e65c057d57f54 Mon Sep 17 00:00:00 2001 From: ashing Date: Wed, 16 Apr 2025 23:00:12 +0800 Subject: [PATCH 14/16] fix: review Signed-off-by: ashing --- internal/controller/consumer_controller.go | 8 +- internal/controller/gateway_controller.go | 7 +- internal/controller/httproute_controller.go | 7 +- internal/controller/ingress_controller.go | 15 +- internal/controller/utils.go | 11 +- internal/provider/adc/adc.go | 176 ++++---------------- internal/provider/adc/config.go | 123 ++++++++++++++ internal/provider/adc/translator/gateway.go | 32 ++-- internal/provider/provider.go | 11 +- 9 files changed, 225 insertions(+), 165 deletions(-) create mode 100644 internal/provider/adc/config.go diff --git a/internal/controller/consumer_controller.go b/internal/controller/consumer_controller.go index bcf72fb70..cb3428e54 100644 --- a/internal/controller/consumer_controller.go +++ b/internal/controller/consumer_controller.go @@ -144,7 +144,13 @@ func (r *ConsumerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c statusErr = err } - if err := ProcessGatewayProxy(r.Client, tctx, gateway); err != nil { + rk := provider.ResourceKind{ + Kind: consumer.Kind, + Namespace: consumer.Namespace, + Name: consumer.Name, + } + + if err := ProcessGatewayProxy(r.Client, tctx, gateway, rk); err != nil { r.Log.Error(err, "failed to process gateway proxy", "gateway", gateway) statusErr = err } diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 014e2b303..91f4a1a99 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -267,7 +267,12 @@ func (r *GatewayReconciler) listGatewaysForHTTPRoute(_ context.Context, obj clie } func (r *GatewayReconciler) processInfrastructure(tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) error { - return ProcessGatewayProxy(r.Client, tctx, gateway) + rk := provider.ResourceKind{ + Kind: gateway.Kind, + Namespace: gateway.Namespace, + Name: gateway.Name, + } + return ProcessGatewayProxy(r.Client, tctx, gateway, rk) } func (r *GatewayReconciler) processListenerConfig(tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) { diff --git a/internal/controller/httproute_controller.go b/internal/controller/httproute_controller.go index 638083adc..ff937e06e 100644 --- a/internal/controller/httproute_controller.go +++ b/internal/controller/httproute_controller.go @@ -114,8 +114,13 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( tctx := provider.NewDefaultTranslateContext() + rk := provider.ResourceKind{ + Kind: hr.Kind, + Namespace: hr.Namespace, + Name: hr.Name, + } for _, gateway := range gateways { - if err := ProcessGatewayProxy(r.Client, tctx, gateway.Gateway); err != nil { + if err := ProcessGatewayProxy(r.Client, tctx, gateway.Gateway, rk); err != nil { acceptStatus.status = false acceptStatus.msg = err.Error() } diff --git a/internal/controller/ingress_controller.go b/internal/controller/ingress_controller.go index 4cf6d8294..bdad00857 100644 --- a/internal/controller/ingress_controller.go +++ b/internal/controller/ingress_controller.go @@ -532,6 +532,18 @@ func (r *IngressReconciler) processIngressClassParameters(ctx context.Context, t return nil } + ingressClassKind := provider.ResourceKind{ + Kind: ingressClass.Kind, + Namespace: ingressClass.Namespace, + Name: ingressClass.Name, + } + + ingressKind := provider.ResourceKind{ + Kind: ingress.Kind, + Namespace: ingress.Namespace, + Name: ingress.Name, + } + parameters := ingressClass.Spec.Parameters // check if the parameters reference GatewayProxy if parameters.APIGroup != nil && *parameters.APIGroup == v1alpha1.GroupVersion.Group && parameters.Kind == "GatewayProxy" { @@ -550,7 +562,8 @@ func (r *IngressReconciler) processIngressClassParameters(ctx context.Context, t } r.Log.Info("found GatewayProxy for IngressClass", "ingressClass", ingressClass.Name, "gatewayproxy", gatewayProxy.Name) - tctx.GatewayProxies = append(tctx.GatewayProxies, *gatewayProxy) + tctx.GatewayProxies[ingressClassKind] = *gatewayProxy + tctx.ParentRefs[ingressKind] = append(tctx.ParentRefs[ingressKind], ingressClassKind) // check if the provider field references a secret if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane { diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 6938be2da..02255e1fa 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -764,7 +764,7 @@ func SplitMetaNamespaceKey(key string) (namespace, name string, err error) { return "", "", fmt.Errorf("unexpected key format: %q", key) } -func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) error { +func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gateway *gatewayv1.Gateway, rk provider.ResourceKind) error { if gateway == nil { return nil } @@ -773,6 +773,12 @@ func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gatew return nil } + gatewayKind := provider.ResourceKind{ + Kind: gateway.Kind, + Namespace: gateway.Namespace, + Name: gateway.Name, + } + ns := gateway.GetNamespace() paramRef := infra.ParametersRef if string(paramRef.Group) == v1alpha1.GroupVersion.Group && string(paramRef.Kind) == "GatewayProxy" { @@ -785,7 +791,8 @@ func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gatew return err } else { log.Info("found GatewayProxy for Gateway", "gateway", gateway.Name, "gatewayproxy", gatewayProxy.Name) - tctx.GatewayProxies = append(tctx.GatewayProxies, *gatewayProxy) + tctx.GatewayProxies[gatewayKind] = *gatewayProxy + tctx.ParentRefs[rk] = append(tctx.ParentRefs[rk], gatewayKind) // Process provider secrets if provider exists if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane { diff --git a/internal/provider/adc/adc.go b/internal/provider/adc/adc.go index 2d94616b6..10bbde44c 100644 --- a/internal/provider/adc/adc.go +++ b/internal/provider/adc/adc.go @@ -7,7 +7,6 @@ import ( "errors" "os" "os/exec" - "slices" "sync" "go.uber.org/zap" @@ -22,28 +21,24 @@ import ( "github.com/api7/api7-ingress-controller/internal/provider" "github.com/api7/api7-ingress-controller/internal/provider/adc/translator" "github.com/api7/gopkg/pkg/log" - "k8s.io/apimachinery/pkg/types" ) -type ResourceKind struct { - Kind string - Namespace string - Name string -} - type adcConfig struct { ServerAddr string Token string } type adcClient struct { - translator *translator.Translator + sync.Mutex + translator *translator.Translator ServerAddr string Token string GatewayGroup string - configs map[ResourceKind][]adcConfig - sync.Mutex + // gateway/ingressclass -> adcConfig + configs map[provider.ResourceKind]adcConfig + // httproute/consumer/ingress/gateway -> gateway/ingressclass + parentRefs map[provider.ResourceKind][]provider.ResourceKind } type Task struct { @@ -60,86 +55,11 @@ func New() (provider.Provider, error) { translator: &translator.Translator{}, ServerAddr: gc.ControlPlane.Endpoints[0], Token: gc.ControlPlane.AdminKey, - configs: make(map[ResourceKind][]adcConfig), + configs: make(map[provider.ResourceKind]adcConfig), + parentRefs: make(map[provider.ResourceKind][]provider.ResourceKind), }, nil } -func (d *adcClient) getConfigsForGatewayProxy(tctx *provider.TranslateContext, gatewayProxy *v1alpha1.GatewayProxy) (*adcConfig, error) { - if gatewayProxy == nil || gatewayProxy.Spec.Provider == nil { - return nil, nil - } - - provider := gatewayProxy.Spec.Provider - if provider.Type != v1alpha1.ProviderTypeControlPlane || provider.ControlPlane == nil { - return nil, nil - } - - endpoints := provider.ControlPlane.Endpoints - if len(endpoints) == 0 { - return nil, errors.New("no endpoints found") - } - - endpoint := endpoints[0] - config := adcConfig{ - ServerAddr: endpoint, - } - - if provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey && provider.ControlPlane.Auth.AdminKey != nil { - if provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { - secretRef := provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef - secret, ok := tctx.Secrets[types.NamespacedName{ - // we should use gateway proxy namespace - Namespace: gatewayProxy.GetNamespace(), - Name: secretRef.Name, - }] - if ok { - if token, ok := secret.Data[secretRef.Key]; ok { - config.Token = string(token) - } - } - } else if provider.ControlPlane.Auth.AdminKey.Value != "" { - config.Token = provider.ControlPlane.Auth.AdminKey.Value - } - } - - if config.Token == "" { - return nil, errors.New("no token found") - } - - return &config, nil -} - -func (d *adcClient) deleteConfigs(rk ResourceKind) { - d.Lock() - defer d.Unlock() - delete(d.configs, rk) -} - -func (d *adcClient) getConfigs(rk ResourceKind) []adcConfig { - d.Lock() - defer d.Unlock() - return d.configs[rk] -} - -func (d *adcClient) updateConfigs(rk ResourceKind, tctx *provider.TranslateContext) error { - d.Lock() - defer d.Unlock() - - var configs []adcConfig - for _, gatewayProxy := range tctx.GatewayProxies { - config, err := d.getConfigsForGatewayProxy(tctx, &gatewayProxy) - if err != nil { - return err - } - if config != nil { - configs = append(configs, *config) - } - } - - d.configs[rk] = configs - return nil -} - func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, obj client.Object) error { log.Debugw("updating object", zap.Any("object", obj)) var ( @@ -148,7 +68,7 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, err error ) - rk := ResourceKind{ + rk := provider.ResourceKind{ Kind: obj.GetObjectKind().GroupVersionKind().Kind, Namespace: obj.GetNamespace(), Name: obj.GetName(), @@ -156,7 +76,7 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, switch t := obj.(type) { case *gatewayv1.HTTPRoute: - result, err = d.handleHTTPRoute(tctx, t.DeepCopy()) + result, err = d.translator.TranslateHTTPRoute(tctx, t.DeepCopy()) resourceTypes = append(resourceTypes, "service") case *gatewayv1.Gateway: result, err = d.translator.TranslateGateway(tctx, t.DeepCopy()) @@ -175,14 +95,29 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, return nil } - // Update configs for non-HTTPRoute types - if _, ok := obj.(*gatewayv1.HTTPRoute); !ok { - if err := d.updateConfigs(rk, tctx); err != nil { + oldParentRefs := d.getParentRefs(rk) + if err := d.updateConfigs(rk, tctx); err != nil { + return err + } + newParentRefs := d.getParentRefs(rk) + deleteConfigs := d.findConfigsToDelete(oldParentRefs, newParentRefs) + configs := d.getConfigs(rk) + + // sync delete + if len(deleteConfigs) > 0 { + err = d.sync(Task{ + Name: obj.GetName(), + Labels: label.GenLabel(obj), + ResourceTypes: resourceTypes, + configs: deleteConfigs, + }) + if err != nil { return err } } - return d.sync(Task{ + // sync update + err = d.sync(Task{ Name: obj.GetName(), Labels: label.GenLabel(obj), Resources: adctypes.Resources{ @@ -193,60 +128,13 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, Consumers: result.Consumers, }, ResourceTypes: resourceTypes, - configs: d.getConfigs(rk), + configs: configs, }) -} - -func (d *adcClient) handleHTTPRoute(tctx *provider.TranslateContext, route *gatewayv1.HTTPRoute) (*translator.TranslateResult, error) { - result, err := d.translator.TranslateHTTPRoute(tctx, route) if err != nil { - return nil, err - } - - if err := d.deleteHTTPRoute(tctx, route); err != nil { - return nil, err - } - - return result, nil -} - -func (d *adcClient) deleteHTTPRoute(tctx *provider.TranslateContext, obj client.Object) error { - rk := ResourceKind{ - Kind: obj.GetObjectKind().GroupVersionKind().Kind, - Namespace: obj.GetNamespace(), - Name: obj.GetName(), - } - - oldConfigs := d.getConfigs(rk) - if err := d.updateConfigs(rk, tctx); err != nil { return err } - newConfigs := d.getConfigs(rk) - - deleteConfigs := d.findConfigsToDelete(oldConfigs, newConfigs) - if len(deleteConfigs) == 0 { - return nil - } - - log.Debugw("http route delete configs", zap.Any("configs", deleteConfigs)) - return d.sync(Task{ - Name: obj.GetName(), - Labels: label.GenLabel(obj), - ResourceTypes: []string{"service"}, - configs: deleteConfigs, - }) -} -func (d *adcClient) findConfigsToDelete(oldConfigs, newConfigs []adcConfig) []adcConfig { - var deleteConfigs []adcConfig - for _, config := range oldConfigs { - if !slices.ContainsFunc(newConfigs, func(c adcConfig) bool { - return c.ServerAddr == config.ServerAddr && c.Token == config.Token - }) { - deleteConfigs = append(deleteConfigs, config) - } - } - return deleteConfigs + return nil } func (d *adcClient) Delete(ctx context.Context, obj client.Object) error { @@ -268,7 +156,7 @@ func (d *adcClient) Delete(ctx context.Context, obj client.Object) error { labels = label.GenLabel(obj) } - rk := ResourceKind{ + rk := provider.ResourceKind{ Kind: obj.GetObjectKind().GroupVersionKind().Kind, Namespace: obj.GetNamespace(), Name: obj.GetName(), diff --git a/internal/provider/adc/config.go b/internal/provider/adc/config.go new file mode 100644 index 000000000..ad625f728 --- /dev/null +++ b/internal/provider/adc/config.go @@ -0,0 +1,123 @@ +package adc + +import ( + "errors" + "slices" + + "github.com/api7/api7-ingress-controller/api/v1alpha1" + "github.com/api7/api7-ingress-controller/internal/provider" + "github.com/api7/gopkg/pkg/log" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" +) + +func (d *adcClient) getConfigsForGatewayProxy(tctx *provider.TranslateContext, gatewayProxy *v1alpha1.GatewayProxy) (*adcConfig, error) { + if gatewayProxy == nil || gatewayProxy.Spec.Provider == nil { + return nil, nil + } + + provider := gatewayProxy.Spec.Provider + if provider.Type != v1alpha1.ProviderTypeControlPlane || provider.ControlPlane == nil { + return nil, nil + } + + endpoints := provider.ControlPlane.Endpoints + if len(endpoints) == 0 { + return nil, errors.New("no endpoints found") + } + + endpoint := endpoints[0] + config := adcConfig{ + ServerAddr: endpoint, + } + + if provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey && provider.ControlPlane.Auth.AdminKey != nil { + if provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { + secretRef := provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef + secret, ok := tctx.Secrets[types.NamespacedName{ + // we should use gateway proxy namespace + Namespace: gatewayProxy.GetNamespace(), + Name: secretRef.Name, + }] + if ok { + if token, ok := secret.Data[secretRef.Key]; ok { + config.Token = string(token) + } + } + } else if provider.ControlPlane.Auth.AdminKey.Value != "" { + config.Token = provider.ControlPlane.Auth.AdminKey.Value + } + } + + if config.Token == "" { + return nil, errors.New("no token found") + } + + return &config, nil +} + +func (d *adcClient) deleteConfigs(rk provider.ResourceKind) { + d.Lock() + defer d.Unlock() + delete(d.configs, rk) + delete(d.parentRefs, rk) +} + +func (d *adcClient) getParentRefs(rk provider.ResourceKind) []provider.ResourceKind { + d.Lock() + defer d.Unlock() + return d.parentRefs[rk] +} + +func (d *adcClient) getConfigs(rk provider.ResourceKind) []adcConfig { + d.Lock() + defer d.Unlock() + parentRefs := d.parentRefs[rk] + configs := make([]adcConfig, 0, len(parentRefs)) + for _, parentRef := range parentRefs { + if config, ok := d.configs[parentRef]; ok { + configs = append(configs, config) + } + } + return configs +} + +func (d *adcClient) updateConfigs(rk provider.ResourceKind, tctx *provider.TranslateContext) error { + d.Lock() + defer d.Unlock() + + // set parent refs + d.parentRefs[rk] = tctx.ParentRefs[rk] + parentRefs := d.parentRefs[rk] + + for _, parentRef := range parentRefs { + gatewayProxy, ok := tctx.GatewayProxies[parentRef] + if !ok { + log.Debugw("no gateway proxy found for parent ref", zap.Any("parentRef", parentRef)) + continue + } + config, err := d.getConfigsForGatewayProxy(tctx, &gatewayProxy) + if err != nil { + return err + } + if config == nil { + log.Debugw("no config found for gateway proxy", zap.Any("parentRef", parentRef)) + continue + } + d.configs[parentRef] = *config + } + + return nil +} + +func (d *adcClient) findConfigsToDelete(oldParentRefs, newParentRefs []provider.ResourceKind) []adcConfig { + var deleteConfigs []adcConfig + for _, parentRef := range oldParentRefs { + if !slices.ContainsFunc(newParentRefs, func(rk provider.ResourceKind) bool { + return rk.Kind == parentRef.Kind && rk.Namespace == parentRef.Namespace && rk.Name == parentRef.Name + }) { + deleteConfigs = append(deleteConfigs, d.configs[parentRef]) + } + } + return deleteConfigs +} diff --git a/internal/provider/adc/translator/gateway.go b/internal/provider/adc/translator/gateway.go index cb325bd4e..976811e18 100644 --- a/internal/provider/adc/translator/gateway.go +++ b/internal/provider/adc/translator/gateway.go @@ -32,21 +32,25 @@ func (t *Translator) TranslateGateway(tctx *provider.TranslateContext, obj *gate result.SSL = append(result.SSL, ssl...) } } - var gatewayProxy *v1alpha1.GatewayProxy - if len(tctx.GatewayProxies) > 0 { - gatewayProxy = &tctx.GatewayProxies[0] - } - if gatewayProxy != nil { - var ( - globalRules = adctypes.Plugins{} - pluginMetadata = adctypes.Plugins{} - ) - // apply plugins from GatewayProxy to global rules - t.fillPluginsFromGatewayProxy(globalRules, gatewayProxy) - t.fillPluginMetadataFromGatewayProxy(pluginMetadata, gatewayProxy) - result.GlobalRules = globalRules - result.PluginMetadata = pluginMetadata + rk := provider.ResourceKind{ + Kind: obj.Kind, + Namespace: obj.Namespace, + Name: obj.Name, } + gatewayProxy, ok := tctx.GatewayProxies[rk] + if !ok { + log.Debugw("no GatewayProxy found for Gateway", zap.String("gateway", obj.Name)) + return result, nil + } + + globalRules := adctypes.Plugins{} + pluginMetadata := adctypes.Plugins{} + // apply plugins from GatewayProxy to global rules + t.fillPluginsFromGatewayProxy(globalRules, &gatewayProxy) + t.fillPluginMetadataFromGatewayProxy(pluginMetadata, &gatewayProxy) + result.GlobalRules = globalRules + result.PluginMetadata = pluginMetadata + return result, nil } diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 5235d2c7f..f0058a623 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -17,10 +17,17 @@ type Provider interface { Delete(context.Context, client.Object) error } +type ResourceKind struct { + Kind string + Namespace string + Name string +} + type TranslateContext struct { BackendRefs []gatewayv1.BackendRef GatewayTLSConfig []gatewayv1.GatewayTLSConfig - GatewayProxies []v1alpha1.GatewayProxy + GatewayProxies map[ResourceKind]v1alpha1.GatewayProxy + ParentRefs map[ResourceKind][]ResourceKind Credentials []v1alpha1.Credential EndpointSlices map[types.NamespacedName][]discoveryv1.EndpointSlice Secrets map[types.NamespacedName]*corev1.Secret @@ -34,5 +41,7 @@ func NewDefaultTranslateContext() *TranslateContext { Secrets: make(map[types.NamespacedName]*corev1.Secret), PluginConfigs: make(map[types.NamespacedName]*v1alpha1.PluginConfig), Services: make(map[types.NamespacedName]*corev1.Service), + GatewayProxies: make(map[ResourceKind]v1alpha1.GatewayProxy), + ParentRefs: make(map[ResourceKind][]ResourceKind), } } From bda972840e5ada4254f0cbe857b060027f1c619a Mon Sep 17 00:00:00 2001 From: ashing Date: Wed, 16 Apr 2025 23:20:28 +0800 Subject: [PATCH 15/16] chore: r Signed-off-by: ashing --- config/samples/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/samples/config.yaml b/config/samples/config.yaml index 33c6113b5..79fd9977d 100644 --- a/config/samples/config.yaml +++ b/config/samples/config.yaml @@ -1,4 +1,4 @@ -log_level: "debug" # The log level of the API7 Ingress Controller. +log_level: "info" # The log level of the API7 Ingress Controller. # the default value is "info". controller_name: gateway.api7.io/api7-ingress-controller # The controller name of the API7 Ingress Controller, From 43c608c971831e9aeb825b65b1a2f1b59d9a7be1 Mon Sep 17 00:00:00 2001 From: ashing Date: Thu, 17 Apr 2025 00:01:18 +0800 Subject: [PATCH 16/16] fix: r Signed-off-by: ashing --- internal/controller/gateway_controller.go | 7 ++++--- internal/provider/provider.go | 14 +++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 91f4a1a99..f18f50d02 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -107,9 +107,10 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct status: true, msg: acceptedMessage("gateway"), } - tctx := &provider.TranslateContext{ - Secrets: make(map[types.NamespacedName]*corev1.Secret), - } + + // create a translate context + tctx := provider.NewDefaultTranslateContext(ctx) + r.processListenerConfig(tctx, gateway) if err := r.processInfrastructure(tctx, gateway); err != nil { acceptStatus = status{ diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 4d051d23b..d21f82118 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -25,20 +25,20 @@ type ResourceKind struct { type TranslateContext struct { context.Context - ParentRefs []gatewayv1.ParentReference - BackendRefs []gatewayv1.BackendRef - GatewayTLSConfig []gatewayv1.GatewayTLSConfig - Credentials []v1alpha1.Credential + ParentRefs []gatewayv1.ParentReference + BackendRefs []gatewayv1.BackendRef + GatewayTLSConfig []gatewayv1.GatewayTLSConfig + Credentials []v1alpha1.Credential + EndpointSlices map[types.NamespacedName][]discoveryv1.EndpointSlice Secrets map[types.NamespacedName]*corev1.Secret PluginConfigs map[types.NamespacedName]*v1alpha1.PluginConfig Services map[types.NamespacedName]*corev1.Service BackendTrafficPolicies map[types.NamespacedName]*v1alpha1.BackendTrafficPolicy + GatewayProxies map[ResourceKind]v1alpha1.GatewayProxy + ResourceParentRefs map[ResourceKind][]ResourceKind StatusUpdaters []client.Object - - GatewayProxies map[ResourceKind]v1alpha1.GatewayProxy - ResourceParentRefs map[ResourceKind][]ResourceKind } func NewDefaultTranslateContext(ctx context.Context) *TranslateContext {