diff --git a/api/v1alpha1/gatewayproxy_types.go b/api/v1alpha1/gatewayproxy_types.go index cb0404213..45f534af6 100644 --- a/api/v1alpha1/gatewayproxy_types.go +++ b/api/v1alpha1/gatewayproxy_types.go @@ -113,6 +113,10 @@ type ControlPlaneProvider struct { // +kubebuilder:validation:MinItems=1 Endpoints []string `json:"endpoints"` + // TlsVerify specifies whether to verify the TLS certificate of the control plane + // +optional + TlsVerify *bool `json:"tlsVerify,omitempty"` + // Auth specifies the authentication configuration // +kubebuilder:validation:Required Auth ControlPlaneAuth `json:"auth"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 33b1901ab..afc73cec5 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -302,6 +302,11 @@ func (in *ControlPlaneProvider) DeepCopyInto(out *ControlPlaneProvider) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.TlsVerify != nil { + in, out := &in.TlsVerify, &out.TlsVerify + *out = new(bool) + **out = **in + } in.Auth.DeepCopyInto(&out.Auth) } diff --git a/config/crd/bases/gateway.apisix.io_gatewayproxies.yaml b/config/crd/bases/gateway.apisix.io_gatewayproxies.yaml index ccf228253..c0dbc3ee3 100644 --- a/config/crd/bases/gateway.apisix.io_gatewayproxies.yaml +++ b/config/crd/bases/gateway.apisix.io_gatewayproxies.yaml @@ -108,6 +108,10 @@ spec: type: string minItems: 1 type: array + tlsVerify: + description: TlsVerify specifies whether to verify the TLS + certificate of the control plane + type: boolean required: - auth - endpoints diff --git a/config/samples/config.yaml b/config/samples/config.yaml index 47f4bb1e9..79fd9977d 100644 --- a/config/samples/config.yaml +++ b/config/samples/config.yaml @@ -24,7 +24,7 @@ gateway_configs: # The configuration of the API7 Gateway. control_plane: admin_key: "${ADMIN_KEY}" # The admin key of the control plane. endpoints: - - ${ENDPOINT} # The endpoint of the control plane. + - ${ENDPOINT} # The endpoint of the control plane. tls_verify: false addresses: # record the status address of the gateway-api gateway - "172.18.0.4" # The LB IP of the gateway service. diff --git a/internal/controller/consumer_controller.go b/internal/controller/consumer_controller.go index 2ef6e30e2..4ea7e3325 100644 --- a/internal/controller/consumer_controller.go +++ b/internal/controller/consumer_controller.go @@ -138,6 +138,23 @@ func (r *ConsumerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c var statusErr error tctx := provider.NewDefaultTranslateContext(ctx) + gateway, err := r.getGateway(ctx, consumer) + if err != nil { + r.Log.Error(err, "failed to get gateway", "consumer", consumer) + statusErr = err + } + + rk := provider.ResourceKind{ + Kind: consumer.Kind, + Namespace: consumer.Namespace, + Name: consumer.Name, + } + + if err := ProcessGatewayProxy(r.Client, tctx, gateway, rk); err != nil { + r.Log.Error(err, "failed to process gateway proxy", "gateway", gateway) + statusErr = err + } + if err := r.processSpec(ctx, tctx, consumer); err != nil { r.Log.Error(err, "failed to process consumer spec", "consumer", consumer) statusErr = err @@ -201,6 +218,22 @@ func (r *ConsumerReconciler) updateStatus(ctx context.Context, consumer *v1alpha return nil } +func (r *ConsumerReconciler) getGateway(ctx context.Context, consumer *v1alpha1.Consumer) (*gatewayv1.Gateway, error) { + ns := consumer.GetNamespace() + if consumer.Spec.GatewayRef.Namespace != nil { + ns = *consumer.Spec.GatewayRef.Namespace + } + gateway := &gatewayv1.Gateway{} + if err := r.Get(ctx, client.ObjectKey{ + Name: consumer.Spec.GatewayRef.Name, + Namespace: ns, + }, gateway); err != nil { + r.Log.Error(err, "failed to get gateway", "gateway", consumer.Spec.GatewayRef.Name) + return nil, err + } + return gateway, nil +} + func (r *ConsumerReconciler) checkGatewayRef(object client.Object) bool { consumer, ok := object.(*v1alpha1.Consumer) if !ok { diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 47e42fa3c..f18f50d02 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -107,9 +107,10 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct status: true, msg: acceptedMessage("gateway"), } - tctx := &provider.TranslateContext{ - Secrets: make(map[types.NamespacedName]*corev1.Secret), - } + + // create a translate context + tctx := provider.NewDefaultTranslateContext(ctx) + r.processListenerConfig(tctx, gateway) if err := r.processInfrastructure(tctx, gateway); err != nil { acceptStatus = status{ @@ -267,28 +268,12 @@ func (r *GatewayReconciler) listGatewaysForHTTPRoute(_ context.Context, obj clie } func (r *GatewayReconciler) processInfrastructure(tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) error { - infra := gateway.Spec.Infrastructure - if infra == nil || infra.ParametersRef == nil { - return nil + rk := provider.ResourceKind{ + Kind: gateway.Kind, + Namespace: gateway.Namespace, + Name: gateway.Name, } - - ns := gateway.GetNamespace() - paramRef := infra.ParametersRef - if string(paramRef.Group) == v1alpha1.GroupVersion.Group && string(paramRef.Kind) == "GatewayProxy" { - gatewayProxy := &v1alpha1.GatewayProxy{} - if err := r.Get(context.Background(), client.ObjectKey{ - Namespace: ns, - Name: paramRef.Name, - }, gatewayProxy); err != nil { - log.Error(err, "failed to get GatewayProxy", "namespace", ns, "name", paramRef.Name) - return err - } else { - log.Info("found GatewayProxy for Gateway", "gateway", gateway.Name, "gatewayproxy", gatewayProxy.Name) - tctx.GatewayProxy = gatewayProxy - } - } - - return nil + return ProcessGatewayProxy(r.Client, tctx, gateway, rk) } func (r *GatewayReconciler) processListenerConfig(tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) { diff --git a/internal/controller/httproute_controller.go b/internal/controller/httproute_controller.go index bc33dd79b..fce1dc2d3 100644 --- a/internal/controller/httproute_controller.go +++ b/internal/controller/httproute_controller.go @@ -114,6 +114,18 @@ func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( tctx := provider.NewDefaultTranslateContext(ctx) + rk := provider.ResourceKind{ + Kind: hr.Kind, + Namespace: hr.Namespace, + Name: hr.Name, + } + for _, gateway := range gateways { + if err := ProcessGatewayProxy(r.Client, tctx, gateway.Gateway, rk); err != nil { + acceptStatus.status = false + acceptStatus.msg = err.Error() + } + } + if err := r.processHTTPRoute(tctx, hr); err != nil { acceptStatus.status = false acceptStatus.msg = err.Error() diff --git a/internal/controller/ingress_controller.go b/internal/controller/ingress_controller.go index ac22a0ea7..d72d131c8 100644 --- a/internal/controller/ingress_controller.go +++ b/internal/controller/ingress_controller.go @@ -2,9 +2,11 @@ package controller import ( "context" + "errors" "fmt" "reflect" + "github.com/api7/api7-ingress-controller/api/v1alpha1" "github.com/api7/api7-ingress-controller/internal/controller/config" "github.com/api7/api7-ingress-controller/internal/controller/indexer" "github.com/api7/api7-ingress-controller/internal/provider" @@ -95,6 +97,12 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // create a translate context tctx := provider.NewDefaultTranslateContext(ctx) + // process IngressClass parameters if they reference GatewayProxy + if err := r.processIngressClassParameters(ctx, tctx, ingress); err != nil { + r.Log.Error(err, "failed to process IngressClass parameters", "ingress", ingress.Name) + return ctrl.Result{}, err + } + // process TLS configuration if err := r.processTLS(tctx, ingress); err != nil { r.Log.Error(err, "failed to process TLS configuration", "ingress", ingress.Name) @@ -122,6 +130,46 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } +// getIngressClass get the ingress class for the ingress +func (r *IngressReconciler) getIngressClass(obj client.Object) (*networkingv1.IngressClass, error) { + ingress := obj.(*networkingv1.Ingress) + + if ingress.Spec.IngressClassName == nil { + // handle the case where IngressClassName is not specified + // find all ingress classes and check if any of them is marked as default + ingressClassList := &networkingv1.IngressClassList{} + if err := r.List(context.Background(), ingressClassList, client.MatchingFields{ + indexer.IngressClass: config.GetControllerName(), + }); err != nil { + r.Log.Error(err, "failed to list ingress classes") + return nil, err + } + + // find the ingress class that is marked as default + for _, ic := range ingressClassList.Items { + if IsDefaultIngressClass(&ic) && matchesController(ic.Spec.Controller) { + log.Debugw("match the default ingress class") + return &ic, nil + } + } + + log.Debugw("no default ingress class found") + return nil, errors.New("no default ingress class found") + } + + // if it does not match, check if the ingress class is controlled by us + ingressClass := networkingv1.IngressClass{} + if err := r.Client.Get(context.Background(), client.ObjectKey{Name: *ingress.Spec.IngressClassName}, &ingressClass); err != nil { + return nil, err + } + + if matchesController(ingressClass.Spec.Controller) { + return &ingressClass, nil + } + + return nil, errors.New("ingress class is not controlled by us") +} + // checkIngressClass check if the ingress uses the ingress class that we control func (r *IngressReconciler) checkIngressClass(obj client.Object) bool { ingress := obj.(*networkingv1.Ingress) @@ -413,6 +461,8 @@ func (r *IngressReconciler) processBackendService(tctx *provider.TranslateContex func (r *IngressReconciler) updateStatus(ctx context.Context, ingress *networkingv1.Ingress) error { var loadBalancerStatus networkingv1.IngressLoadBalancerStatus + // todo: remove using default config, use the StatusAddress And PublishService in the gateway proxy + // 1. use the IngressStatusAddress in the config statusAddresses := config.GetIngressStatusAddress() if len(statusAddresses) > 0 { @@ -469,3 +519,84 @@ func (r *IngressReconciler) updateStatus(ctx context.Context, ingress *networkin return nil } + +// processIngressClassParameters processes the IngressClass parameters that reference GatewayProxy +func (r *IngressReconciler) processIngressClassParameters(ctx context.Context, tctx *provider.TranslateContext, ingress *networkingv1.Ingress) error { + ingressClass, err := r.getIngressClass(ingress) + if err != nil { + r.Log.Error(err, "failed to get IngressClass", "name", ingress.Spec.IngressClassName) + return err + } + + if ingressClass.Spec.Parameters == nil { + return nil + } + + ingressClassKind := provider.ResourceKind{ + Kind: ingressClass.Kind, + Namespace: ingressClass.Namespace, + Name: ingressClass.Name, + } + + ingressKind := provider.ResourceKind{ + Kind: ingress.Kind, + Namespace: ingress.Namespace, + Name: ingress.Name, + } + + parameters := ingressClass.Spec.Parameters + // check if the parameters reference GatewayProxy + if parameters.APIGroup != nil && *parameters.APIGroup == v1alpha1.GroupVersion.Group && parameters.Kind == "GatewayProxy" { + ns := ingress.GetNamespace() + if parameters.Namespace != nil { + ns = *parameters.Namespace + } + + gatewayProxy := &v1alpha1.GatewayProxy{} + if err := r.Get(ctx, client.ObjectKey{ + Namespace: ns, + Name: parameters.Name, + }, gatewayProxy); err != nil { + r.Log.Error(err, "failed to get GatewayProxy", "namespace", ns, "name", parameters.Name) + return err + } + + r.Log.Info("found GatewayProxy for IngressClass", "ingressClass", ingressClass.Name, "gatewayproxy", gatewayProxy.Name) + tctx.GatewayProxies[ingressClassKind] = *gatewayProxy + tctx.ResourceParentRefs[ingressKind] = append(tctx.ResourceParentRefs[ingressKind], ingressClassKind) + + // check if the provider field references a secret + if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane { + if gatewayProxy.Spec.Provider.ControlPlane != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { + + secretRef := gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef + secret := &corev1.Secret{} + if err := r.Get(ctx, client.ObjectKey{ + Namespace: ns, + Name: secretRef.Name, + }, secret); err != nil { + r.Log.Error(err, "failed to get secret for GatewayProxy provider", + "namespace", ns, + "name", secretRef.Name) + return err + } + + r.Log.Info("found secret for GatewayProxy provider", + "ingressClass", ingressClass.Name, + "gatewayproxy", gatewayProxy.Name, + "secret", secretRef.Name) + + tctx.Secrets[types.NamespacedName{ + Namespace: ns, + Name: secretRef.Name, + }] = secret + } + } + } + + return nil +} diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 9282117ea..0033bbcab 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -5,12 +5,16 @@ import ( "fmt" "strings" + "github.com/api7/api7-ingress-controller/api/v1alpha1" "github.com/api7/api7-ingress-controller/internal/controller/config" + "github.com/api7/api7-ingress-controller/internal/provider" + "github.com/api7/gopkg/pkg/log" "github.com/samber/lo" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -759,3 +763,70 @@ func SplitMetaNamespaceKey(key string) (namespace, name string, err error) { return "", "", fmt.Errorf("unexpected key format: %q", key) } + +func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gateway *gatewayv1.Gateway, rk provider.ResourceKind) error { + if gateway == nil { + return nil + } + infra := gateway.Spec.Infrastructure + if infra == nil || infra.ParametersRef == nil { + return nil + } + + gatewayKind := provider.ResourceKind{ + Kind: gateway.Kind, + Namespace: gateway.Namespace, + Name: gateway.Name, + } + + ns := gateway.GetNamespace() + paramRef := infra.ParametersRef + if string(paramRef.Group) == v1alpha1.GroupVersion.Group && string(paramRef.Kind) == "GatewayProxy" { + gatewayProxy := &v1alpha1.GatewayProxy{} + if err := r.Get(context.Background(), client.ObjectKey{ + Namespace: ns, + Name: paramRef.Name, + }, gatewayProxy); err != nil { + log.Error(err, "failed to get GatewayProxy", "namespace", ns, "name", paramRef.Name) + return err + } else { + log.Info("found GatewayProxy for Gateway", "gateway", gateway.Name, "gatewayproxy", gatewayProxy.Name) + tctx.GatewayProxies[gatewayKind] = *gatewayProxy + tctx.ResourceParentRefs[rk] = append(tctx.ResourceParentRefs[rk], gatewayKind) + + // Process provider secrets if provider exists + if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane { + if gatewayProxy.Spec.Provider.ControlPlane != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && + gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { + + secretRef := gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef + secret := &corev1.Secret{} + if err := r.Get(context.Background(), client.ObjectKey{ + Namespace: ns, + Name: secretRef.Name, + }, secret); err != nil { + log.Error(err, "failed to get secret for GatewayProxy provider", + "namespace", ns, + "name", secretRef.Name) + return err + } + + log.Info("found secret for GatewayProxy provider", + "gateway", gateway.Name, + "gatewayproxy", gatewayProxy.Name, + "secret", secretRef.Name) + + tctx.Secrets[types.NamespacedName{ + Namespace: ns, + Name: secretRef.Name, + }] = secret + } + } + } + } + + return nil +} diff --git a/internal/provider/adc/adc.go b/internal/provider/adc/adc.go index 285de1a1a..10bbde44c 100644 --- a/internal/provider/adc/adc.go +++ b/internal/provider/adc/adc.go @@ -7,13 +7,14 @@ import ( "errors" "os" "os/exec" + "sync" "go.uber.org/zap" networkingv1 "k8s.io/api/networking/v1" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" - types "github.com/api7/api7-ingress-controller/api/adc" + adctypes "github.com/api7/api7-ingress-controller/api/adc" "github.com/api7/api7-ingress-controller/api/v1alpha1" "github.com/api7/api7-ingress-controller/internal/controller/config" "github.com/api7/api7-ingress-controller/internal/controller/label" @@ -22,19 +23,30 @@ import ( "github.com/api7/gopkg/pkg/log" ) +type adcConfig struct { + ServerAddr string + Token string +} + type adcClient struct { - translator *translator.Translator + sync.Mutex + translator *translator.Translator ServerAddr string Token string GatewayGroup string + // gateway/ingressclass -> adcConfig + configs map[provider.ResourceKind]adcConfig + // httproute/consumer/ingress/gateway -> gateway/ingressclass + parentRefs map[provider.ResourceKind][]provider.ResourceKind } type Task struct { Name string - Resources types.Resources + Resources adctypes.Resources Labels map[string]string ResourceTypes []string + configs []adcConfig } func New() (provider.Provider, error) { @@ -43,6 +55,8 @@ func New() (provider.Provider, error) { translator: &translator.Translator{}, ServerAddr: gc.ControlPlane.Endpoints[0], Token: gc.ControlPlane.AdminKey, + configs: make(map[provider.ResourceKind]adcConfig), + parentRefs: make(map[provider.ResourceKind][]provider.ResourceKind), }, nil } @@ -54,6 +68,12 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, err error ) + rk := provider.ResourceKind{ + Kind: obj.GetObjectKind().GroupVersionKind().Kind, + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + switch t := obj.(type) { case *gatewayv1.HTTPRoute: result, err = d.translator.TranslateHTTPRoute(tctx, t.DeepCopy()) @@ -75,10 +95,32 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, return nil } - return d.sync(Task{ + oldParentRefs := d.getParentRefs(rk) + if err := d.updateConfigs(rk, tctx); err != nil { + return err + } + newParentRefs := d.getParentRefs(rk) + deleteConfigs := d.findConfigsToDelete(oldParentRefs, newParentRefs) + configs := d.getConfigs(rk) + + // sync delete + if len(deleteConfigs) > 0 { + err = d.sync(Task{ + Name: obj.GetName(), + Labels: label.GenLabel(obj), + ResourceTypes: resourceTypes, + configs: deleteConfigs, + }) + if err != nil { + return err + } + } + + // sync update + err = d.sync(Task{ Name: obj.GetName(), Labels: label.GenLabel(obj), - Resources: types.Resources{ + Resources: adctypes.Resources{ GlobalRules: result.GlobalRules, PluginMetadata: result.PluginMetadata, Services: result.Services, @@ -86,7 +128,13 @@ func (d *adcClient) Update(ctx context.Context, tctx *provider.TranslateContext, Consumers: result.Consumers, }, ResourceTypes: resourceTypes, + configs: configs, }) + if err != nil { + return err + } + + return nil } func (d *adcClient) Delete(ctx context.Context, obj client.Object) error { @@ -108,11 +156,26 @@ func (d *adcClient) Delete(ctx context.Context, obj client.Object) error { labels = label.GenLabel(obj) } - return d.sync(Task{ + rk := provider.ResourceKind{ + Kind: obj.GetObjectKind().GroupVersionKind().Kind, + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } + + configs := d.getConfigs(rk) + + err := d.sync(Task{ Name: obj.GetName(), Labels: labels, ResourceTypes: resourceTypes, + configs: configs, }) + if err != nil { + return err + } + + d.deleteConfigs(rk) + return nil } func (d *adcClient) sync(task Task) error { @@ -150,12 +213,44 @@ func (d *adcClient) sync(task Task) error { args = append(args, "--include-resource-type", t) } + if len(task.configs) > 0 { + log.Debugw("syncing resources with multiple configs", zap.Any("configs", task.configs)) + for _, config := range task.configs { + if err := d.execADC(config, args); err != nil { + return err + } + } + } else { + // todo: remove using default config + log.Debugw("syncing resources with default config") + if err := d.execADC(adcConfig{ + ServerAddr: d.ServerAddr, + Token: d.Token, + }, args); err != nil { + return err + } + } + + return nil +} + +func (d *adcClient) execADC(config adcConfig, args []string) error { + // todo: use adc config + serverAddr := d.ServerAddr + if config.ServerAddr != "" { + serverAddr = config.ServerAddr + } + token := d.Token + if config.Token != "" { + token = config.Token + } + adcEnv := []string{ "ADC_EXPERIMENTAL_FEATURE_FLAGS=remote-state-file,parallel-backend-request", "ADC_RUNNING_MODE=ingress", "ADC_BACKEND=api7ee", - "ADC_SERVER=" + d.ServerAddr, - "ADC_TOKEN=" + d.Token, + "ADC_SERVER=" + serverAddr, + "ADC_TOKEN=" + token, } var stdout, stderr bytes.Buffer @@ -167,7 +262,7 @@ func (d *adcClient) sync(task Task) error { log.Debug("running adc command", zap.String("command", cmd.String()), zap.Strings("env", adcEnv)) - var result types.SyncResult + var result adctypes.SyncResult if err := cmd.Run(); err != nil { stderrStr := stderr.String() stdoutStr := stdout.String() diff --git a/internal/provider/adc/config.go b/internal/provider/adc/config.go new file mode 100644 index 000000000..b7207697e --- /dev/null +++ b/internal/provider/adc/config.go @@ -0,0 +1,123 @@ +package adc + +import ( + "errors" + "slices" + + "github.com/api7/api7-ingress-controller/api/v1alpha1" + "github.com/api7/api7-ingress-controller/internal/provider" + "github.com/api7/gopkg/pkg/log" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" +) + +func (d *adcClient) getConfigsForGatewayProxy(tctx *provider.TranslateContext, gatewayProxy *v1alpha1.GatewayProxy) (*adcConfig, error) { + if gatewayProxy == nil || gatewayProxy.Spec.Provider == nil { + return nil, nil + } + + provider := gatewayProxy.Spec.Provider + if provider.Type != v1alpha1.ProviderTypeControlPlane || provider.ControlPlane == nil { + return nil, nil + } + + endpoints := provider.ControlPlane.Endpoints + if len(endpoints) == 0 { + return nil, errors.New("no endpoints found") + } + + endpoint := endpoints[0] + config := adcConfig{ + ServerAddr: endpoint, + } + + if provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey && provider.ControlPlane.Auth.AdminKey != nil { + if provider.ControlPlane.Auth.AdminKey.ValueFrom != nil && provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil { + secretRef := provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef + secret, ok := tctx.Secrets[types.NamespacedName{ + // we should use gateway proxy namespace + Namespace: gatewayProxy.GetNamespace(), + Name: secretRef.Name, + }] + if ok { + if token, ok := secret.Data[secretRef.Key]; ok { + config.Token = string(token) + } + } + } else if provider.ControlPlane.Auth.AdminKey.Value != "" { + config.Token = provider.ControlPlane.Auth.AdminKey.Value + } + } + + if config.Token == "" { + return nil, errors.New("no token found") + } + + return &config, nil +} + +func (d *adcClient) deleteConfigs(rk provider.ResourceKind) { + d.Lock() + defer d.Unlock() + delete(d.configs, rk) + delete(d.parentRefs, rk) +} + +func (d *adcClient) getParentRefs(rk provider.ResourceKind) []provider.ResourceKind { + d.Lock() + defer d.Unlock() + return d.parentRefs[rk] +} + +func (d *adcClient) getConfigs(rk provider.ResourceKind) []adcConfig { + d.Lock() + defer d.Unlock() + parentRefs := d.parentRefs[rk] + configs := make([]adcConfig, 0, len(parentRefs)) + for _, parentRef := range parentRefs { + if config, ok := d.configs[parentRef]; ok { + configs = append(configs, config) + } + } + return configs +} + +func (d *adcClient) updateConfigs(rk provider.ResourceKind, tctx *provider.TranslateContext) error { + d.Lock() + defer d.Unlock() + + // set parent refs + d.parentRefs[rk] = tctx.ResourceParentRefs[rk] + parentRefs := d.parentRefs[rk] + + for _, parentRef := range parentRefs { + gatewayProxy, ok := tctx.GatewayProxies[parentRef] + if !ok { + log.Debugw("no gateway proxy found for parent ref", zap.Any("parentRef", parentRef)) + continue + } + config, err := d.getConfigsForGatewayProxy(tctx, &gatewayProxy) + if err != nil { + return err + } + if config == nil { + log.Debugw("no config found for gateway proxy", zap.Any("parentRef", parentRef)) + continue + } + d.configs[parentRef] = *config + } + + return nil +} + +func (d *adcClient) findConfigsToDelete(oldParentRefs, newParentRefs []provider.ResourceKind) []adcConfig { + var deleteConfigs []adcConfig + for _, parentRef := range oldParentRefs { + if !slices.ContainsFunc(newParentRefs, func(rk provider.ResourceKind) bool { + return rk.Kind == parentRef.Kind && rk.Namespace == parentRef.Namespace && rk.Name == parentRef.Name + }) { + deleteConfigs = append(deleteConfigs, d.configs[parentRef]) + } + } + return deleteConfigs +} diff --git a/internal/provider/adc/translator/gateway.go b/internal/provider/adc/translator/gateway.go index 64835c7a2..976811e18 100644 --- a/internal/provider/adc/translator/gateway.go +++ b/internal/provider/adc/translator/gateway.go @@ -32,17 +32,25 @@ func (t *Translator) TranslateGateway(tctx *provider.TranslateContext, obj *gate result.SSL = append(result.SSL, ssl...) } } - if tctx.GatewayProxy != nil { - var ( - globalRules = adctypes.Plugins{} - pluginMetadata = adctypes.Plugins{} - ) - // apply plugins from GatewayProxy to global rules - t.fillPluginsFromGatewayProxy(globalRules, tctx.GatewayProxy) - t.fillPluginMetadataFromGatewayProxy(pluginMetadata, tctx.GatewayProxy) - result.GlobalRules = globalRules - result.PluginMetadata = pluginMetadata + rk := provider.ResourceKind{ + Kind: obj.Kind, + Namespace: obj.Namespace, + Name: obj.Name, } + gatewayProxy, ok := tctx.GatewayProxies[rk] + if !ok { + log.Debugw("no GatewayProxy found for Gateway", zap.String("gateway", obj.Name)) + return result, nil + } + + globalRules := adctypes.Plugins{} + pluginMetadata := adctypes.Plugins{} + // apply plugins from GatewayProxy to global rules + t.fillPluginsFromGatewayProxy(globalRules, &gatewayProxy) + t.fillPluginMetadataFromGatewayProxy(pluginMetadata, &gatewayProxy) + result.GlobalRules = globalRules + result.PluginMetadata = pluginMetadata + return result, nil } diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 395d7eaa7..d21f82118 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -17,18 +17,26 @@ type Provider interface { Delete(context.Context, client.Object) error } +type ResourceKind struct { + Kind string + Namespace string + Name string +} + type TranslateContext struct { context.Context - ParentRefs []gatewayv1.ParentReference - BackendRefs []gatewayv1.BackendRef - GatewayTLSConfig []gatewayv1.GatewayTLSConfig - GatewayProxy *v1alpha1.GatewayProxy - Credentials []v1alpha1.Credential + ParentRefs []gatewayv1.ParentReference + BackendRefs []gatewayv1.BackendRef + GatewayTLSConfig []gatewayv1.GatewayTLSConfig + Credentials []v1alpha1.Credential + EndpointSlices map[types.NamespacedName][]discoveryv1.EndpointSlice Secrets map[types.NamespacedName]*corev1.Secret PluginConfigs map[types.NamespacedName]*v1alpha1.PluginConfig Services map[types.NamespacedName]*corev1.Service BackendTrafficPolicies map[types.NamespacedName]*v1alpha1.BackendTrafficPolicy + GatewayProxies map[ResourceKind]v1alpha1.GatewayProxy + ResourceParentRefs map[ResourceKind][]ResourceKind StatusUpdaters []client.Object } @@ -41,5 +49,7 @@ func NewDefaultTranslateContext(ctx context.Context) *TranslateContext { PluginConfigs: make(map[types.NamespacedName]*v1alpha1.PluginConfig), Services: make(map[types.NamespacedName]*corev1.Service), BackendTrafficPolicies: make(map[types.NamespacedName]*v1alpha1.BackendTrafficPolicy), + GatewayProxies: make(map[ResourceKind]v1alpha1.GatewayProxy), + ResourceParentRefs: make(map[ResourceKind][]ResourceKind), } } diff --git a/test/e2e/gatewayapi/gatewayproxy.go b/test/e2e/gatewayapi/gatewayproxy.go index 37bb39dd2..e6160aca5 100644 --- a/test/e2e/gatewayapi/gatewayproxy.go +++ b/test/e2e/gatewayapi/gatewayproxy.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/api7/api7-ingress-controller/test/e2e/framework" "github.com/api7/api7-ingress-controller/test/e2e/scaffold" ) @@ -60,6 +61,15 @@ kind: GatewayProxy metadata: name: api7-proxy-config spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" plugins: - name: response-rewrite enabled: true @@ -178,7 +188,7 @@ spec: Expect(gcYaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message") By("Create GatewayProxy with enabled plugin") - err = s.CreateResourceFromString(gatewayProxyWithEnabledPlugin) + err = s.CreateResourceFromString(fmt.Sprintf(gatewayProxyWithEnabledPlugin, framework.DashboardTLSEndpoint, s.AdminKey())) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy with enabled plugin") time.Sleep(5 * time.Second) @@ -196,7 +206,7 @@ spec: AfterEach(func() { By("Clean up resources") - _ = s.DeleteResourceFromString(gatewayProxyWithEnabledPlugin) + _ = s.DeleteResourceFromString(fmt.Sprintf(gatewayProxyWithEnabledPlugin, framework.DashboardTLSEndpoint, s.AdminKey())) _ = s.DeleteResourceFromString(fmt.Sprintf(httpRouteForTest, "api7")) _ = s.DeleteResourceFromString(fmt.Sprintf(gatewayWithProxy, gatewayClassName)) }) @@ -314,4 +324,77 @@ spec: Body().Contains(`{"error_msg":"404 Route Not Found"}`) }) }) + + var ( + gatewayProxyWithInvalidProviderType = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config +spec: + provider: + type: "InvalidType" +` + gatewayProxyWithMissingControlPlane = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config +spec: + provider: + type: "ControlPlane" +` + gatewayProxyWithValidProvider = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config +spec: + provider: + type: "ControlPlane" + controlPlane: + endpoints: + - "http://localhost:9180" + auth: + type: "AdminKey" + adminKey: + value: "test-key" +` + ) + + Context("Test GatewayProxy Provider Validation", func() { + AfterEach(func() { + By("Clean up GatewayProxy resources") + _ = s.DeleteResourceFromString(gatewayProxyWithInvalidProviderType) + _ = s.DeleteResourceFromString(gatewayProxyWithMissingControlPlane) + _ = s.DeleteResourceFromString(gatewayProxyWithValidProvider) + }) + + It("Should reject invalid provider type", func() { + By("Create GatewayProxy with invalid provider type") + err := s.CreateResourceFromString(gatewayProxyWithInvalidProviderType) + Expect(err).To(HaveOccurred(), "creating GatewayProxy with invalid provider type") + Expect(err.Error()).To(ContainSubstring("Invalid value")) + }) + + It("Should reject missing controlPlane configuration", func() { + By("Create GatewayProxy with missing controlPlane") + err := s.CreateResourceFromString(gatewayProxyWithMissingControlPlane) + Expect(err).To(HaveOccurred(), "creating GatewayProxy with missing controlPlane") + Expect(err.Error()).To(ContainSubstring("controlPlane must be specified when type is ControlPlane")) + }) + + It("Should accept valid provider configuration", func() { + By("Create GatewayProxy with valid provider") + err := s.CreateResourceFromString(gatewayProxyWithValidProvider) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy with valid provider") + + Eventually(func() string { + gpYaml, err := s.GetResourceYaml("GatewayProxy", "api7-proxy-config") + Expect(err).NotTo(HaveOccurred(), "getting GatewayProxy yaml") + return gpYaml + }).WithTimeout(8*time.Second).ProbeEvery(2*time.Second). + Should(ContainSubstring(`"type":"ControlPlane"`), "checking GatewayProxy is applied") + }) + }) }) diff --git a/test/e2e/ingress/ingress.go b/test/e2e/ingress/ingress.go index c5ebb119d..e4520005d 100644 --- a/test/e2e/ingress/ingress.go +++ b/test/e2e/ingress/ingress.go @@ -188,4 +188,184 @@ spec: Status(200) }) }) + + Context("IngressClass with GatewayProxy", func() { + gatewayProxyYaml := ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config + namespace: default +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" +` + + gatewayProxyWithSecretYaml := ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config-with-secret + namespace: default +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + valueFrom: + secretKeyRef: + name: admin-secret + key: admin-key +` + + var ingressClassWithProxy = ` +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: api7-with-proxy + annotations: + ingressclass.kubernetes.io/is-default-class: "true" +spec: + controller: "gateway.api7.io/api7-ingress-controller" + parameters: + apiGroup: "gateway.apisix.io" + kind: "GatewayProxy" + name: "api7-proxy-config" + namespace: "default" + scope: "Namespace" +` + + var ingressClassWithProxySecret = ` +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: api7-with-proxy-secret +spec: + controller: "gateway.api7.io/api7-ingress-controller" + parameters: + apiGroup: "gateway.apisix.io" + kind: "GatewayProxy" + name: "api7-proxy-config-with-secret" + namespace: "default" + scope: "Namespace" +` + + var testIngress = ` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: api7-ingress-with-proxy +spec: + ingressClassName: api7-with-proxy + rules: + - host: proxy.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: httpbin-service-e2e-test + port: + number: 80 +` + + var testIngressWithSecret = ` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: api7-ingress-with-proxy-secret +spec: + ingressClassName: api7-with-proxy-secret + rules: + - host: proxy-secret.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: httpbin-service-e2e-test + port: + number: 80 +` + + It("Test IngressClass with GatewayProxy", func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) + + By("create GatewayProxy") + err := s.CreateResourceFromStringWithNamespace(gatewayProxy, "default") + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create IngressClass with GatewayProxy reference") + err = s.CreateResourceFromStringWithNamespace(ingressClassWithProxy, "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass with GatewayProxy") + time.Sleep(5 * time.Second) + + By("create Ingress with GatewayProxy IngressClass") + err = s.CreateResourceFromString(testIngress) + Expect(err).NotTo(HaveOccurred(), "creating Ingress with GatewayProxy IngressClass") + time.Sleep(5 * time.Second) + + By("verify HTTP request") + s.NewAPISIXClient(). + GET("/get"). + WithHost("proxy.example.com"). + Expect(). + Status(200) + }) + + It("Test IngressClass with GatewayProxy using Secret", func() { + By("create admin key secret") + adminSecret := fmt.Sprintf(` +apiVersion: v1 +kind: Secret +metadata: + name: admin-secret + namespace: default +type: Opaque +stringData: + admin-key: %s +`, s.AdminKey()) + err := s.CreateResourceFromStringWithNamespace(adminSecret, "default") + Expect(err).NotTo(HaveOccurred(), "creating admin secret") + time.Sleep(5 * time.Second) + + By("create GatewayProxy with Secret reference") + gatewayProxy := fmt.Sprintf(gatewayProxyWithSecretYaml, framework.DashboardTLSEndpoint) + err = s.CreateResourceFromStringWithNamespace(gatewayProxy, "default") + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy with Secret") + time.Sleep(5 * time.Second) + + By("create IngressClass with GatewayProxy reference") + err = s.CreateResourceFromStringWithNamespace(ingressClassWithProxySecret, "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass with GatewayProxy") + time.Sleep(5 * time.Second) + + By("create Ingress with GatewayProxy IngressClass") + err = s.CreateResourceFromString(testIngressWithSecret) + Expect(err).NotTo(HaveOccurred(), "creating Ingress with GatewayProxy IngressClass") + time.Sleep(5 * time.Second) + + By("verify HTTP request") + s.NewAPISIXClient(). + GET("/get"). + WithHost("proxy-secret.example.com"). + Expect(). + Status(200) + }) + }) })