From bfb0284cf8ae1b4a2373432162db594026ebd8b5 Mon Sep 17 00:00:00 2001 From: ashing Date: Thu, 17 Apr 2025 11:56:46 +0800 Subject: [PATCH 1/6] fix: remove default config Signed-off-by: ashing --- charts/templates/configmap.yaml | 10 --- cmd/root/root.go | 4 - config/samples/config.yaml | 24 +++--- docs/configure.md | 10 --- internal/controller/config/config.go | 80 +------------------ internal/controller/config/types.go | 27 +++---- internal/controller/gateway_controller.go | 47 ++++++----- internal/controller/ingress_controller.go | 54 ++++++++----- internal/controller/utils.go | 5 ++ internal/provider/adc/adc.go | 26 ++---- .../provider/controlplane/controlplane.go | 5 +- 11 files changed, 98 insertions(+), 194 deletions(-) diff --git a/charts/templates/configmap.yaml b/charts/templates/configmap.yaml index b6d04b8ae..7b373bfa1 100644 --- a/charts/templates/configmap.yaml +++ b/charts/templates/configmap.yaml @@ -10,13 +10,3 @@ data: controller_name: {{ .Values.controllerName | default "gateway.api7.io/api7-ingress-controller" }} leader_election_id: "api7-ingress-controller-leader" - - gateway_configs: - - name: "api7ee" - control_plane: - admin_key: "{{ required "Provide a valid admin key!" .Values.admin.key }}" - endpoints: - - "{{ .Values.admin.endpoint }}" - tls_verify: {{ .Values.admin.tlsVerify }} - addresses: # record the status address of the gateway-api gateway - - "{{ .Values.statusAddress }}" diff --git a/cmd/root/root.go b/cmd/root/root.go index d0bc06ad5..98152ab32 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -91,7 +91,6 @@ func newAPI7IngressController() *cobra.Command { cfg := config.ControllerConfig var configPath string - var controlPlanesFlag GatewayConfigsFlag cmd := &cobra.Command{ Use: "api7-ingress-controller [command]", Long: "Yet another Ingress controller for Kubernetes using api7ee Gateway as the high performance reverse proxy.", @@ -104,8 +103,6 @@ func newAPI7IngressController() *cobra.Command { } cfg = c config.SetControllerConfig(c) - } else { - cfg.GatewayConfigs = controlPlanesFlag.GatewayConfigs } if err := cfg.Validate(); err != nil { @@ -153,7 +150,6 @@ func newAPI7IngressController() *cobra.Command { cmd.Flags().StringVar(&cfg.MetricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") cmd.Flags().StringVar(&cfg.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - cmd.Flags().Var(&controlPlanesFlag, "control-planes", "Control planes configuration in YAML format") cmd.Flags().StringVar(&cfg.LogLevel, "log-level", config.DefaultLogLevel, "The log level for api7-ingress-controller") cmd.Flags().StringVar(&cfg.ControllerName, "controller-name", diff --git a/config/samples/config.yaml b/config/samples/config.yaml index 79fd9977d..e60a20b19 100644 --- a/config/samples/config.yaml +++ b/config/samples/config.yaml @@ -16,15 +16,15 @@ leader_election: # will wait between tries of actions with the controller. disable: false # Whether to disable leader election. -ingress_class: api7 # The ingress class name of the API7 Ingress Controller. -ingress_publish_service: "" # The service name of the ingress publish service. -ingress_status_address: [] # The status address of the ingress. -gateway_configs: # The configuration of the API7 Gateway. -- name: api7 # The name of the Gateway in the Gateway API. - control_plane: - admin_key: "${ADMIN_KEY}" # The admin key of the control plane. - endpoints: - - ${ENDPOINT} # The endpoint of the control plane. - tls_verify: false - addresses: # record the status address of the gateway-api gateway - - "172.18.0.4" # The LB IP of the gateway service. +# ingress_class: api7 # The ingress class name of the API7 Ingress Controller. +# ingress_publish_service: "" # The service name of the ingress publish service. +# ingress_status_address: [] # The status address of the ingress. +# gateway_configs: # The configuration of the API7 Gateway. +# - name: api7 # The name of the Gateway in the Gateway API. +# control_plane: +# admin_key: "${ADMIN_KEY}" # The admin key of the control plane. +# endpoints: +# - ${ENDPOINT} # The endpoint of the control plane. +# tls_verify: false +# addresses: # record the status address of the gateway-api gateway +# - "172.18.0.4" # The LB IP of the gateway service. diff --git a/docs/configure.md b/docs/configure.md index 3056257e9..80862e669 100644 --- a/docs/configure.md +++ b/docs/configure.md @@ -14,16 +14,6 @@ controller_name: gateway.api7.io/api7-ingress-controller # The controller name leader_election_id: "api7-ingress-controller-leader" # The leader election ID for the API7 Ingress Controller. # The default value is "api7-ingress-controller-leader". - -gateway_configs: # The configuration of the API7 Gateway. -- name: api7 # The name of the Gateway in the Gateway API. - control_plane: - admin_key: "${ADMIN_KEY}" # The admin key of the control plane. - endpoints: - - ${ENDPOINT}/apisix/admin # The endpoint of the control plane. - tls_verify: false - addresses: # record the status address of the gateway-api gateway - - "172.18.0.4" # The LB IP of the gateway service. ``` ### Controller Name diff --git a/internal/controller/config/config.go b/internal/controller/config/config.go index 932ae2c8d..0cd870ffe 100644 --- a/internal/controller/config/config.go +++ b/internal/controller/config/config.go @@ -30,7 +30,6 @@ func NewDefaultConfig() *Config { LeaderElectionID: DefaultLeaderElectionID, ProbeAddr: DefaultProbeAddr, MetricsAddr: DefaultMetricsAddr, - IngressClass: DefaultIngressClass, LeaderElection: NewLeaderElection(), } } @@ -84,21 +83,13 @@ func NewConfigFromFile(filename string) (*Config, error) { } func (c *Config) Validate() error { - - if len(c.GatewayConfigs) == 0 { - return fmt.Errorf("gateway_configs config is required") - } - for _, gc := range c.GatewayConfigs { - if err := c.validateGatewayConfig(gc); err != nil { - return fmt.Errorf("failed to validate control_planes: %w", err) - } - } if c.ControllerName == "" { return fmt.Errorf("controller_name is required") } return nil } +//nolint:unused func (c *Config) validateGatewayConfig(gc *GatewayConfig) error { if gc.Name == "" { @@ -118,75 +109,6 @@ func (c *Config) validateGatewayConfig(gc *GatewayConfig) error { return nil } -var gatewayNameMap map[string]*GatewayConfig -var gatewayNameList []string - -func initGatewayNameMap() { - if gatewayNameMap == nil { - gatewayNameMap = make(map[string]*GatewayConfig) - for _, gc := range ControllerConfig.GatewayConfigs { - gatewayNameMap[gc.Name] = gc - } - } -} - -func GetControlPlaneConfigByGatewatName(gatewatName string) *ControlPlaneConfig { - initGatewayNameMap() - if gc, ok := gatewayNameMap[gatewatName]; ok { - return gc.ControlPlane - } - return nil -} - -func GetGatewayConfig(gatewayName string) *GatewayConfig { - initGatewayNameMap() - if gc, ok := gatewayNameMap[gatewayName]; ok { - return gc - } - return nil -} - -func GetFirstGatewayConfig() *GatewayConfig { - if len(ControllerConfig.GatewayConfigs) > 0 { - return ControllerConfig.GatewayConfigs[0] - } - return nil -} - -func GetGatewayAddresses(gatewayName string) []string { - initGatewayNameMap() - if gc, ok := gatewayNameMap[gatewayName]; ok { - return gc.Addresses - } - return nil -} - -func GatewayConfigs() []*GatewayConfig { - return ControllerConfig.GatewayConfigs -} - -func GatewayNameList() []string { - if gatewayNameList == nil { - gatewayNameList = make([]string, 0, len(ControllerConfig.GatewayConfigs)) - for _, gc := range ControllerConfig.GatewayConfigs { - gatewayNameList = append(gatewayNameList, gc.Name) - } - } - return gatewayNameList -} - -func GetIngressClass() string { - return ControllerConfig.IngressClass -} - -func GetIngressPublishService() string { - return ControllerConfig.IngressPublishService -} - -func GetIngressStatusAddress() []string { - return ControllerConfig.IngressStatusAddress -} - func GetControllerName() string { return ControllerConfig.ControllerName } diff --git a/internal/controller/config/types.go b/internal/controller/config/types.go index a821e7bb0..f247380eb 100644 --- a/internal/controller/config/types.go +++ b/internal/controller/config/types.go @@ -17,9 +17,6 @@ const ( // DefaultLogLevel is the default log level for apisix-ingress-controller. DefaultLogLevel = "info" - // DefaultIngressClass is the default ingress class name for Ingress resources - DefaultIngressClass = "api7" - DefaultMetricsAddr = ":8080" DefaultProbeAddr = ":8081" ) @@ -27,20 +24,16 @@ const ( // Config contains all config items which are necessary for // apisix-ingress-controller's running. type Config struct { - CertFilePath string `json:"cert_file" yaml:"cert_file"` - KeyFilePath string `json:"key_file" yaml:"key_file"` - LogLevel string `json:"log_level" yaml:"log_level"` - ControllerName string `json:"controller_name" yaml:"controller_name"` - LeaderElectionID string `json:"leader_election_id" yaml:"leader_election_id"` - GatewayConfigs []*GatewayConfig `json:"gateway_configs" yaml:"gateway_configs"` - MetricsAddr string `json:"metrics_addr" yaml:"metrics_addr"` - EnableHTTP2 bool `json:"enable_http2" yaml:"enable_http2"` - ProbeAddr string `json:"probe_addr" yaml:"probe_addr"` - SecureMetrics bool `json:"secure_metrics" yaml:"secure_metrics"` - IngressClass string `json:"ingress_class" yaml:"ingress_class"` - IngressPublishService string `json:"ingress_publish_service" yaml:"ingress_publish_service"` - IngressStatusAddress []string `json:"ingress_status_address" yaml:"ingress_status_address"` - LeaderElection *LeaderElection `json:"leader_election" yaml:"leader_election"` + CertFilePath string `json:"cert_file" yaml:"cert_file"` + KeyFilePath string `json:"key_file" yaml:"key_file"` + LogLevel string `json:"log_level" yaml:"log_level"` + ControllerName string `json:"controller_name" yaml:"controller_name"` + LeaderElectionID string `json:"leader_election_id" yaml:"leader_election_id"` + MetricsAddr string `json:"metrics_addr" yaml:"metrics_addr"` + EnableHTTP2 bool `json:"enable_http2" yaml:"enable_http2"` + ProbeAddr string `json:"probe_addr" yaml:"probe_addr"` + SecureMetrics bool `json:"secure_metrics" yaml:"secure_metrics"` + LeaderElection *LeaderElection `json:"leader_election" yaml:"leader_election"` } type GatewayConfig struct { diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index f18f50d02..cc67d2cd0 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -6,7 +6,6 @@ import ( "reflect" "github.com/api7/api7-ingress-controller/api/v1alpha1" - "github.com/api7/api7-ingress-controller/internal/controller/config" "github.com/api7/api7-ingress-controller/internal/controller/indexer" "github.com/api7/api7-ingress-controller/internal/provider" "github.com/api7/gopkg/pkg/log" @@ -81,23 +80,6 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } conditionProgrammedStatus, conditionProgrammedMsg := true, "Programmed" - cfg := config.GetFirstGatewayConfig() - - var addrs []gatewayv1.GatewayStatusAddress - - if len(gateway.Status.Addresses) != len(cfg.Addresses) { - for _, addr := range cfg.Addresses { - if addr == "" { - continue - } - addrs = append(addrs, - gatewayv1.GatewayStatusAddress{ - Value: addr, - }, - ) - } - } - r.Log.Info("gateway has been accepted", "gateway", gateway.GetName()) type status struct { status bool @@ -119,6 +101,35 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } } + var addrs []gatewayv1.GatewayStatusAddress + + rk := provider.ResourceKind{ + Kind: gateway.Kind, + Namespace: gateway.Namespace, + Name: gateway.Name, + } + + gatewayProxy, ok := tctx.GatewayProxies[rk] + if !ok { + acceptStatus = status{ + status: false, + msg: "gateway proxy not found", + } + } else { + if len(gateway.Status.Addresses) != len(gatewayProxy.Spec.StatusAddress) { + for _, addr := range gatewayProxy.Spec.StatusAddress { + if addr == "" { + continue + } + addrs = append(addrs, + gatewayv1.GatewayStatusAddress{ + Value: addr, + }, + ) + } + } + } + if err := r.Provider.Update(ctx, tctx, gateway); err != nil { acceptStatus = status{ status: false, diff --git a/internal/controller/ingress_controller.go b/internal/controller/ingress_controller.go index d72d131c8..325529694 100644 --- a/internal/controller/ingress_controller.go +++ b/internal/controller/ingress_controller.go @@ -97,9 +97,15 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // create a translate context tctx := provider.NewDefaultTranslateContext(ctx) + ingressClass, err := r.getIngressClass(ingress) + if err != nil { + r.Log.Error(err, "failed to get IngressClass") + return ctrl.Result{}, err + } + // process IngressClass parameters if they reference GatewayProxy - if err := r.processIngressClassParameters(ctx, tctx, ingress); err != nil { - r.Log.Error(err, "failed to process IngressClass parameters", "ingress", ingress.Name) + if err := r.processIngressClassParameters(ctx, tctx, ingress, ingressClass); err != nil { + r.Log.Error(err, "failed to process IngressClass parameters", "ingressClass", ingressClass.Name) return ctrl.Result{}, err } @@ -122,7 +128,7 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } // update the ingress status - if err := r.updateStatus(ctx, ingress); err != nil { + if err := r.updateStatus(ctx, tctx, ingress, ingressClass); err != nil { r.Log.Error(err, "failed to update ingress status", "ingress", ingress.Name) return ctrl.Result{}, err } @@ -197,14 +203,7 @@ func (r *IngressReconciler) checkIngressClass(obj client.Object) bool { return false } - configuredClass := config.GetIngressClass() - // if the ingress class name matches the configured ingress class name, return true - if *ingress.Spec.IngressClassName == configuredClass { - log.Debugw("match the configured ingress class name") - return true - } - - // if it does not match, check if the ingress class is controlled by us + // check if the ingress class is controlled by us ingressClass := networkingv1.IngressClass{} if err := r.Client.Get(context.Background(), client.ObjectKey{Name: *ingress.Spec.IngressClassName}, &ingressClass); err != nil { return false @@ -458,13 +457,23 @@ func (r *IngressReconciler) processBackendService(tctx *provider.TranslateContex } // updateStatus update the status of the ingress -func (r *IngressReconciler) updateStatus(ctx context.Context, ingress *networkingv1.Ingress) error { +func (r *IngressReconciler) updateStatus(ctx context.Context, tctx *provider.TranslateContext, ingress *networkingv1.Ingress, ingressClass *networkingv1.IngressClass) error { var loadBalancerStatus networkingv1.IngressLoadBalancerStatus - // todo: remove using default config, use the StatusAddress And PublishService in the gateway proxy + ingressClassKind := provider.ResourceKind{ + Kind: ingressClass.Kind, + Namespace: ingressClass.Namespace, + Name: ingressClass.Name, + } + + gatewayProxy, ok := tctx.GatewayProxies[ingressClassKind] + if !ok { + log.Debugw("no gateway proxy found for ingress class", zap.String("ingressClass", ingressClass.Name)) + return nil + } // 1. use the IngressStatusAddress in the config - statusAddresses := config.GetIngressStatusAddress() + statusAddresses := gatewayProxy.Spec.StatusAddress if len(statusAddresses) > 0 { for _, addr := range statusAddresses { if addr == "" { @@ -476,7 +485,7 @@ func (r *IngressReconciler) updateStatus(ctx context.Context, ingress *networkin } } else { // 2. if the IngressStatusAddress is not configured, try to use the PublishService - publishService := config.GetIngressPublishService() + publishService := gatewayProxy.Spec.PublishService if publishService != "" { // parse the namespace/name format namespace, name, err := SplitMetaNamespaceKey(publishService) @@ -521,13 +530,7 @@ func (r *IngressReconciler) updateStatus(ctx context.Context, ingress *networkin } // processIngressClassParameters processes the IngressClass parameters that reference GatewayProxy -func (r *IngressReconciler) processIngressClassParameters(ctx context.Context, tctx *provider.TranslateContext, ingress *networkingv1.Ingress) error { - ingressClass, err := r.getIngressClass(ingress) - if err != nil { - r.Log.Error(err, "failed to get IngressClass", "name", ingress.Spec.IngressClassName) - return err - } - +func (r *IngressReconciler) processIngressClassParameters(ctx context.Context, tctx *provider.TranslateContext, ingress *networkingv1.Ingress, ingressClass *networkingv1.IngressClass) error { if ingressClass.Spec.Parameters == nil { return nil } @@ -598,5 +601,12 @@ func (r *IngressReconciler) processIngressClassParameters(ctx context.Context, t } } + // if gateway proxy is not found, return error + _, ok := tctx.GatewayProxies[ingressClassKind] + if !ok { + r.Log.Error(fmt.Errorf("no gateway proxy found for ingress class"), "failed to process IngressClass parameters", "ingressClass", ingressClass.Name) + return fmt.Errorf("no gateway proxy found for ingress class") + } + return nil } diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 0033bbcab..4e80311e8 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -828,5 +828,10 @@ func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gatew } } + _, ok := tctx.GatewayProxies[gatewayKind] + if !ok { + return fmt.Errorf("no gateway proxy found for gateway: %s", gateway.Name) + } + return nil } diff --git a/internal/provider/adc/adc.go b/internal/provider/adc/adc.go index 10bbde44c..80b8a07fd 100644 --- a/internal/provider/adc/adc.go +++ b/internal/provider/adc/adc.go @@ -16,7 +16,6 @@ import ( adctypes "github.com/api7/api7-ingress-controller/api/adc" "github.com/api7/api7-ingress-controller/api/v1alpha1" - "github.com/api7/api7-ingress-controller/internal/controller/config" "github.com/api7/api7-ingress-controller/internal/controller/label" "github.com/api7/api7-ingress-controller/internal/provider" "github.com/api7/api7-ingress-controller/internal/provider/adc/translator" @@ -50,11 +49,8 @@ type Task struct { } func New() (provider.Provider, error) { - gc := config.GetFirstGatewayConfig() return &adcClient{ translator: &translator.Translator{}, - ServerAddr: gc.ControlPlane.Endpoints[0], - Token: gc.ControlPlane.AdminKey, configs: make(map[provider.ResourceKind]adcConfig), parentRefs: make(map[provider.ResourceKind][]provider.ResourceKind), }, nil @@ -181,6 +177,11 @@ func (d *adcClient) Delete(ctx context.Context, obj client.Object) error { func (d *adcClient) sync(task Task) error { log.Debugw("syncing resources", zap.Any("task", task)) + if len(task.configs) == 0 { + log.Errorw("no adc configs provided", zap.Any("task", task)) + return errors.New("no adc configs provided") + } + data, err := json.Marshal(task.Resources) if err != nil { return err @@ -213,20 +214,9 @@ func (d *adcClient) sync(task Task) error { args = append(args, "--include-resource-type", t) } - if len(task.configs) > 0 { - log.Debugw("syncing resources with multiple configs", zap.Any("configs", task.configs)) - for _, config := range task.configs { - if err := d.execADC(config, args); err != nil { - return err - } - } - } else { - // todo: remove using default config - log.Debugw("syncing resources with default config") - if err := d.execADC(adcConfig{ - ServerAddr: d.ServerAddr, - Token: d.Token, - }, args); err != nil { + log.Debugw("syncing resources with multiple configs", zap.Any("configs", task.configs)) + for _, config := range task.configs { + if err := d.execADC(config, args); err != nil { return err } } diff --git a/internal/provider/controlplane/controlplane.go b/internal/provider/controlplane/controlplane.go index 8dcc4b182..6522a5921 100644 --- a/internal/provider/controlplane/controlplane.go +++ b/internal/provider/controlplane/controlplane.go @@ -19,22 +19,19 @@ type dashboardProvider struct { c dashboard.Dashboard } +//nolint:unused func NewDashboard() (provider.Provider, error) { control, err := dashboard.NewClient() if err != nil { return nil, err } - gc := config.GetFirstGatewayConfig() if err := control.AddCluster(context.TODO(), &dashboard.ClusterOptions{ Name: "default", Labels: map[string]string{ "k8s/controller-name": config.ControllerConfig.ControllerName, }, ControllerName: config.ControllerConfig.ControllerName, - BaseURL: gc.ControlPlane.Endpoints[0], - AdminKey: gc.ControlPlane.AdminKey, - SkipTLSVerify: !*gc.ControlPlane.TLSVerify, SyncCache: true, }); err != nil { return nil, err From b8852fad2eda5bbda62153d740c3a6f50154ee83 Mon Sep 17 00:00:00 2001 From: ashing Date: Thu, 17 Apr 2025 15:50:52 +0800 Subject: [PATCH 2/6] fix: ingress test case Signed-off-by: ashing --- config/samples/config.yaml | 2 +- test/e2e/framework/manifests/ingress.yaml | 10 --- test/e2e/ingress/ingress.go | 90 +++++++++++------------ test/e2e/scaffold/ingress.go | 1 - test/e2e/scaffold/scaffold.go | 1 - 5 files changed, 44 insertions(+), 60 deletions(-) diff --git a/config/samples/config.yaml b/config/samples/config.yaml index e60a20b19..8344ce041 100644 --- a/config/samples/config.yaml +++ b/config/samples/config.yaml @@ -1,4 +1,4 @@ -log_level: "info" # The log level of the API7 Ingress Controller. +log_level: "debug" # The log level of the API7 Ingress Controller. # the default value is "info". controller_name: gateway.api7.io/api7-ingress-controller # The controller name of the API7 Ingress Controller, diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index 9d4216e0f..0e8f4d7e3 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -325,16 +325,6 @@ data: controller_name: {{ .ControllerName | default "gateway.api7.io/api7-ingress-controller" }} leader_election_id: "api7-ingress-controller-leader" - - gateway_configs: - - name: "api7ee" - control_plane: - admin_key: "{{ .AdminKey }}" - endpoints: - - "{{ .AdminEnpoint }}" - tls_verify: {{ .AdminTLSVerify }} - addresses: # record the status address of the gateway-api gateway - - "{{ .StatusAddress }}" --- apiVersion: v1 kind: Service diff --git a/test/e2e/ingress/ingress.go b/test/e2e/ingress/ingress.go index e4520005d..6bcc5e11a 100644 --- a/test/e2e/ingress/ingress.go +++ b/test/e2e/ingress/ingress.go @@ -30,58 +30,34 @@ var _ = Describe("Test Ingress", func() { ControllerName: "gateway.api7.io/api7-ingress-controller", }) - Context("Basic Ingress Functionality", func() { - var defaultIngressClass = ` -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - name: api7 -spec: - controller: "gateway.api7.io/api7-ingress-controller" -` - - var defaultIngress = ` -apiVersion: networking.k8s.io/v1 -kind: Ingress + var gatewayProxyYaml = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy metadata: - name: api7-ingress + name: api7-proxy-config + namespace: default spec: - ingressClassName: api7 - rules: - - host: example.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: httpbin-service-e2e-test - port: - number: 80 + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" ` - It("Create Ingress", func() { - By("create IngressClass") - err := s.CreateResourceFromStringWithNamespace(defaultIngressClass, "") - Expect(err).NotTo(HaveOccurred(), "creating IngressClass") - time.Sleep(5 * time.Second) + Context("Ingress TLS", func() { + It("Check if SSL resource was created", func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) - By("create Ingress") - err = s.CreateResourceFromString(defaultIngress) - Expect(err).NotTo(HaveOccurred(), "creating Ingress") + By("create GatewayProxy") + err := s.CreateResourceFromStringWithNamespace(gatewayProxy, "default") + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") time.Sleep(5 * time.Second) - By("verify HTTP request") - s.NewAPISIXClient(). - GET("/get"). - WithHost("example.com"). - Expect(). - Status(200) - }) - }) - - Context("Ingress TLS", func() { - It("Check if SSL resource was created", func() { secretName := _secretName host := "api6.com" createSecret(s, secretName) @@ -93,6 +69,12 @@ metadata: name: api7 spec: controller: "gateway.api7.io/api7-ingress-controller" + parameters: + apiGroup: "gateway.apisix.io" + kind: "GatewayProxy" + name: "api7-proxy-config" + namespace: "default" + scope: "Namespace" ` var tlsIngress = fmt.Sprintf(` @@ -120,7 +102,7 @@ spec: `, host, secretName, host) By("create IngressClass") - err := s.CreateResourceFromStringWithNamespace(defaultIngressClass, "") + err = s.CreateResourceFromStringWithNamespace(defaultIngressClass, "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") time.Sleep(5 * time.Second) @@ -148,6 +130,12 @@ metadata: ingressclass.kubernetes.io/is-default-class: "true" spec: controller: "gateway.api7.io/api7-ingress-controller" + parameters: + apiGroup: "gateway.apisix.io" + kind: "GatewayProxy" + name: "api7-proxy-config" + namespace: "default" + scope: "Namespace" ` var defaultIngress = ` @@ -170,8 +158,16 @@ spec: ` It("Test IngressClass Selection", func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) + + By("create GatewayProxy") + err := s.CreateResourceFromStringWithNamespace(gatewayProxy, "default") + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + By("create Default IngressClass") - err := s.CreateResourceFromStringWithNamespace(defaultIngressClass, "") + err = s.CreateResourceFromStringWithNamespace(defaultIngressClass, "") Expect(err).NotTo(HaveOccurred(), "creating Default IngressClass") time.Sleep(5 * time.Second) diff --git a/test/e2e/scaffold/ingress.go b/test/e2e/scaffold/ingress.go index da7e2cc16..0f8a4c64a 100644 --- a/test/e2e/scaffold/ingress.go +++ b/test/e2e/scaffold/ingress.go @@ -11,6 +11,5 @@ func (s *Scaffold) deployIngress() { AdminTLSVerify: false, Namespace: s.namespace, AdminEnpoint: framework.DashboardTLSEndpoint, - StatusAddress: s.gatewayAddress, }) } diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index cf10b0c78..e46a8a4f0 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -83,7 +83,6 @@ type Scaffold struct { nodes []corev1.Node dataplaneService *corev1.Service httpbinService *corev1.Service - gatewayAddress string finalizers []func() label map[string]string From b0315886ec17992601b76753f2bf9353af2c93b4 Mon Sep 17 00:00:00 2001 From: ashing Date: Thu, 17 Apr 2025 16:34:44 +0800 Subject: [PATCH 3/6] fix: gateway Signed-off-by: ashing --- internal/controller/utils.go | 5 +-- test/e2e/gatewayapi/gateway.go | 62 ++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 4e80311e8..5467eb8ce 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -10,6 +10,7 @@ import ( "github.com/api7/api7-ingress-controller/internal/provider" "github.com/api7/gopkg/pkg/log" "github.com/samber/lo" + "go.uber.org/zap" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -787,10 +788,10 @@ func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gatew Namespace: ns, Name: paramRef.Name, }, gatewayProxy); err != nil { - log.Error(err, "failed to get GatewayProxy", "namespace", ns, "name", paramRef.Name) + log.Errorw("failed to get GatewayProxy", zap.String("namespace", ns), zap.String("name", paramRef.Name), zap.Error(err)) return err } else { - log.Info("found GatewayProxy for Gateway", "gateway", gateway.Name, "gatewayproxy", gatewayProxy.Name) + log.Infow("found GatewayProxy for Gateway", zap.String("namespace", gateway.Namespace), zap.String("name", gateway.Name)) tctx.GatewayProxies[gatewayKind] = *gatewayProxy tctx.ResourceParentRefs[rk] = append(tctx.ResourceParentRefs[rk], gatewayKind) diff --git a/test/e2e/gatewayapi/gateway.go b/test/e2e/gatewayapi/gateway.go index 6320e5c9c..6972dd49b 100644 --- a/test/e2e/gatewayapi/gateway.go +++ b/test/e2e/gatewayapi/gateway.go @@ -30,6 +30,23 @@ var _ = Describe("Test Gateway", func() { ControllerName: "gateway.api7.io/api7-ingress-controller", }) + var gatewayProxyYaml = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" +` + Context("Gateway", func() { var defautlGatewayClass = ` apiVersion: gateway.networking.k8s.io/v1 @@ -51,6 +68,11 @@ spec: - name: http1 protocol: HTTP port: 80 + infrastructure: + parametersRef: + group: gateway.apisix.io + kind: GatewayProxy + name: api7-proxy-config ` var noClassGateway = ` @@ -64,11 +86,22 @@ spec: - name: http1 protocol: HTTP port: 80 + infrastructure: + parametersRef: + group: gateway.apisix.io + kind: GatewayProxy + name: api7-proxy-config ` It("Create Gateway", func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) + err := s.CreateResourceFromString(gatewayProxy) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + By("create GatewayClass") - err := s.CreateResourceFromStringWithNamespace(defautlGatewayClass, "") + err = s.CreateResourceFromStringWithNamespace(defautlGatewayClass, "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) @@ -103,6 +136,13 @@ spec: Context("Gateway SSL", func() { It("Check if SSL resource was created", func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) + err := s.CreateResourceFromString(gatewayProxy) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create secret") secretName := _secretName host := "api6.com" createSecret(s, secretName) @@ -132,9 +172,14 @@ spec: - kind: Secret group: "" name: %s + infrastructure: + parametersRef: + group: gateway.apisix.io + kind: GatewayProxy + name: api7-proxy-config `, host, secretName) By("create GatewayClass") - err := s.CreateResourceFromStringWithNamespace(defaultGatewayClass, "") + err = s.CreateResourceFromStringWithNamespace(defaultGatewayClass, "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) @@ -152,6 +197,12 @@ spec: Context("Gateway SSL with and without hostname", func() { It("Check if SSL resource was created", func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) + err := s.CreateResourceFromString(gatewayProxy) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + secretName := _secretName createSecret(s, secretName) var defaultGatewayClass = ` @@ -194,9 +245,14 @@ spec: - group: "" kind: Secret name: %s + infrastructure: + parametersRef: + group: gateway.apisix.io + kind: GatewayProxy + name: api7-proxy-config `, secretName, secretName) By("create GatewayClass") - err := s.CreateResourceFromStringWithNamespace(defaultGatewayClass, "") + err = s.CreateResourceFromStringWithNamespace(defaultGatewayClass, "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) From a15337eb33a7eb315c335af9e6ed549131237e7f Mon Sep 17 00:00:00 2001 From: ashing Date: Thu, 17 Apr 2025 17:16:26 +0800 Subject: [PATCH 4/6] fix: test http route Signed-off-by: ashing --- test/e2e/gatewayapi/httproute.go | 43 ++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index 509c1e042..e78f8e7ad 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -16,6 +16,23 @@ import ( var _ = Describe("Test HTTPRoute", func() { s := scaffold.NewDefaultScaffold() + var gatewayProxyYaml = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" +` + var defautlGatewayClass = ` apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -36,6 +53,11 @@ spec: - name: http1 protocol: HTTP port: 80 + infrastructure: + parametersRef: + group: gateway.apisix.io + kind: GatewayProxy + name: api7-proxy-config ` var defautlGatewayHTTPS = ` apiVersion: gateway.networking.k8s.io/v1 @@ -54,6 +76,11 @@ spec: - kind: Secret group: "" name: test-apisix-tls + infrastructure: + parametersRef: + group: gateway.apisix.io + kind: GatewayProxy + name: api7-proxy-config ` var ResourceApplied = func(resourType, resourceName, resourceRaw string, observedGeneration int) { @@ -76,9 +103,15 @@ spec: } var beforeEachHTTP = func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) + err := s.CreateResourceFromString(gatewayProxy) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + By("create GatewayClass") gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix()) - err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) @@ -101,11 +134,17 @@ spec: } var beforeEachHTTPS = func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) + err := s.CreateResourceFromString(gatewayProxy) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + secretName := _secretName createSecret(s, secretName) By("create GatewayClass") gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix()) - err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) From d7f500ff5c67156e9c0f66a77dabc7fb8a1fe724 Mon Sep 17 00:00:00 2001 From: ashing Date: Thu, 17 Apr 2025 17:50:50 +0800 Subject: [PATCH 5/6] fix: consumer test case Signed-off-by: ashing --- test/e2e/crds/consumer.go | 36 +++++++++++++++++++++++++++++--- test/e2e/gatewayapi/httproute.go | 2 +- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/test/e2e/crds/consumer.go b/test/e2e/crds/consumer.go index cb0d0b40f..0c9b3542e 100644 --- a/test/e2e/crds/consumer.go +++ b/test/e2e/crds/consumer.go @@ -7,12 +7,30 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/api7/api7-ingress-controller/test/e2e/framework" "github.com/api7/api7-ingress-controller/test/e2e/scaffold" ) var _ = Describe("Test Consumer", func() { s := scaffold.NewDefaultScaffold() + var gatewayProxyYaml = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" +` + var defaultGatewayClass = ` apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -33,6 +51,11 @@ spec: - name: http1 protocol: HTTP port: 80 + infrastructure: + parametersRef: + group: gateway.apisix.io + kind: GatewayProxy + name: api7-proxy-config ` var defaultHTTPRoute = ` @@ -67,7 +90,7 @@ spec: filters: - type: ExtensionRef extensionRef: - group: gateway.api7.io + group: gateway.apisix.io kind: PluginConfig name: auth-plugin-config backendRefs: @@ -76,10 +99,16 @@ spec: ` var beforeEachHTTP = func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) + err := s.CreateResourceFromString(gatewayProxy) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + By("create GatewayClass") gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix()) gatewayString := fmt.Sprintf(defaultGatewayClass, gatewayClassName, s.GetControllerName()) - err := s.CreateResourceFromStringWithNamespace(gatewayString, "") + err = s.CreateResourceFromStringWithNamespace(gatewayString, "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) @@ -187,7 +216,8 @@ spec: }) Context("Credential", func() { - var defaultCredential = `apiVersion: gateway.apisix.io/v1alpha1 + var defaultCredential = ` +apiVersion: gateway.apisix.io/v1alpha1 kind: Consumer metadata: name: consumer-sample diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index e78f8e7ad..e0af23c35 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -655,7 +655,7 @@ spec: filters: - type: ExtensionRef extensionRef: - group: gateway.api7.io + group: gateway.apisix.io kind: PluginConfig name: example-plugin-config backendRefs: From cbcc4fa6712774f2fbfd835402dc2ce52f684f28 Mon Sep 17 00:00:00 2001 From: ashing Date: Thu, 17 Apr 2025 18:13:18 +0800 Subject: [PATCH 6/6] fix: gateway proxy Signed-off-by: ashing --- test/e2e/gatewayapi/controller.go | 31 +++++++++++ test/e2e/gatewayapi/gatewayproxy.go | 79 +++++++++++------------------ 2 files changed, 62 insertions(+), 48 deletions(-) diff --git a/test/e2e/gatewayapi/controller.go b/test/e2e/gatewayapi/controller.go index 488e5e92f..d06b94ac0 100644 --- a/test/e2e/gatewayapi/controller.go +++ b/test/e2e/gatewayapi/controller.go @@ -8,10 +8,29 @@ import ( . "github.com/onsi/gomega" "github.com/stretchr/testify/assert" + "github.com/api7/api7-ingress-controller/test/e2e/framework" "github.com/api7/api7-ingress-controller/test/e2e/scaffold" ) var _ = Describe("Check if controller cache gets synced with correct resources", func() { + + var gatewayProxyYaml = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: api7-proxy-config +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" +` + var defautlGatewayClass = ` apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -34,6 +53,11 @@ spec: - name: http1 protocol: HTTP port: 80 + infrastructure: + parametersRef: + group: gateway.apisix.io + kind: GatewayProxy + name: api7-proxy-config ` var ResourceApplied = func(s *scaffold.Scaffold, resourType, resourceName, ns, resourceRaw string, observedGeneration int) { @@ -63,6 +87,13 @@ metadata: `, gatewayName)) Expect(err).NotTo(HaveOccurred(), "creating namespace") By(fmt.Sprintf("create GatewayClass for controller %s", s.GetControllerName())) + + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) + err = s.CreateResourceFromStringWithNamespace(gatewayProxy, gatewayName) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix()) err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, gatewayName, s.GetControllerName()), gatewayName) Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") diff --git a/test/e2e/gatewayapi/gatewayproxy.go b/test/e2e/gatewayapi/gatewayproxy.go index e6160aca5..ec94bf3ce 100644 --- a/test/e2e/gatewayapi/gatewayproxy.go +++ b/test/e2e/gatewayapi/gatewayproxy.go @@ -42,19 +42,6 @@ spec: name: api7-proxy-config ` - var gatewayWithoutProxy = ` -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: api7 -spec: - gatewayClassName: %s - listeners: - - name: http - protocol: HTTP - port: 80 -` - var gatewayProxyWithEnabledPlugin = ` apiVersion: gateway.apisix.io/v1alpha1 kind: GatewayProxy @@ -84,6 +71,15 @@ kind: GatewayProxy metadata: name: api7-proxy-config spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" plugins: - name: response-rewrite enabled: false @@ -98,6 +94,15 @@ kind: GatewayProxy metadata: name: api7-proxy-config spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" plugins: - name: error-page enabled: true @@ -117,6 +122,15 @@ kind: GatewayProxy metadata: name: api7-proxy-config spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" plugins: - name: error-page enabled: true @@ -226,7 +240,7 @@ spec: resp.Header("X-Proxy-Test").IsEqual("enabled") By("Update GatewayProxy with disabled plugin") - err := s.CreateResourceFromString(gatewayProxyWithDisabledPlugin) + err := s.CreateResourceFromString(fmt.Sprintf(gatewayProxyWithDisabledPlugin, framework.DashboardTLSEndpoint, s.AdminKey())) Expect(err).NotTo(HaveOccurred(), "updating GatewayProxy with disabled plugin") time.Sleep(5 * time.Second) @@ -242,37 +256,6 @@ spec: resp.Header("X-Proxy-Test").IsEmpty() }) - - It("Should work normally without GatewayProxy", func() { - By("Create HTTPRoute for Gateway with GatewayProxy") - resourceApplied("HTTPRoute", "test-route", fmt.Sprintf(httpRouteForTest, "api7"), 1) - - By("Check if the plugin is applied") - resp := s.NewAPISIXClient(). - GET("/get"). - WithHost("example.com"). - Expect(). - Status(200) - - resp.Header("X-Proxy-Test").IsEqual("enabled") - - By("Update Gateway without GatewayProxy") - err := s.CreateResourceFromString(fmt.Sprintf(gatewayWithoutProxy, gatewayClassName)) - Expect(err).NotTo(HaveOccurred(), "updating Gateway without GatewayProxy") - time.Sleep(5 * time.Second) - - By("Create HTTPRoute for Gateway without GatewayProxy") - resourceApplied("HTTPRoute", "test-route", fmt.Sprintf(httpRouteForTest, "api7"), 1) - - By("Check if the route works without plugin") - resp = s.NewAPISIXClient(). - GET("/get"). - WithHost("example.com"). - Expect(). - Status(200) - - resp.Header("X-Proxy-Test").IsEmpty() - }) }) Context("Test Gateway with PluginMetadata", func() { @@ -282,7 +265,7 @@ spec: It("Should work OK with error-page", func() { By("Update GatewayProxy with PluginMetadata") - err = s.CreateResourceFromString(gatewayProxyWithPluginMetadata0) + err = s.CreateResourceFromString(fmt.Sprintf(gatewayProxyWithPluginMetadata0, framework.DashboardTLSEndpoint, s.AdminKey())) Expect(err).ShouldNot(HaveOccurred()) time.Sleep(5 * time.Second) @@ -298,7 +281,7 @@ spec: Body().Contains("404 from plugin metadata") By("Update GatewayProxy with PluginMetadata") - err = s.CreateResourceFromString(gatewayProxyWithPluginMetadata1) + err = s.CreateResourceFromString(fmt.Sprintf(gatewayProxyWithPluginMetadata1, framework.DashboardTLSEndpoint, s.AdminKey())) Expect(err).ShouldNot(HaveOccurred()) time.Sleep(5 * time.Second) @@ -311,7 +294,7 @@ spec: Body().Contains(`{"error_msg":"404 Route Not Found"}`) By("Delete GatewayProxy") - err = s.DeleteResourceFromString(gatewayProxyWithPluginMetadata0) + err = s.DeleteResourceFromString(fmt.Sprintf(gatewayProxyWithPluginMetadata0, framework.DashboardTLSEndpoint, s.AdminKey())) Expect(err).ShouldNot(HaveOccurred()) time.Sleep(5 * time.Second)