diff --git a/.github/workflows/apisix-conformance-test.yml b/.github/workflows/apisix-conformance-test.yml index 7af6e98f8..322ca92d9 100644 --- a/.github/workflows/apisix-conformance-test.yml +++ b/.github/workflows/apisix-conformance-test.yml @@ -46,7 +46,7 @@ jobs: id: go uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" - name: Install kind run: | @@ -71,7 +71,7 @@ jobs: - name: Setup Go Env uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" - name: Login to Registry uses: docker/login-action@v1 diff --git a/.github/workflows/apisix-e2e-test.yml b/.github/workflows/apisix-e2e-test.yml index 1add09a52..84fed26c2 100644 --- a/.github/workflows/apisix-e2e-test.yml +++ b/.github/workflows/apisix-e2e-test.yml @@ -43,7 +43,7 @@ jobs: id: go uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" - name: Install kind run: | @@ -71,7 +71,7 @@ jobs: - name: Setup Go Env uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" - name: Login to Registry uses: docker/login-action@v1 diff --git a/.github/workflows/conformance-test.yml b/.github/workflows/conformance-test.yml index 6a30b6808..3d7b150df 100644 --- a/.github/workflows/conformance-test.yml +++ b/.github/workflows/conformance-test.yml @@ -43,7 +43,7 @@ jobs: id: go uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" - name: Install kind run: | @@ -69,7 +69,7 @@ jobs: - name: Setup Go Env uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" - name: Login to Registry uses: docker/login-action@v1 with: diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 8e7e57cc7..da300717c 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -43,7 +43,7 @@ jobs: id: go uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" - name: Install kind run: | @@ -58,7 +58,7 @@ jobs: e2e-test: needs: - prepare - runs-on: buildjet-2vcpu-ubuntu-2204 + runs-on: buildjet-4vcpu-ubuntu-2204 steps: - name: Checkout uses: actions/checkout@v4 @@ -68,7 +68,7 @@ jobs: - name: Setup Go Env uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" - name: Login to Registry uses: docker/login-action@v1 diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 2f5c57b24..0b41e3a4a 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -36,7 +36,7 @@ jobs: - name: Setup Go Env uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" - name: check go path for debug run: go env diff --git a/.github/workflows/push-docker-v2-dev.yaml b/.github/workflows/push-docker-v2-dev.yaml index 0b37186f6..c9622db46 100644 --- a/.github/workflows/push-docker-v2-dev.yaml +++ b/.github/workflows/push-docker-v2-dev.yaml @@ -35,7 +35,7 @@ jobs: - name: Setup Go Env uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" # - name: Set up QEMU # uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/push-docker.yaml b/.github/workflows/push-docker.yaml index 53c2b8efa..d1d4d0e1a 100644 --- a/.github/workflows/push-docker.yaml +++ b/.github/workflows/push-docker.yaml @@ -33,7 +33,7 @@ jobs: - name: Setup Go Env uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index aa62c0e0d..af697cc7c 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -36,7 +36,7 @@ jobs: - name: Setup Go Env uses: actions/setup-go@v4 with: - go-version: "1.22" + go-version: "1.23" - name: Run unit test working-directory: ./ run: | diff --git a/README.md b/README.md index 5fb2e5fbe..c6ec3f869 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The APISIX Ingress Controller allows you to run the APISIX Gateway as a Kubernet ### Prerequisites -* go version v1.22.0+ +* go version v1.23.0+ * docker version 17.03+. * kubectl version v1.11.3+. * Access to a Kubernetes v1.11.3+ cluster. diff --git a/go.mod b/go.mod index 05aeda446..a5d0cffbe 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/apache/apisix-ingress-controller -go 1.22.0 +go 1.23.0 -toolchain go1.22.5 +toolchain go1.23.7 require ( github.com/Masterminds/sprig/v3 v3.2.3 @@ -10,6 +10,7 @@ require ( github.com/gavv/httpexpect/v2 v2.16.0 github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/gruntwork-io/terratest v0.47.0 github.com/hashicorp/go-memdb v1.3.4 @@ -18,6 +19,7 @@ require ( github.com/onsi/ginkgo/v2 v2.20.0 github.com/onsi/gomega v1.34.1 github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.19.1 github.com/samber/lo v1.47.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 @@ -99,7 +101,6 @@ require ( github.com/google/btree v1.1.2 // indirect github.com/google/cel-go v0.20.1 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect @@ -152,7 +153,6 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pquerna/otp v1.2.0 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/internal/controller/consumer_controller.go b/internal/controller/consumer_controller.go index ef02e42f1..fd51f87a1 100644 --- a/internal/controller/consumer_controller.go +++ b/internal/controller/consumer_controller.go @@ -19,7 +19,6 @@ package controller import ( "context" - "fmt" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" @@ -276,13 +275,9 @@ func (r *ConsumerReconciler) updateStatus(consumer *v1alpha1.Consumer, err error NamespacedName: utils.NamespacedName(consumer), Resource: consumer.DeepCopy(), Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - t, ok := obj.(*v1alpha1.Consumer) - if !ok { - err := fmt.Errorf("unsupported object type %T", obj) - panic(err) - } - t.Status = consumer.Status - return t + cp := obj.(*v1alpha1.Consumer).DeepCopy() + cp.Status = consumer.Status + return cp }), }) } diff --git a/internal/controller/gatewayclass_controller.go b/internal/controller/gatewayclass_controller.go index ec3a7d92a..61d87683a 100644 --- a/internal/controller/gatewayclass_controller.go +++ b/internal/controller/gatewayclass_controller.go @@ -121,13 +121,9 @@ func (r *GatewayClassReconciler) Reconcile(ctx context.Context, req ctrl.Request NamespacedName: utils.NamespacedName(gc), Resource: gc.DeepCopy(), Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - t, ok := obj.(*gatewayv1.GatewayClass) - if !ok { - err := fmt.Errorf("unsupported object type %T", obj) - panic(err) - } - t.Status = gc.Status - return t + cp := obj.(*gatewayv1.GatewayClass).DeepCopy() + cp.Status = gc.Status + return cp }), }) } diff --git a/internal/controller/httproutepolicy.go b/internal/controller/httproutepolicy.go index 91b163b6a..f829cb6c8 100644 --- a/internal/controller/httproutepolicy.go +++ b/internal/controller/httproutepolicy.go @@ -20,7 +20,6 @@ package controller import ( "cmp" "context" - "fmt" "slices" networkingv1 "k8s.io/api/networking/v1" @@ -92,13 +91,9 @@ func (r *HTTPRouteReconciler) processHTTPRoutePolicies(tctx *provider.TranslateC NamespacedName: utils.NamespacedName(&policy), Resource: policy.DeepCopy(), Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - t, ok := obj.(*v1alpha1.HTTPRoutePolicy) - if !ok { - err := fmt.Errorf("unsupported object type %T", obj) - panic(err) - } - t.Status = policy.Status - return t + cp := obj.(*v1alpha1.HTTPRoutePolicy).DeepCopy() + cp.Status = policy.Status + return cp }), }) } @@ -170,13 +165,9 @@ func (r *IngressReconciler) processHTTPRoutePolicies(tctx *provider.TranslateCon NamespacedName: utils.NamespacedName(&policy), Resource: policy.DeepCopy(), Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - t, ok := obj.(*v1alpha1.HTTPRoutePolicy) - if !ok { - err := fmt.Errorf("unsupported object type %T", obj) - panic(err) - } - t.Status = policy.Status - return t + cp := obj.(*v1alpha1.HTTPRoutePolicy).DeepCopy() + cp.Status = policy.Status + return cp }), }) } @@ -282,13 +273,9 @@ func updateDeleteAncestors(updater status.Updater, policy v1alpha1.HTTPRoutePoli NamespacedName: utils.NamespacedName(&policy), Resource: policy.DeepCopy(), Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - t, ok := obj.(*v1alpha1.HTTPRoutePolicy) - if !ok { - err := fmt.Errorf("unsupported object type %T", obj) - panic(err) - } - t.Status = policy.Status - return t + cp := obj.(*v1alpha1.HTTPRoutePolicy).DeepCopy() + cp.Status = policy.Status + return cp }), }) } diff --git a/internal/controller/ingress_controller.go b/internal/controller/ingress_controller.go index d2a55079e..86c4d7ff0 100644 --- a/internal/controller/ingress_controller.go +++ b/internal/controller/ingress_controller.go @@ -639,13 +639,9 @@ func (r *IngressReconciler) updateStatus(ctx context.Context, tctx *provider.Tra NamespacedName: utils.NamespacedName(ingress), Resource: ingress.DeepCopy(), Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - t, ok := obj.(*networkingv1.Ingress) - if !ok { - err := fmt.Errorf("unsupported object type %T", obj) - panic(err) - } - t.Status = ingress.Status - return t + cp := obj.(*networkingv1.Ingress).DeepCopy() + cp.Status = ingress.Status + return cp }), }) return nil diff --git a/internal/controller/policies.go b/internal/controller/policies.go index 3c8f8bfa7..c272c71ec 100644 --- a/internal/controller/policies.go +++ b/internal/controller/policies.go @@ -158,13 +158,9 @@ func ProcessBackendTrafficPolicy( NamespacedName: utils.NamespacedName(policy), Resource: policy.DeepCopy(), Mutator: status.MutatorFunc(func(obj client.Object) client.Object { - t, ok := obj.(*v1alpha1.BackendTrafficPolicy) - if !ok { - err := fmt.Errorf("unsupported object type %T", obj) - panic(err) - } - t.Status = policy.Status - return t + cp := obj.(*v1alpha1.BackendTrafficPolicy).DeepCopy() + cp.Status = policy.Status + return cp }), }) } diff --git a/internal/controller/status/updater.go b/internal/controller/status/updater.go index 47ac41ee3..019c18468 100644 --- a/internal/controller/status/updater.go +++ b/internal/controller/status/updater.go @@ -22,16 +22,25 @@ import ( "sync" "github.com/go-logr/logr" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/apache/apisix-ingress-controller/api/v1alpha1" + v2 "github.com/apache/apisix-ingress-controller/api/v2" + types "github.com/apache/apisix-ingress-controller/internal/types" + pkgmetrics "github.com/apache/apisix-ingress-controller/pkg/metrics" ) const UpdateChannelBufferSize = 1000 type Update struct { - NamespacedName types.NamespacedName + NamespacedName k8stypes.NamespacedName Resource client.Object Mutator Mutator } @@ -50,6 +59,13 @@ func (m MutatorFunc) Mutate(obj client.Object) client.Object { return m(obj) } +var cmpIgnoreLastTT = cmp.Options{ + cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime"), + cmpopts.IgnoreMapEntries(func(k string, _ any) bool { + return k == "lastTransitionTime" + }), +} + type UpdateHandler struct { log logr.Logger client client.Client @@ -94,8 +110,20 @@ func (u *UpdateHandler) updateStatus(ctx context.Context, update Update) error { return nil } + if statusEqual(obj, newObj, cmpIgnoreLastTT) { + u.log.V(1).Info("status is equal, skipping update", "name", update.NamespacedName.Name, + "namespace", update.NamespacedName.Namespace, + "kind", types.KindOf(obj)) + return nil + } + newObj.SetUID(obj.GetUID()) + u.log.Info("updating status", "name", update.NamespacedName.Name, + "namespace", update.NamespacedName.Namespace, + "kind", types.KindOf(newObj), + ) + return u.client.Status().Update(ctx, newObj) } @@ -110,8 +138,12 @@ func (u *UpdateHandler) Start(ctx context.Context) error { case <-ctx.Done(): return nil case update := <-u.updateChannel: - u.log.Info("received a status update", "namespace", update.NamespacedName.Namespace, - "name", update.NamespacedName.Name) + // Decrement queue length after removing item from queue + pkgmetrics.DecStatusQueueLength() + u.log.V(1).Info("received a status update", "namespace", update.NamespacedName.Namespace, + "name", update.NamespacedName.Name, + "kind", types.KindOf(update.Resource), + ) u.apply(ctx, update) } @@ -137,4 +169,85 @@ type UpdateWriter struct { func (u *UpdateWriter) Update(update Update) { u.wg.Wait() u.updateChannel <- update + // Increment queue length after adding new item + pkgmetrics.IncStatusQueueLength() +} + +func statusEqual(a, b any, opts ...cmp.Option) bool { + var statusA, statusB any + + switch a := a.(type) { + case *gatewayv1.GatewayClass: + b, ok := b.(*gatewayv1.GatewayClass) + if !ok { + return false + } + statusA, statusB = a.Status, b.Status + + case *gatewayv1.Gateway: + b, ok := b.(*gatewayv1.Gateway) + if !ok { + return false + } + statusA, statusB = a.Status, b.Status + + case *gatewayv1.HTTPRoute: + b, ok := b.(*gatewayv1.HTTPRoute) + if !ok { + return false + } + statusA, statusB = a.Status, b.Status + case *v2.ApisixRoute: + b, ok := b.(*v2.ApisixRoute) + if !ok { + return false + } + statusA, statusB = a.Status, b.Status + case *v2.ApisixGlobalRule: + b, ok := b.(*v2.ApisixGlobalRule) + if !ok { + return false + } + statusA, statusB = a.Status, b.Status + case *v2.ApisixPluginConfig: + b, ok := b.(*v2.ApisixPluginConfig) + if !ok { + return false + } + statusA, statusB = a.Status, b.Status + case *v2.ApisixTls: + b, ok := b.(*v2.ApisixTls) + if !ok { + return false + } + statusA, statusB = a.Status, b.Status + case *v2.ApisixConsumer: + b, ok := b.(*v2.ApisixConsumer) + if !ok { + return false + } + statusA, statusB = a.Status, b.Status + case *v1alpha1.HTTPRoutePolicy: + b, ok := b.(*v1alpha1.HTTPRoutePolicy) + if !ok { + return false + } + statusA, statusB = a.Status, b.Status + case *v1alpha1.BackendTrafficPolicy: + b, ok := b.(*v1alpha1.BackendTrafficPolicy) + if !ok { + return false + } + statusA, statusB = a.Status, b.Status + case *v1alpha1.Consumer: + b, ok := b.(*v1alpha1.Consumer) + if !ok { + return false + } + statusA, statusB = a.Status, b.Status + default: + return false + } + + return cmp.Equal(statusA, statusB, opts...) } diff --git a/internal/manager/run.go b/internal/manager/run.go index b48746c16..a27b27325 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -42,6 +42,7 @@ import ( "github.com/apache/apisix-ingress-controller/internal/controller/config" "github.com/apache/apisix-ingress-controller/internal/controller/status" "github.com/apache/apisix-ingress-controller/internal/provider/adc" + _ "github.com/apache/apisix-ingress-controller/pkg/metrics" ) var ( diff --git a/internal/provider/adc/adc.go b/internal/provider/adc/adc.go index fe88da1be..6b217ef4a 100644 --- a/internal/provider/adc/adc.go +++ b/internal/provider/adc/adc.go @@ -43,6 +43,7 @@ import ( "github.com/apache/apisix-ingress-controller/internal/provider/adc/translator" "github.com/apache/apisix-ingress-controller/internal/types" "github.com/apache/apisix-ingress-controller/internal/utils" + pkgmetrics "github.com/apache/apisix-ingress-controller/pkg/metrics" ) type adcConfig struct { @@ -389,6 +390,8 @@ func (d *adcClient) sync(ctx context.Context, task Task) error { return nil } + var errs types.ADCExecutionErrors + // for global rules, we need to list all global rules and set it to the task resources if slices.Contains(task.ResourceTypes, "global_rule") { for _, config := range task.configs { @@ -410,42 +413,93 @@ func (d *adcClient) sync(ctx context.Context, task Task) error { task.Resources.GlobalRules = globalrule log.Debugw("syncing resources global rules", zap.Any("globalRules", task.Resources.GlobalRules)) + fileIOStart := time.Now() syncFilePath, cleanup, err := prepareSyncFile(task.Resources) if err != nil { + pkgmetrics.RecordFileIODuration("prepare_sync_file", "failure", time.Since(fileIOStart).Seconds()) return err } + pkgmetrics.RecordFileIODuration("prepare_sync_file", "success", time.Since(fileIOStart).Seconds()) defer cleanup() args := BuildADCExecuteArgs(syncFilePath, task.Labels, task.ResourceTypes) - if err := d.executor.Execute(ctx, d.BackendMode, config, args); err != nil { + // Record sync duration for each config + startTime := time.Now() + resourceType := strings.Join(task.ResourceTypes, ",") + if resourceType == "" { + resourceType = "all" + } + + err = d.executor.Execute(ctx, d.BackendMode, config, args) + duration := time.Since(startTime).Seconds() + + status := "success" + if err != nil { + status = "failure" log.Errorw("failed to execute adc command", zap.Error(err), zap.Any("config", config)) - return err + + var execErr types.ADCExecutionError + if errors.As(err, &execErr) { + errs.Errors = append(errs.Errors, execErr) + pkgmetrics.RecordExecutionError(config.Name, execErr.Name) + } else { + pkgmetrics.RecordExecutionError(config.Name, "unknown") + } } + + // Record metrics + pkgmetrics.RecordSyncDuration(config.Name, resourceType, status, duration) } + if len(errs.Errors) > 0 { + return errs + } return nil } + // Record file I/O duration + fileIOStart := time.Now() // every task resources is the same, so we can use the first config to prepare the sync file syncFilePath, cleanup, err := prepareSyncFile(task.Resources) if err != nil { + pkgmetrics.RecordFileIODuration("prepare_sync_file", "failure", time.Since(fileIOStart).Seconds()) return err } + pkgmetrics.RecordFileIODuration("prepare_sync_file", "success", time.Since(fileIOStart).Seconds()) defer cleanup() args := BuildADCExecuteArgs(syncFilePath, task.Labels, task.ResourceTypes) - var errs types.ADCExecutionErrors for _, config := range task.configs { - if err := d.executor.Execute(ctx, d.BackendMode, config, args); err != nil { + // Record sync duration for each config + startTime := time.Now() + resourceType := strings.Join(task.ResourceTypes, ",") + if resourceType == "" { + resourceType = "all" + } + + err := d.executor.Execute(ctx, d.BackendMode, config, args) + duration := time.Since(startTime).Seconds() + + status := "success" + if err != nil { + status = "failure" log.Errorw("failed to execute adc command", zap.Error(err), zap.Any("config", config)) + var execErr types.ADCExecutionError if errors.As(err, &execErr) { errs.Errors = append(errs.Errors, execErr) + pkgmetrics.RecordExecutionError(config.Name, execErr.Name) + } else { + pkgmetrics.RecordExecutionError(config.Name, "unknown") } } + + // Record metrics + pkgmetrics.RecordSyncDuration(config.Name, resourceType, status, duration) } + if len(errs.Errors) > 0 { return errs } diff --git a/internal/provider/adc/config.go b/internal/provider/adc/config.go index 7eb55fc58..6617e28c9 100644 --- a/internal/provider/adc/config.go +++ b/internal/provider/adc/config.go @@ -207,7 +207,6 @@ func (d *adcClient) updateConfigForGatewayProxy(tctx *provider.TranslateContext, for _, ref := range referrers { d.configs[ref] = *config } - return nil } diff --git a/internal/types/k8s.go b/internal/types/k8s.go index 4bdf1e93d..63bc9190c 100644 --- a/internal/types/k8s.go +++ b/internal/types/k8s.go @@ -17,21 +17,77 @@ package types +import ( + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/networking/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/apache/apisix-ingress-controller/api/v1alpha1" + v2 "github.com/apache/apisix-ingress-controller/api/v2" +) + const DefaultIngressClassAnnotation = "ingressclass.kubernetes.io/is-default-class" const ( - KindGateway = "Gateway" - KindHTTPRoute = "HTTPRoute" - KindGatewayClass = "GatewayClass" - KindIngress = "Ingress" - KindIngressClass = "IngressClass" - KindGatewayProxy = "GatewayProxy" - KindSecret = "Secret" - KindService = "Service" - KindApisixRoute = "ApisixRoute" - KindApisixGlobalRule = "ApisixGlobalRule" - KindApisixPluginConfig = "ApisixPluginConfig" - KindPod = "Pod" - KindApisixTls = "ApisixTls" - KindApisixConsumer = "ApisixConsumer" + KindGateway = "Gateway" + KindHTTPRoute = "HTTPRoute" + KindGatewayClass = "GatewayClass" + KindIngress = "Ingress" + KindIngressClass = "IngressClass" + KindGatewayProxy = "GatewayProxy" + KindSecret = "Secret" + KindService = "Service" + KindApisixRoute = "ApisixRoute" + KindApisixGlobalRule = "ApisixGlobalRule" + KindApisixPluginConfig = "ApisixPluginConfig" + KindPod = "Pod" + KindApisixTls = "ApisixTls" + KindApisixConsumer = "ApisixConsumer" + KindHTTPRoutePolicy = "HTTPRoutePolicy" + KindBackendTrafficPolicy = "BackendTrafficPolicy" + KindConsumer = "Consumer" + KindPluginConfig = "PluginConfig" ) + +func KindOf(obj any) string { + switch obj.(type) { + case *gatewayv1.Gateway: + return KindGateway + case *gatewayv1.HTTPRoute: + return KindHTTPRoute + case *gatewayv1.GatewayClass: + return KindGatewayClass + case *v1.Ingress: + return KindIngress + case *v1.IngressClass: + return KindIngressClass + case *corev1.Secret: + return KindSecret + case *corev1.Service: + return KindService + case *v2.ApisixRoute: + return KindApisixRoute + case *v2.ApisixGlobalRule: + return KindApisixGlobalRule + case *v2.ApisixPluginConfig: + return KindApisixPluginConfig + case *corev1.Pod: + return KindPod + case *v2.ApisixTls: + return KindApisixTls + case *v2.ApisixConsumer: + return KindApisixConsumer + case *v1alpha1.HTTPRoutePolicy: + return KindHTTPRoutePolicy + case *v1alpha1.BackendTrafficPolicy: + return KindBackendTrafficPolicy + case *v1alpha1.GatewayProxy: + return KindGatewayProxy + case *v1alpha1.Consumer: + return KindConsumer + case *v1alpha1.PluginConfig: + return KindPluginConfig + default: + return "Unknown" + } +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 000000000..c9537fe12 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + // ADC sync operation duration histogram + ADCSyncDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "apisix_ingress_adc_sync_duration_seconds", + Help: "Time spent on ADC sync operations", + Buckets: prometheus.DefBuckets, + }, + []string{"config_name", "resource_type", "status"}, + ) + + // ADC sync operation counter + ADCSyncTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "apisix_ingress_adc_sync_total", + Help: "Total number of ADC sync operations", + }, + []string{"config_name", "resource_type", "status"}, + ) + + // ADC execution errors counter + ADCExecutionErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "apisix_ingress_adc_execution_errors_total", + Help: "Total number of ADC execution errors", + }, + []string{"config_name", "error_type"}, + ) + + // Status update channel queue length gauge + StatusUpdateQueueLength = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "apisix_ingress_status_update_queue_length", + Help: "Current length of the status update queue", + }, + ) + + // File I/O operation duration histogram + FileIODuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "apisix_ingress_file_io_duration_seconds", + Help: "Time spent on file I/O operations", + Buckets: prometheus.DefBuckets, + }, + []string{"operation", "status"}, + ) +) + +// init registers all metrics with the global prometheus registry +func init() { + // Register metrics with controller-runtime's metrics registry + metrics.Registry.MustRegister( + ADCSyncDuration, + ADCSyncTotal, + ADCExecutionErrors, + StatusUpdateQueueLength, + FileIODuration, + ) +} + +// RecordSyncDuration records the duration of an ADC sync operation +func RecordSyncDuration(configName, resourceType, status string, duration float64) { + ADCSyncDuration.WithLabelValues(configName, resourceType, status).Observe(duration) + ADCSyncTotal.WithLabelValues(configName, resourceType, status).Inc() +} + +// RecordExecutionError records an ADC execution error +func RecordExecutionError(configName, errorType string) { + ADCExecutionErrors.WithLabelValues(configName, errorType).Inc() +} + +// UpdateStatusQueueLength updates the status update queue length gauge +func UpdateStatusQueueLength(length float64) { + StatusUpdateQueueLength.Set(length) +} + +// IncStatusQueueLength increments the status update queue length gauge by 1 +func IncStatusQueueLength() { + StatusUpdateQueueLength.Inc() +} + +// DecStatusQueueLength decrements the status update queue length gauge by 1 +func DecStatusQueueLength() { + StatusUpdateQueueLength.Dec() +} + +// RecordFileIODuration records the duration of a file I/O operation +func RecordFileIODuration(operation, status string, duration float64) { + FileIODuration.WithLabelValues(operation, status).Observe(duration) +} diff --git a/test/e2e/apisix/e2e_test.go b/test/e2e/apisix/e2e_test.go index ea814157e..03fe0ca61 100644 --- a/test/e2e/apisix/e2e_test.go +++ b/test/e2e/apisix/e2e_test.go @@ -24,7 +24,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - _ "github.com/apache/apisix-ingress-controller/test/e2e/crds" + _ "github.com/apache/apisix-ingress-controller/test/e2e/crds/v1alpha1" + _ "github.com/apache/apisix-ingress-controller/test/e2e/crds/v2" "github.com/apache/apisix-ingress-controller/test/e2e/framework" _ "github.com/apache/apisix-ingress-controller/test/e2e/gatewayapi" _ "github.com/apache/apisix-ingress-controller/test/e2e/ingress" diff --git a/test/e2e/crds/backendtrafficpolicy.go b/test/e2e/crds/v1alpha1/backendtrafficpolicy.go similarity index 99% rename from test/e2e/crds/backendtrafficpolicy.go rename to test/e2e/crds/v1alpha1/backendtrafficpolicy.go index 82b88a201..d91888c5f 100644 --- a/test/e2e/crds/backendtrafficpolicy.go +++ b/test/e2e/crds/v1alpha1/backendtrafficpolicy.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package gatewayapi +package v1alpha1 import ( "fmt" diff --git a/test/e2e/crds/consumer.go b/test/e2e/crds/v1alpha1/consumer.go similarity index 99% rename from test/e2e/crds/consumer.go rename to test/e2e/crds/v1alpha1/consumer.go index 26b8554f5..d2db2a5b7 100644 --- a/test/e2e/crds/consumer.go +++ b/test/e2e/crds/v1alpha1/consumer.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package gatewayapi +package v1alpha1 import ( "fmt" diff --git a/test/e2e/crds/gatewayproxy.go b/test/e2e/crds/v1alpha1/gatewayproxy.go similarity index 99% rename from test/e2e/crds/gatewayproxy.go rename to test/e2e/crds/v1alpha1/gatewayproxy.go index 60cbead9a..2c582dd20 100644 --- a/test/e2e/crds/gatewayproxy.go +++ b/test/e2e/crds/v1alpha1/gatewayproxy.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package gatewayapi +package v1alpha1 import ( "context" diff --git a/test/e2e/apisix/basic.go b/test/e2e/crds/v2/basic.go similarity index 99% rename from test/e2e/apisix/basic.go rename to test/e2e/crds/v2/basic.go index 245419f1c..1d16a9f78 100644 --- a/test/e2e/apisix/basic.go +++ b/test/e2e/crds/v2/basic.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package apisix +package v2 import ( . "github.com/onsi/ginkgo/v2" diff --git a/test/e2e/apisix/consumer.go b/test/e2e/crds/v2/consumer.go similarity index 79% rename from test/e2e/apisix/consumer.go rename to test/e2e/crds/v2/consumer.go index 31668c246..ee243fb1e 100644 --- a/test/e2e/apisix/consumer.go +++ b/test/e2e/crds/v2/consumer.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package apisix +package v2 import ( "fmt" @@ -275,9 +275,6 @@ spec: ` ) - request := func(path string, username, password string) int { - return s.NewAPISIXClient().GET(path).WithBasicAuth(username, password).WithHost("httpbin").Expect().Raw().StatusCode - } It("Basic tests", func() { By("apply ApisixRoute") applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, &apiv2.ApisixRoute{}, defaultApisixRoute) @@ -286,22 +283,50 @@ spec: applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "test-consumer"}, &apiv2.ApisixConsumer{}, basicAuth) By("verify ApisixRoute with ApisixConsumer") - Eventually(request).WithArguments("/get", "invalid-username", "invalid-password"). - WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusUnauthorized)) - - Eventually(request).WithArguments("/get", "test-user", "test-password").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + BasicAuth: &scaffold.BasicAuth{ + Username: "invalid-username", + Password: "invalid-password", + }, + Check: scaffold.WithExpectedStatus(http.StatusUnauthorized), + }) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + BasicAuth: &scaffold.BasicAuth{ + Username: "test-user", + Password: "test-password", + }, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("Delete ApisixConsumer") err := s.DeleteResource("ApisixConsumer", "test-consumer") Expect(err).ShouldNot(HaveOccurred(), "deleting ApisixConsumer") - Eventually(request).WithArguments("/get", "test-user", "test-password"). - WithTimeout(5 * time.Second).ProbeEvery(time.Second). - Should(Equal(http.StatusUnauthorized)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + BasicAuth: &scaffold.BasicAuth{ + Username: "test-user", + Password: "test-password", + }, + Check: scaffold.WithExpectedStatus(http.StatusUnauthorized), + }) By("delete ApisixRoute") err = s.DeleteResource("ApisixRoute", "default") Expect(err).ShouldNot(HaveOccurred(), "deleting ApisixRoute") - Eventually(request).WithArguments("/headers", "", "").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) }) It("SecretRef tests", func() { @@ -316,29 +341,70 @@ spec: applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "test-consumer"}, &apiv2.ApisixConsumer{}, basicAuthWithSecret) By("verify ApisixRoute with ApisixConsumer") - Eventually(request).WithArguments("/get", "", "").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusUnauthorized)) - Eventually(request).WithArguments("/get", "foo", "bar").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + Check: scaffold.WithExpectedStatus(http.StatusUnauthorized), + }) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + BasicAuth: &scaffold.BasicAuth{ + Username: "foo", + Password: "bar", + }, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("update Secret") err = s.CreateResourceFromString(secretUpdated) Expect(err).ShouldNot(HaveOccurred(), "updating Secret for ApisixConsumer") - - Eventually(request).WithArguments("/get", "foo", "bar").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusUnauthorized)) - Eventually(request).WithArguments("/get", "foo-new-user", "bar-new-password"). - WithTimeout(5 * time.Second).ProbeEvery(time.Second). - Should(Equal(http.StatusOK)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + BasicAuth: &scaffold.BasicAuth{ + Username: "foo", + Password: "bar", + }, + Check: scaffold.WithExpectedStatus(http.StatusUnauthorized), + }) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + BasicAuth: &scaffold.BasicAuth{ + Username: "foo-new-user", + Password: "bar-new-password", + }, + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) By("Delete ApisixConsumer") err = s.DeleteResource("ApisixConsumer", "test-consumer") Expect(err).ShouldNot(HaveOccurred(), "deleting ApisixConsumer") - Eventually(request).WithArguments("/get", "foo-new-user", "bar-new-password"). - WithTimeout(5 * time.Second).ProbeEvery(time.Second). - Should(Equal(http.StatusUnauthorized)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + BasicAuth: &scaffold.BasicAuth{ + Username: "foo-new-user", + Password: "bar-new-password", + }, + Check: scaffold.WithExpectedStatus(http.StatusUnauthorized), + }) By("delete ApisixRoute") err = s.DeleteResource("ApisixRoute", "default") Expect(err).ShouldNot(HaveOccurred(), "deleting ApisixRoute") - Eventually(request).WithArguments("/get", "", "").WithTimeout(5 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "httpbin", + Check: scaffold.WithExpectedStatus(http.StatusNotFound), + }) }) }) }) diff --git a/test/e2e/apisix/globalrule.go b/test/e2e/crds/v2/globalrule.go similarity index 99% rename from test/e2e/apisix/globalrule.go rename to test/e2e/crds/v2/globalrule.go index 0e3f2f415..738d43e0d 100644 --- a/test/e2e/apisix/globalrule.go +++ b/test/e2e/crds/v2/globalrule.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package apisix +package v2 import ( "fmt" diff --git a/test/e2e/apisix/pluginconfig.go b/test/e2e/crds/v2/pluginconfig.go similarity index 99% rename from test/e2e/apisix/pluginconfig.go rename to test/e2e/crds/v2/pluginconfig.go index 2c4c27fc5..589427eb9 100644 --- a/test/e2e/apisix/pluginconfig.go +++ b/test/e2e/crds/v2/pluginconfig.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package apisix +package v2 import ( "fmt" diff --git a/test/e2e/apisix/route.go b/test/e2e/crds/v2/route.go similarity index 94% rename from test/e2e/apisix/route.go rename to test/e2e/crds/v2/route.go index eb447f63d..70b6191ac 100644 --- a/test/e2e/apisix/route.go +++ b/test/e2e/crds/v2/route.go @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. -package apisix +package v2 import ( "context" "fmt" + "io" "net" "net/http" "time" @@ -96,6 +97,37 @@ spec: err := s.DeleteResource("ApisixRoute", "default") Expect(err).ShouldNot(HaveOccurred(), "deleting ApisixRoute") Eventually(request).WithArguments("/headers").WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusNotFound)) + + By("request /metrics endpoint from controller") + + // Get the metrics service endpoint + metricsURL := s.GetMetricsEndpoint() + + By("verify metrics content") + resp, err := http.Get(metricsURL) + Expect(err).ShouldNot(HaveOccurred(), "request metrics endpoint") + defer func() { + _ = resp.Body.Close() + }() + + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + + body, err := io.ReadAll(resp.Body) + Expect(err).ShouldNot(HaveOccurred(), "read metrics response") + + bodyStr := string(body) + + // Verify prometheus format + Expect(resp.Header.Get("Content-Type")).Should(ContainSubstring("text/plain; version=0.0.4; charset=utf-8")) + + // Verify specific metrics from metrics.go exist + Expect(bodyStr).Should(ContainSubstring("apisix_ingress_adc_sync_duration_seconds")) + Expect(bodyStr).Should(ContainSubstring("apisix_ingress_adc_sync_total")) + Expect(bodyStr).Should(ContainSubstring("apisix_ingress_status_update_queue_length")) + Expect(bodyStr).Should(ContainSubstring("apisix_ingress_file_io_duration_seconds")) + + // Log metrics for debugging + fmt.Printf("Metrics endpoint response:\n%s\n", bodyStr) }) It("Test plugins in ApisixRoute", func() { diff --git a/test/e2e/apisix/status.go b/test/e2e/crds/v2/status.go similarity index 79% rename from test/e2e/apisix/status.go rename to test/e2e/crds/v2/status.go index 7bc0b6a53..c4807ee79 100644 --- a/test/e2e/apisix/status.go +++ b/test/e2e/crds/v2/status.go @@ -15,18 +15,21 @@ // specific language governing permissions and limitations // under the License. -package apisix +package v2 import ( "fmt" + "log" "os" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/yaml" apiv2 "github.com/apache/apisix-ingress-controller/api/v2" + "github.com/apache/apisix-ingress-controller/internal/provider/adc" "github.com/apache/apisix-ingress-controller/test/e2e/framework" "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" ) @@ -100,16 +103,31 @@ spec: Expect(err).NotTo(HaveOccurred(), "creating ApisixRoute with valid plugin") By("check ApisixRoute status") - s.RetryAssertion(func() string { - output, _ := s.GetOutputFromString("ar", "default", "-o", "yaml") - return output - }).Should( - And( - ContainSubstring(`status: "False"`), - ContainSubstring(`reason: SyncFailed`), - ContainSubstring(`unknown plugin [non-existent-plugin]`), - ), - ) + if os.Getenv("PROVIDER_TYPE") == "apisix" { + s.RetryAssertion(func() string { + output, _ := s.GetOutputFromString("ar", "default", "-o", "yaml") + log.Printf("output: %s", output) + return output + }).Should( + And( + ContainSubstring(`status: "False"`), + ContainSubstring(`reason: SyncFailed`), + ContainSubstring(`unknown plugin [non-existent-plugin]`), + ), + ) + } else { + s.RetryAssertion(func() string { + output, _ := s.GetOutputFromString("ar", "default", "-o", "yaml") + log.Printf("output: %s", output) + return output + }).Should( + And( + ContainSubstring(`status: "False"`), + ContainSubstring(`reason: SyncFailed`), + ContainSubstring(`(non-existent-plugin) not found`), + ), + ) + } By("Update ApisixRoute") applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, &apiv2.ApisixRoute{}, ar) @@ -124,6 +142,9 @@ spec: }) It("dataplane unavailable", func() { + if os.Getenv("PROVIDER_TYPE") == adc.BackendModeAPI7EE { + Skip("skip for api7ee mode because it use dashboard admin api") + } By("apply ApisixRoute") applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, &apiv2.ApisixRoute{}, ar) @@ -171,6 +192,32 @@ spec: Check: scaffold.WithExpectedStatus(200), }) }) + + It("update the same status only once", func() { + By("apply ApisixRoute") + applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"}, &apiv2.ApisixRoute{}, ar) + + output, _ := s.GetOutputFromString("ar", "default", "-o", "yaml") + + var route apiv2.ApisixRoute + err := yaml.Unmarshal([]byte(output), &route) + Expect(err).NotTo(HaveOccurred(), "unmarshalling ApisixRoute") + + Expect(route.Status.Conditions).Should(HaveLen(1), "should have one condition") + + s.Deployer.ScaleIngress(0) + s.Deployer.ScaleIngress(1) + + output, _ = s.GetOutputFromString("ar", "default", "-o", "yaml") + + var route2 apiv2.ApisixRoute + err = yaml.Unmarshal([]byte(output), &route2) + Expect(err).NotTo(HaveOccurred(), "unmarshalling ApisixRoute") + + Expect(route2.Status.Conditions).Should(HaveLen(1), "should have one condition") + Expect(route2.Status.Conditions[0].LastTransitionTime).To(Equal(route.Status.Conditions[0].LastTransitionTime), + "should not update the same status condition again") + }) }) Context("Test HTTPRoute Sync Status", func() { @@ -263,6 +310,9 @@ spec: }) It("dataplane unavailable", func() { + if os.Getenv("PROVIDER_TYPE") == adc.BackendModeAPI7EE { + Skip("skip for api7ee mode because it use dashboard admin api") + } By("Create HTTPRoute") err := s.CreateResourceFromString(httproute) Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute") diff --git a/test/e2e/apisix/tls.go b/test/e2e/crds/v2/tls.go similarity index 99% rename from test/e2e/apisix/tls.go rename to test/e2e/crds/v2/tls.go index 4a6359379..e91da3e1a 100644 --- a/test/e2e/apisix/tls.go +++ b/test/e2e/crds/v2/tls.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package apisix +package v2 import ( "context" diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 90e182ccc..e65f4214b 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -25,7 +25,8 @@ import ( . "github.com/onsi/gomega" _ "github.com/apache/apisix-ingress-controller/test/e2e/api7" - _ "github.com/apache/apisix-ingress-controller/test/e2e/crds" + _ "github.com/apache/apisix-ingress-controller/test/e2e/crds/v1alpha1" + _ "github.com/apache/apisix-ingress-controller/test/e2e/crds/v2" "github.com/apache/apisix-ingress-controller/test/e2e/framework" _ "github.com/apache/apisix-ingress-controller/test/e2e/gatewayapi" _ "github.com/apache/apisix-ingress-controller/test/e2e/ingress" diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index 6b4adbbc8..b01beca1d 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -90,6 +90,9 @@ spec: - name: admin containerPort: 9180 protocol: TCP + - name: control + containerPort: 9090 + protocol: TCP volumeMounts: - name: config-writable mountPath: /usr/local/apisix/conf diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index f6dc990aa..7cb652fe7 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -346,10 +346,10 @@ metadata: namespace: {{ .Namespace }} spec: ports: - - name: https - port: 8443 + - name: metrics + port: 8080 protocol: TCP - targetPort: 8443 + targetPort: 8080 selector: control-plane: controller-manager --- @@ -399,19 +399,16 @@ spec: initialDelaySeconds: 15 periodSeconds: 20 name: manager + ports: + - name: metrics + containerPort: 8080 + protocol: TCP readinessProbe: httpGet: path: /readyz port: 8081 initialDelaySeconds: 5 periodSeconds: 10 - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 10m - memory: 64Mi securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/test/e2e/scaffold/api7_deployer.go b/test/e2e/scaffold/api7_deployer.go index d6949c017..d4846bf7e 100644 --- a/test/e2e/scaffold/api7_deployer.go +++ b/test/e2e/scaffold/api7_deployer.go @@ -54,7 +54,7 @@ func (s *API7Deployer) BeforeEach() { Namespace: s.namespace, } if s.opts.ControllerName == "" { - s.opts.ControllerName = fmt.Sprintf("%s/%d", DefaultControllerName, time.Now().Nanosecond()) + s.opts.ControllerName = fmt.Sprintf("%s/%s", DefaultControllerName, s.namespace) } s.finalizers = nil if s.label == nil { @@ -122,14 +122,14 @@ func (s *API7Deployer) AfterEach() { Expect(err).NotTo(HaveOccurred(), "cleaning up additional gateway group") } - // if the test case is successful, just delete namespace - err := k8s.DeleteNamespaceE(s.t, s.kubectlOptions, s.namespace) - Expect(err).NotTo(HaveOccurred(), "deleting namespace "+s.namespace) - for i := len(s.finalizers) - 1; i >= 0; i-- { runWithRecover(s.finalizers[i]) } + // if the test case is successful, just delete namespace + err := k8s.DeleteNamespaceE(s.t, s.kubectlOptions, s.namespace) + Expect(err).NotTo(HaveOccurred(), "deleting namespace "+s.namespace) + // Wait for a while to prevent the worker node being overwhelming // (new cases will be run). time.Sleep(3 * time.Second) diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 3c2105ed4..e532bbb24 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -444,3 +444,12 @@ func NewClient(scheme, host string) *httpexpect.Expect { ), }) } + +func (s *Scaffold) GetMetricsEndpoint() string { + tunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-ingress-controller-manager-metrics-service", 8080, 8080) + if err := tunnel.ForwardPortE(s.t); err != nil { + return "" + } + s.addFinalizers(tunnel.Close) + return fmt.Sprintf("http://%s/metrics", tunnel.Endpoint()) +}