From bc97d322069f7a40fa7949583e1a62b4f67e0f67 Mon Sep 17 00:00:00 2001 From: Tina Usova Date: Tue, 2 Sep 2025 16:40:13 +0100 Subject: [PATCH 01/11] Adding logging to framework methodes --- tests/framework/ngf.go | 7 +++++-- tests/suite/advanced_routing_test.go | 2 ++ tests/suite/system_suite_test.go | 3 +++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/framework/ngf.go b/tests/framework/ngf.go index 660cb79e8e..26a7390775 100644 --- a/tests/framework/ngf.go +++ b/tests/framework/ngf.go @@ -42,6 +42,7 @@ type InstallationConfig struct { func InstallGatewayAPI(apiVersion string) ([]byte, error) { apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion) GinkgoWriter.Printf("Installing Gateway API version %q at API path %q\n", apiVersion, apiPath) + GinkgoWriter.Printf("Installing Gateway API version %q at API path %q\n", apiVersion, apiPath) cmd := exec.CommandContext( context.Background(), @@ -62,14 +63,18 @@ func InstallGatewayAPI(apiVersion string) ([]byte, error) { func UninstallGatewayAPI(apiVersion string) ([]byte, error) { apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion) GinkgoWriter.Printf("Uninstalling Gateway API version %q at API path %q\n", apiVersion, apiPath) + GinkgoWriter.Printf("Uninstalling Gateway API version %q at API path %q\n", apiVersion, apiPath) output, err := exec.CommandContext(context.Background(), "kubectl", "delete", "-f", apiPath).CombinedOutput() if err != nil && !strings.Contains(string(output), "not found") { GinkgoWriter.Printf("Error uninstalling Gateway API version %q: %v\n", apiVersion, err) + GinkgoWriter.Printf("Error uninstalling Gateway API version %q: %v\n", apiVersion, err) + return output, err } GinkgoWriter.Printf("Successfully uninstalled Gateway API version %q\n", apiVersion) + GinkgoWriter.Printf("Successfully uninstalled Gateway API version %q\n", apiVersion) return nil, nil } @@ -102,8 +107,6 @@ func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) { // CreateLicenseSecret creates the NGINX Plus JWT secret. func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) error { - GinkgoWriter.Printf("Creating NGINX Plus license secret in namespace %q from file %q\n", namespace, filename) - conf, err := os.ReadFile(filename) if err != nil { readFileErr := fmt.Errorf("error reading file %q: %w", filename, err) diff --git a/tests/suite/advanced_routing_test.go b/tests/suite/advanced_routing_test.go index 1163089cc5..22d6490dda 100644 --- a/tests/suite/advanced_routing_test.go +++ b/tests/suite/advanced_routing_test.go @@ -119,6 +119,8 @@ func expectRequestToRespondFromExpectedServer( GinkgoWriter.Printf("Expecting request to respond from the server %q\n", expServerName) status, body, err := framework.Get(appURL, address, timeoutConfig.RequestTimeout, headers, queryParams) if err != nil { + GinkgoWriter.Printf("ERROR making GET request to %q: %v\n", appURL, err) + return err } diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index 71d610bf5a..3285e8a450 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -237,6 +237,7 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo installCfg.NgfImageRepository = *ngfImageRepository installCfg.NginxImageRepository = *nginxImageRepository if *plusEnabled && cfg.nfr { + GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = *nginxPlusImageRepository } installCfg.ImageTag = *imageTag @@ -246,9 +247,11 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo chartVersion = "0.0.0-edge" installCfg.ChartVersion = chartVersion if *plusEnabled && cfg.nfr { + GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = fmt.Sprintf(formatNginxPlusEdgeImagePath, *gkeProject) } case *plusEnabled && cfg.nfr: + GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = fmt.Sprintf(formatNginxPlusEdgeImagePath, *gkeProject) } From 1daebfe4a15318ec52581000c00524a0c1ed6a48 Mon Sep 17 00:00:00 2001 From: Tina Usova Date: Wed, 3 Sep 2025 08:28:53 +0100 Subject: [PATCH 02/11] Remove extra comments --- tests/suite/system_suite_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index 3285e8a450..71d610bf5a 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -237,7 +237,6 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo installCfg.NgfImageRepository = *ngfImageRepository installCfg.NginxImageRepository = *nginxImageRepository if *plusEnabled && cfg.nfr { - GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = *nginxPlusImageRepository } installCfg.ImageTag = *imageTag @@ -247,11 +246,9 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo chartVersion = "0.0.0-edge" installCfg.ChartVersion = chartVersion if *plusEnabled && cfg.nfr { - GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = fmt.Sprintf(formatNginxPlusEdgeImagePath, *gkeProject) } case *plusEnabled && cfg.nfr: - GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = fmt.Sprintf(formatNginxPlusEdgeImagePath, *gkeProject) } From e4d3968ccfe100404e9cdc0685741d50aa3c856c Mon Sep 17 00:00:00 2001 From: Tina Usova Date: Thu, 4 Sep 2025 13:53:18 +0100 Subject: [PATCH 03/11] Remove boolean from parameters --- tests/suite/advanced_routing_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/suite/advanced_routing_test.go b/tests/suite/advanced_routing_test.go index 22d6490dda..1163089cc5 100644 --- a/tests/suite/advanced_routing_test.go +++ b/tests/suite/advanced_routing_test.go @@ -119,8 +119,6 @@ func expectRequestToRespondFromExpectedServer( GinkgoWriter.Printf("Expecting request to respond from the server %q\n", expServerName) status, body, err := framework.Get(appURL, address, timeoutConfig.RequestTimeout, headers, queryParams) if err != nil { - GinkgoWriter.Printf("ERROR making GET request to %q: %v\n", appURL, err) - return err } From 8ae86a7d2ef64d741ffee1a82d5c8e3ae36bbe36 Mon Sep 17 00:00:00 2001 From: Tina Usova Date: Tue, 2 Sep 2025 16:40:13 +0100 Subject: [PATCH 04/11] Add NFR tests logging --- tests/framework/crossplane.go | 53 ++- tests/framework/generate_manifests.go | 3 - tests/framework/info.go | 4 +- tests/framework/load.go | 9 +- tests/framework/logging.go | 9 + tests/framework/ngf.go | 22 +- tests/framework/prometheus.go | 149 ++++++- tests/framework/request.go | 11 +- tests/framework/resourcemanager.go | 596 ++++++++++++++------------ tests/framework/results.go | 135 +++++- tests/suite/advanced_routing_test.go | 7 +- tests/suite/client_settings_test.go | 7 +- tests/suite/dataplane_perf_test.go | 10 +- tests/suite/graceful_recovery_test.go | 84 +++- tests/suite/longevity_test.go | 8 +- tests/suite/nginxgateway_test.go | 15 +- tests/suite/reconfig_test.go | 72 +++- tests/suite/sample_test.go | 5 +- tests/suite/scale_test.go | 129 ++++-- tests/suite/snippets_filter_test.go | 15 +- tests/suite/system_suite_test.go | 17 +- tests/suite/tracing_test.go | 17 +- tests/suite/upgrade_test.go | 18 +- tests/suite/upstream_settings_test.go | 15 +- 24 files changed, 941 insertions(+), 469 deletions(-) diff --git a/tests/framework/crossplane.go b/tests/framework/crossplane.go index 02a16b6cb5..90b50faef4 100644 --- a/tests/framework/crossplane.go +++ b/tests/framework/crossplane.go @@ -8,6 +8,7 @@ import ( "strings" "time" + . "github.com/onsi/ginkgo/v2" core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -42,10 +43,13 @@ const crossplaneImageName = "nginx-crossplane:latest" // ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field, // and returns whether or not that field exists where it should. -func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) error { +func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField, opts ...Option) error { b, err := json.Marshal(conf) if err != nil { - return fmt.Errorf("error marshaling nginx config: %w", err) + marshalErr := fmt.Errorf("error marshaling nginx config: %w", err) + GinkgoWriter.Printf("%v\n", marshalErr) + + return marshalErr } for _, config := range conf.Config { @@ -55,7 +59,7 @@ func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) err for _, directive := range config.Parsed { if expFieldCfg.Server == "" && expFieldCfg.Upstream == "" { - if expFieldCfg.fieldFound(directive) { + if expFieldCfg.fieldFound(directive, opts...) { return nil } continue @@ -65,13 +69,15 @@ func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) err return nil } - if expFieldCfg.Upstream != "" && fieldExistsInUpstream(expFieldCfg, *directive) { + if expFieldCfg.Upstream != "" && fieldExistsInUpstream(expFieldCfg, *directive, opts...) { return nil } } } + directiveErr := fmt.Errorf("directive %s not found in: nginx config %s", expFieldCfg.Directive, string(b)) + GinkgoWriter.Printf("ERROR: %v\n", directiveErr) - return fmt.Errorf("directive %s not found in: nginx config %s", expFieldCfg.Directive, string(b)) + return directiveErr } func fieldExistsInServer( @@ -94,7 +100,16 @@ func fieldExistsInServer( func fieldExistsInUpstream( expFieldCfg ExpectedNginxField, directive Directive, + opts ...Option, ) bool { + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf( + "Checking upstream for directive %q with value %q\n", + expFieldCfg.Directive, + expFieldCfg.Value, + ) + } if directive.Directive == "upstream" && directive.Args[0] == expFieldCfg.Upstream { for _, directive := range directive.Block { if expFieldCfg.fieldFound(directive) { @@ -115,7 +130,8 @@ func getServerName(serverBlock Directives) string { return "" } -func (e ExpectedNginxField) fieldFound(directive *Directive) bool { +func (e ExpectedNginxField) fieldFound(directive *Directive, opts ...Option) bool { + options := LogOptions(opts...) arg := strings.Join(directive.Args, " ") valueMatch := arg == e.Value @@ -123,7 +139,20 @@ func (e ExpectedNginxField) fieldFound(directive *Directive) bool { valueMatch = strings.Contains(arg, e.Value) } - return directive.Directive == e.Directive && valueMatch + if directive.Directive == e.Directive && valueMatch { + if options.logEnabled { + GinkgoWriter.Printf( + "Found field %q with value %q in field %q with value %q\n", + e.Directive, + e.Value, + directive.Directive, + arg, + ) + } + return true + } + + return false } func fieldExistsInLocation(locationDirective *Directive, expFieldCfg ExpectedNginxField) bool { @@ -201,7 +230,10 @@ func injectCrossplaneContainer( podClient := k8sClient.CoreV1().Pods(namespace) if _, err := podClient.UpdateEphemeralContainers(ctx, ngfPodName, pod, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("error adding ephemeral container: %w", err) + containerErr := fmt.Errorf("error adding ephemeral container: %w", err) + GinkgoWriter.Printf("%v\n", containerErr) + + return containerErr } return nil @@ -231,7 +263,10 @@ func createCrossplaneExecutor( exec, err := remotecommand.NewSPDYExecutor(k8sConfig, http.MethodPost, req.URL()) if err != nil { - return nil, fmt.Errorf("error creating executor: %w", err) + executorErr := fmt.Errorf("error creating executor: %w", err) + GinkgoWriter.Printf("%v\n", executorErr) + + return nil, executorErr } return exec, nil diff --git a/tests/framework/generate_manifests.go b/tests/framework/generate_manifests.go index c7a00e234a..8e5d436fd6 100644 --- a/tests/framework/generate_manifests.go +++ b/tests/framework/generate_manifests.go @@ -7,7 +7,6 @@ import ( "io" "text/template" - . "github.com/onsi/ginkgo/v2" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" @@ -222,7 +221,6 @@ func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, err } func generateSecrets(secrets []string) ([]client.Object, error) { - GinkgoWriter.Printf("Generating secrets\n") objects := make([]client.Object, 0, len(secrets)) for _, secret := range secrets { @@ -239,7 +237,6 @@ func generateSecrets(secrets []string) ([]client.Object, error) { objects = append(objects, objs...) } - GinkgoWriter.Printf("Generated %d secrets\n", len(objects)) return objects, nil } diff --git a/tests/framework/info.go b/tests/framework/info.go index c485edc9aa..8f36929bc5 100644 --- a/tests/framework/info.go +++ b/tests/framework/info.go @@ -84,11 +84,11 @@ func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) { } // AddNginxLogsAndEventsToReport adds nginx logs and events from the namespace to the report if the spec failed. -func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string) { +func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string, opts ...Option) { if CurrentSpecReport().Failed() { var returnLogs string - nginxPodNames, _ := GetReadyNginxPodNames(rm.K8sClient, namespace, rm.TimeoutConfig.GetStatusTimeout) + nginxPodNames, _ := rm.GetReadyNginxPodNames(namespace, rm.TimeoutConfig.GetStatusTimeout, opts...) for _, nginxPodName := range nginxPodNames { returnLogs += fmt.Sprintf("Logs for Nginx Pod %s:\n", nginxPodName) diff --git a/tests/framework/load.go b/tests/framework/load.go index c687f275b1..d6caf312b5 100644 --- a/tests/framework/load.go +++ b/tests/framework/load.go @@ -7,6 +7,7 @@ import ( "net/http" "time" + . "github.com/onsi/ginkgo/v2" vegeta "github.com/tsenart/vegeta/v12/lib" ) @@ -49,6 +50,7 @@ type Metrics struct { // RunLoadTest uses Vegeta to send traffic to the provided Targets at the given rate for the given duration and writes // the results to the provided file. func RunLoadTest(cfg LoadTestConfig) (vegeta.Results, Metrics) { + GinkgoWriter.Printf("Running load test: %s\n", cfg.Description) vegTargets := convertTargetToVegetaTarget(cfg.Targets) targeter := vegeta.NewStaticTargeter(vegTargets...) @@ -61,7 +63,12 @@ func RunLoadTest(cfg LoadTestConfig) (vegeta.Results, Metrics) { Timeout: vegeta.DefaultTimeout, Transport: &http.Transport{ DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { - return dialer.DialContext(ctx, network, cfg.Proxy) + conn, err := dialer.DialContext(ctx, network, cfg.Proxy) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during dialing %q in %q network, error: %s\n", cfg.Proxy, network, err) + } + + return conn, err }, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, //nolint:gosec // self-signed cert for testing diff --git a/tests/framework/logging.go b/tests/framework/logging.go index aaf9f54a48..ddc02672a3 100644 --- a/tests/framework/logging.go +++ b/tests/framework/logging.go @@ -11,3 +11,12 @@ func WithLoggingDisabled() Option { opts.logEnabled = false } } + +func LogOptions(opts ...Option) *Options { + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + + return options +} diff --git a/tests/framework/ngf.go b/tests/framework/ngf.go index 26a7390775..27eddcd0de 100644 --- a/tests/framework/ngf.go +++ b/tests/framework/ngf.go @@ -14,7 +14,6 @@ import ( apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -106,7 +105,9 @@ func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) { } // CreateLicenseSecret creates the NGINX Plus JWT secret. -func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) error { +func CreateLicenseSecret(rm ResourceManager, namespace, filename string) error { + GinkgoWriter.Printf("Creating NGINX Plus license secret in namespace %q from file %q\n", namespace, filename) + conf, err := os.ReadFile(filename) if err != nil { readFileErr := fmt.Errorf("error reading file %q: %w", filename, err) @@ -124,11 +125,8 @@ func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) er }, } - if err := k8sClient.Create(ctx, ns); err != nil && !apierrors.IsAlreadyExists(err) { - createNSErr := fmt.Errorf("error creating namespace: %w", err) - GinkgoWriter.Printf("%v\n", createNSErr) - - return createNSErr + if err := rm.Create(ctx, ns); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("error creating namespace: %w", err) } secret := &core.Secret{ @@ -141,7 +139,7 @@ func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) er }, } - if err := k8sClient.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) { + if err := rm.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) { createSecretErr := fmt.Errorf("error creating secret: %w", err) GinkgoWriter.Printf("%v\n", createSecretErr) @@ -188,7 +186,7 @@ func UpgradeNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) { } // UninstallNGF uninstalls NGF. -func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, error) { +func UninstallNGF(cfg InstallationConfig, rm ResourceManager) ([]byte, error) { args := []string{ "uninstall", cfg.ReleaseName, "--namespace", cfg.Namespace, } @@ -202,20 +200,20 @@ func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, erro ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err = k8sClient.Delete(ctx, &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cfg.Namespace}}) + err = rm.Delete(ctx, &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cfg.Namespace}}, nil) if err != nil && !apierrors.IsNotFound(err) { return nil, err } var crList apiext.CustomResourceDefinitionList - if err := k8sClient.List(ctx, &crList); err != nil { + if err := rm.List(ctx, &crList); err != nil { return nil, err } for _, cr := range crList.Items { if strings.Contains(cr.Spec.Group, "gateway.nginx.org") { cr := cr - if err := k8sClient.Delete(ctx, &cr); err != nil && !apierrors.IsNotFound(err) { + if err := rm.Delete(ctx, &cr, nil); err != nil && !apierrors.IsNotFound(err) { return nil, err } } diff --git a/tests/framework/prometheus.go b/tests/framework/prometheus.go index 6ce8a300bb..d03ca4e44f 100644 --- a/tests/framework/prometheus.go +++ b/tests/framework/prometheus.go @@ -10,6 +10,7 @@ import ( "os/exec" "time" + . "github.com/onsi/ginkgo/v2" "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" @@ -49,7 +50,10 @@ func InstallPrometheus( "https://prometheus-community.github.io/helm-charts", ).CombinedOutput() if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to add Prometheus helm repo: %w; output: %s", err, string(output)) + prometheusErr := fmt.Errorf("failed to add Prometheus helm repo: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", prometheusErr) + + return PrometheusInstance{}, prometheusErr } output, err = exec.CommandContext( @@ -59,7 +63,10 @@ func InstallPrometheus( "update", ).CombinedOutput() if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to update helm repos: %w; output: %s", err, string(output)) + helmReposErr := fmt.Errorf("failed to update helm repos: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", helmReposErr) + + return PrometheusInstance{}, helmReposErr } scrapeInterval := fmt.Sprintf("%ds", int(cfg.ScrapeInterval.Seconds())) @@ -77,24 +84,36 @@ func InstallPrometheus( "--wait", ).CombinedOutput() if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to install Prometheus: %w; output: %s", err, string(output)) + prometheusInstallationErr := fmt.Errorf("failed to install Prometheus: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", prometheusInstallationErr) + + return PrometheusInstance{}, prometheusInstallationErr } pods, err := rm.GetPods(prometheusNamespace, client.MatchingLabels{ "app.kubernetes.io/name": "prometheus", }) if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to get Prometheus pods: %w", err) + podsErr := fmt.Errorf("failed to get Prometheus pods: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", podsErr) + + return PrometheusInstance{}, podsErr } if len(pods) != 1 { - return PrometheusInstance{}, fmt.Errorf("expected one Prometheus pod, found %d", len(pods)) + manyPodsErr := fmt.Errorf("expected one Prometheus pod, found %d", len(pods)) + GinkgoWriter.Printf("ERROR: %v\n", manyPodsErr) + + return PrometheusInstance{}, manyPodsErr } pod := pods[0] if pod.Status.PodIP == "" { - return PrometheusInstance{}, errors.New("the Prometheus pod has no IP") + podIPErr := errors.New("the Prometheus pod has no IP") + GinkgoWriter.Printf("ERROR: %v\n", podIPErr) + + return PrometheusInstance{}, podIPErr } var queryTimeout time.Duration @@ -114,6 +133,7 @@ func InstallPrometheus( // UninstallPrometheus uninstalls Prometheus from the cluster. func UninstallPrometheus(rm ResourceManager) error { + GinkgoWriter.Printf("Uninstalling Prometheus from namespace %q\n", prometheusNamespace) output, err := exec.CommandContext( context.Background(), "helm", @@ -122,11 +142,17 @@ func UninstallPrometheus(rm ResourceManager) error { "-n", prometheusNamespace, ).CombinedOutput() if err != nil { - return fmt.Errorf("failed to uninstall Prometheus: %w; output: %s", err, string(output)) + uninstallErr := fmt.Errorf("failed to uninstall Prometheus: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", uninstallErr) + + return uninstallErr } if err := rm.DeleteNamespace(prometheusNamespace); err != nil { - return fmt.Errorf("failed to delete Prometheus namespace: %w", err) + deleteNSErr := fmt.Errorf("failed to delete Prometheus namespace: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", deleteNSErr) + + return deleteNSErr } return nil @@ -150,8 +176,12 @@ type PrometheusInstance struct { // PortForward starts port forwarding to the Prometheus instance. func (ins *PrometheusInstance) PortForward(config *rest.Config, stopCh <-chan struct{}) error { + GinkgoWriter.Printf("Starting port forwarding to Prometheus pod %q in namespace %q\n", ins.podName, ins.podNamespace) if ins.portForward { - panic("port forwarding already started") + infoMsg := "port forwarding already started" + GinkgoWriter.Printf("INFO: %s\n", infoMsg) + + panic(infoMsg) } ins.portForward = true @@ -161,6 +191,7 @@ func (ins *PrometheusInstance) PortForward(config *rest.Config, stopCh <-chan st } func (ins *PrometheusInstance) getAPIClient() (v1.API, error) { + GinkgoWriter.Printf("Creating Prometheus API client for pod %q in namespace %q\n", ins.podName, ins.podNamespace) var endpoint string if ins.portForward { endpoint = fmt.Sprintf("http://localhost:%d", PrometheusPortForwardPort) @@ -175,6 +206,8 @@ func (ins *PrometheusInstance) getAPIClient() (v1.API, error) { c, err := api.NewClient(cfg) if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating Prometheus API client: %v\n", err) + return nil, err } @@ -185,7 +218,10 @@ func (ins *PrometheusInstance) ensureAPIClient() error { if ins.apiClient == nil { ac, err := ins.getAPIClient() if err != nil { - return fmt.Errorf("failed to get Prometheus API client: %w", err) + apiClientErr := fmt.Errorf("failed to get Prometheus API client: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", apiClientErr) + + return apiClientErr } ins.apiClient = ac } @@ -195,6 +231,7 @@ func (ins *PrometheusInstance) ensureAPIClient() error { // Query sends a query to Prometheus. func (ins *PrometheusInstance) Query(query string) (model.Value, error) { + GinkgoWriter.Printf("Querying Prometheus with query: %q\n", query) ctx, cancel := context.WithTimeout(context.Background(), ins.queryTimeout) defer cancel() @@ -209,10 +246,14 @@ func (ins *PrometheusInstance) QueryWithCtx(ctx context.Context, query string) ( result, warnings, err := ins.apiClient.Query(ctx, query, time.Time{}) if err != nil { - return nil, fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", queryErr) + + return nil, queryErr } if len(warnings) > 0 { + GinkgoWriter.Printf("WARNING: Prometheus query returned warnings: %v\n", warnings) slog.InfoContext(context.Background(), "Prometheus query returned warnings", "query", query, @@ -235,16 +276,23 @@ func (ins *PrometheusInstance) QueryRange(query string, promRange v1.Range) (mod func (ins *PrometheusInstance) QueryRangeWithCtx(ctx context.Context, query string, promRange v1.Range, ) (model.Value, error) { + GinkgoWriter.Printf("Querying Prometheus with range query: %q\n", query) if err := ins.ensureAPIClient(); err != nil { + GinkgoWriter.Printf("ERROR during ensureAPIClient for prometheus: %v\n", err) + return nil, err } result, warnings, err := ins.apiClient.QueryRange(ctx, query, promRange) if err != nil { - return nil, fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", queryErr) + + return nil, queryErr } if len(warnings) > 0 { + GinkgoWriter.Printf("WARNING: Prometheus range query returned warnings: %v\n", warnings) slog.InfoContext(context.Background(), "Prometheus range query returned warnings", "query", query, @@ -260,11 +308,17 @@ func (ins *PrometheusInstance) QueryRangeWithCtx(ctx context.Context, func GetFirstValueOfPrometheusVector(val model.Value) (float64, error) { res, ok := val.(model.Vector) if !ok { - return 0, fmt.Errorf("expected a vector, got %T", val) + valueErr := fmt.Errorf("expected a vector, got %T", val) + GinkgoWriter.Printf("ERROR: %v\n", valueErr) + + return 0, valueErr } if len(res) == 0 { - return 0, errors.New("empty vector") + vectorErr := errors.New("empty vector") + GinkgoWriter.Printf("ERROR: %v\n", vectorErr) + + return 0, vectorErr } return float64(res[0].Value), nil @@ -272,8 +326,11 @@ func GetFirstValueOfPrometheusVector(val model.Value) (float64, error) { // WritePrometheusMatrixToCSVFile writes a Prometheus matrix to a CSV file. func WritePrometheusMatrixToCSVFile(fileName string, value model.Value) error { + GinkgoWriter.Printf("Writing Prometheus matrix to CSV file %q\n", fileName) file, err := os.Create(fileName) if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating file %q: %v\n", fileName, err) + return err } defer file.Close() @@ -282,13 +339,18 @@ func WritePrometheusMatrixToCSVFile(fileName string, value model.Value) error { matrix, ok := value.(model.Matrix) if !ok { - return fmt.Errorf("expected a matrix, got %T", value) + matrixErr := fmt.Errorf("expected a matrix, got %T", value) + GinkgoWriter.Printf("ERROR: %v\n", matrixErr) + + return matrixErr } for _, sample := range matrix { for _, pair := range sample.Values { record := []string{fmt.Sprint(pair.Timestamp.Unix()), pair.Value.String()} if err := csvWriter.Write(record); err != nil { + GinkgoWriter.Printf("ERROR: %v\n", err) + return err } } @@ -409,18 +471,30 @@ func CreateMetricExistChecker( query string, getTime func() time.Time, modifyTime func(), + opts ...Option, ) func() error { return func() error { queryWithTimestamp := fmt.Sprintf("%s @ %d", query, getTime().Unix()) + options := LogOptions(opts...) result, err := promInstance.Query(queryWithTimestamp) if err != nil { - return fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating metric existence checker: %v\n", queryErr) + } + + return queryErr } if result.String() == "" { modifyTime() - return errors.New("empty result") + emptyResultErr := errors.New("empty result") + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating metric existence checker: %v\n", emptyResultErr) + } + + return emptyResultErr } return nil @@ -436,6 +510,7 @@ func CreateEndTimeFinder( endTime *time.Time, queryRangeStep time.Duration, ) func() error { + GinkgoWriter.Printf("Creating end time finder with start time %v and initial end time %v\n", startTime, endTime) return func() error { result, err := promInstance.QueryRange(query, v1.Range{ Start: startTime, @@ -443,12 +518,18 @@ func CreateEndTimeFinder( Step: queryRangeStep, }) if err != nil { - return fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + GinkgoWriter.Printf("ERROR during creating end time finder: %v\n", queryErr) + + return queryErr } if result.String() == "" { *endTime = time.Now() - return errors.New("empty result") + emptyResultsErr := errors.New("empty result") + GinkgoWriter.Printf("ERROR during creating end time finder: %v\n", emptyResultsErr) + + return emptyResultsErr } return nil @@ -457,14 +538,29 @@ func CreateEndTimeFinder( // CreateResponseChecker returns a function that checks if there is a successful response from a url. func CreateResponseChecker(url, address string, requestTimeout time.Duration, opts ...Option) func() error { + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Starting checking response for url %q and address %q\n", url, address) + } + return func() error { status, _, err := Get(url, address, requestTimeout, nil, nil, opts...) if err != nil { - return fmt.Errorf("bad response: %w", err) + badReqErr := fmt.Errorf("bad response: %w", err) + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating response checker: %v\n", badReqErr) + } + + return badReqErr } if status != 200 { - return fmt.Errorf("unexpected status code: %d", status) + statusErr := fmt.Errorf("unexpected status code: %d", status) + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating response checker: %v\n", statusErr) + } + + return statusErr } return nil @@ -474,11 +570,15 @@ func CreateResponseChecker(url, address string, requestTimeout time.Duration, op func getFirstValueOfVector(query string, promInstance PrometheusInstance) (float64, error) { result, err := promInstance.Query(query) if err != nil { + GinkgoWriter.Printf("ERROR querying Prometheus during getting first value of vector: %v\n", err) + return 0, err } val, err := GetFirstValueOfPrometheusVector(result) if err != nil { + GinkgoWriter.Printf("ERROR getting first value of Prometheus vector: %v\n", err) + return 0, err } @@ -488,12 +588,17 @@ func getFirstValueOfVector(query string, promInstance PrometheusInstance) (float func getBuckets(query string, promInstance PrometheusInstance) ([]Bucket, error) { result, err := promInstance.Query(query) if err != nil { + GinkgoWriter.Printf("ERROR querying Prometheus during getting buckets: %v\n", err) + return nil, err } res, ok := result.(model.Vector) if !ok { - return nil, errors.New("could not convert result to vector") + convertationErr := errors.New("could not convert result to vector") + GinkgoWriter.Printf("ERROR during getting buckets: %v\n", convertationErr) + + return nil, convertationErr } buckets := make([]Bucket, 0, len(res)) diff --git a/tests/framework/request.go b/tests/framework/request.go index a04d8ccc66..bd8663b8a3 100644 --- a/tests/framework/request.go +++ b/tests/framework/request.go @@ -24,10 +24,7 @@ func Get( headers, queryParams map[string]string, opts ...Option, ) (int, string, error) { - options := &Options{logEnabled: true} - for _, opt := range opts { - opt(options) - } + options := LogOptions(opts...) resp, err := makeRequest(http.MethodGet, url, address, nil, timeout, headers, queryParams, opts...) if err != nil { @@ -99,11 +96,7 @@ func makeRequest( ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - options := &Options{logEnabled: true} - - for _, opt := range opts { - opt(options) - } + options := LogOptions(opts...) if options.logEnabled { requestDetails := fmt.Sprintf( "Method: %s, URL: %s, Address: %s, Headers: %v, QueryParams: %v\n", diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go index f51f17920a..d795877f11 100644 --- a/tests/framework/resourcemanager.go +++ b/tests/framework/resourcemanager.go @@ -76,8 +76,11 @@ type ClusterInfo struct { } // Apply creates or updates Kubernetes resources defined as Go objects. -func (rm *ResourceManager) Apply(resources []client.Object) error { - GinkgoWriter.Printf("Applying resources defined as Go objects\n") +func (rm *ResourceManager) Apply(resources []client.Object, opts ...Option) error { + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Applying resources defined as Go objects\n") + } ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) defer cancel() @@ -101,25 +104,13 @@ func (rm *ResourceManager) Apply(resources []client.Object) error { } } - if err := rm.K8sClient.Get(ctx, client.ObjectKeyFromObject(resource), obj); err != nil { + if err := rm.Get(ctx, client.ObjectKeyFromObject(resource), obj, opts...); err != nil { if !apierrors.IsNotFound(err) { - notFoundErr := fmt.Errorf("error getting resource: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during getting Kubernetes resources: %s\n", - notFoundErr, - ) - - return notFoundErr + return err } - if err := rm.K8sClient.Create(ctx, resource); err != nil { - creatingResourceErr := fmt.Errorf("error creating resource: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during applying creates Kubernetes resources: %s\n", - creatingResourceErr, - ) - - return creatingResourceErr + if err := rm.Create(ctx, resource); err != nil { + return fmt.Errorf("error creating resource: %w", err) } continue @@ -128,83 +119,68 @@ func (rm *ResourceManager) Apply(resources []client.Object) error { // Some tests modify resources that are also modified by NGF (to update their status), so conflicts are possible // For example, a Gateway resource. err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - if err := rm.K8sClient.Get(ctx, client.ObjectKeyFromObject(resource), obj); err != nil { - GinkgoWriter.Printf( - "ERROR occurred during getting Kubernetes resources on retries: %s\n", - err, - ) - + if err := rm.Get(ctx, client.ObjectKeyFromObject(resource), obj); err != nil { return err } resource.SetResourceVersion(obj.GetResourceVersion()) - updateErr := rm.K8sClient.Update(ctx, resource) - if updateErr != nil { - GinkgoWriter.Printf( - "ERROR occurred during updating Kubernetes resources on retries: %s\n", - updateErr, - ) - } - return updateErr + return rm.Update(ctx, resource, nil) }) if err != nil { retryErr := fmt.Errorf("error updating resource: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during retries: %s\n", - retryErr, - ) + GinkgoWriter.Printf("%s\n", retryErr) return retryErr } } - GinkgoWriter.Printf("Resources defined as Go objects applied successfully\n") - + if options.logEnabled { + GinkgoWriter.Printf("Resources defined as Go objects applied successfully\n") + } return nil } // ApplyFromFiles creates or updates Kubernetes resources defined within the provided YAML files. -func (rm *ResourceManager) ApplyFromFiles(files []string, namespace string) error { +func (rm *ResourceManager) ApplyFromFiles(files []string, namespace string, opts ...Option) error { + options := LogOptions(opts...) for _, file := range files { - GinkgoWriter.Printf("Applying resources from file: %q to namespace %q\n", file, namespace) + if options.logEnabled { + GinkgoWriter.Printf("\nApplying resources from file: %q to namespace %q\n", file, namespace) + } data, err := rm.GetFileContents(file) if err != nil { - GinkgoWriter.Printf("ERROR occurred during getting file contents for file %q, error: %s\n", file, err) - return err } if err = rm.ApplyFromBuffer(data, namespace); err != nil { - GinkgoWriter.Printf("ERROR occurred during applying resources from file %q, error: %s\n", file, err) - return err } } - GinkgoWriter.Printf("Resources from files applied successfully to namespace %q,\n", namespace) + if options.logEnabled { + GinkgoWriter.Printf("Resources from files applied successfully to namespace %q,\n", namespace) + } return nil } -func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace string) error { +func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace string, opts ...Option) error { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) defer cancel() + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Applying resources from buffer to namespace %q\n", namespace) + } handlerFunc := func(obj unstructured.Unstructured) error { obj.SetNamespace(namespace) nsName := types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()} fetchedObj := obj.DeepCopy() - if err := rm.K8sClient.Get(ctx, nsName, fetchedObj); err != nil { + if err := rm.Get(ctx, nsName, fetchedObj, opts...); err != nil { if !apierrors.IsNotFound(err) { - getResourceErr := fmt.Errorf("error getting resource: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting resource from buffer, error: %s\n", getResourceErr) - - return getResourceErr + return err } - if err := rm.K8sClient.Create(ctx, &obj); err != nil { - createResourceErr := fmt.Errorf("error creating resource: %w", err) - GinkgoWriter.Printf("ERROR occurred during creating resource from buffer, error: %s\n", createResourceErr) - - return createResourceErr + if err := rm.Create(ctx, &obj); err != nil { + return fmt.Errorf("error creating resource: %w", err) } return nil @@ -213,26 +189,16 @@ func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace strin // Some tests modify resources that are also modified by NGF (to update their status), so conflicts are possible // For example, a Gateway resource. err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - if err := rm.K8sClient.Get(ctx, nsName, fetchedObj); err != nil { - GinkgoWriter.Printf( - "ERROR occurred during getting resource from buffer on retries, error: %s\n", - err, - ) - + if err := rm.Get(ctx, nsName, fetchedObj); err != nil { return err } obj.SetResourceVersion(fetchedObj.GetResourceVersion()) - updateErr := rm.K8sClient.Update(ctx, &obj) - if updateErr != nil { - GinkgoWriter.Printf("ERROR occurred during updating resource from buffer, error: %s\n", updateErr) - } - return updateErr + return rm.Update(ctx, &obj, nil) }) if err != nil { retryErr := fmt.Errorf("error updating resource: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during retries, while update from buffer error: %s\n", + GinkgoWriter.Printf("%s\n", retryErr, ) @@ -246,17 +212,14 @@ func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace strin } // Delete deletes Kubernetes resources defined as Go objects. -func (rm *ResourceManager) Delete(resources []client.Object, opts ...client.DeleteOption) error { +func (rm *ResourceManager) DeleteResources(resources []client.Object, opts ...client.DeleteOption) error { GinkgoWriter.Printf("Deleting resources\n") ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteTimeout) defer cancel() for _, resource := range resources { - if err := rm.K8sClient.Delete(ctx, resource, opts...); err != nil && !apierrors.IsNotFound(err) { - delErr := fmt.Errorf("error deleting resource: %w", err) - GinkgoWriter.Printf("ERROR occurred during deleting resource, error: %s\n", delErr) - - return delErr + if err := rm.Delete(ctx, resource, opts); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting resource: %w", err) } } GinkgoWriter.Printf("Resources deleted successfully\n") @@ -264,29 +227,22 @@ func (rm *ResourceManager) Delete(resources []client.Object, opts ...client.Dele return nil } -func (rm *ResourceManager) DeleteNamespace(name string) error { +func (rm *ResourceManager) DeleteNamespace(name string, opts ...Option) error { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteNamespaceTimeout) GinkgoWriter.Printf("Deleting namespace %q\n", name) defer cancel() ns := &core.Namespace{} - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Name: name}, ns); err != nil { + if err := rm.Get(ctx, types.NamespacedName{Name: name}, ns, opts...); err != nil { if apierrors.IsNotFound(err) { - GinkgoWriter.Printf("Namespace %q not found, nothing to delete\n", name) - return nil } - getNsErr := fmt.Errorf("error getting namespace: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting namespace, error: %s\n", getNsErr) - return getNsErr + return fmt.Errorf("error getting namespace: %w", err) } - if err := rm.K8sClient.Delete(ctx, ns); err != nil { - delErr := fmt.Errorf("error deleting namespace: %w", err) - GinkgoWriter.Printf("ERROR occurred during deleting namespace, error: %s\n", delErr) - - return delErr + if err := rm.Delete(ctx, ns, nil, opts...); err != nil { + return fmt.Errorf("error deleting namespace: %w", err) } GinkgoWriter.Printf("Waiting for namespace %q to be deleted\n", name) @@ -296,23 +252,19 @@ func (rm *ResourceManager) DeleteNamespace(name string) error { 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Name: name}, ns); err != nil { + if err := rm.Get(ctx, types.NamespacedName{Name: name}, ns, opts...); err != nil { if apierrors.IsNotFound(err) { - GinkgoWriter.Printf("Namespace %q not found (deleted)\n", name) - return true, nil } - getNsErr := fmt.Errorf("error getting namespace: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting namespace, error: %s\n", getNsErr) - return false, getNsErr + return false, fmt.Errorf("error getting namespace: %w", err) } return false, nil }) } -func (rm *ResourceManager) DeleteNamespaces(names []string) error { +func (rm *ResourceManager) DeleteNamespaces(names []string, opts ...Option) error { GinkgoWriter.Printf("Deleting namespaces: %v\n", names) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteNamespaceTimeout*2) defer cancel() @@ -321,15 +273,12 @@ func (rm *ResourceManager) DeleteNamespaces(names []string) error { for _, name := range names { ns := &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}} - if err := rm.K8sClient.Delete(ctx, ns); err != nil { + if err := rm.Delete(ctx, ns, nil, opts...); err != nil { if apierrors.IsNotFound(err) { - GinkgoWriter.Printf("Namespace %q not found, nothing to delete\n", name) continue } - delNsErr := fmt.Errorf("error deleting namespace: %w", err) - GinkgoWriter.Printf("ERROR occurred during deleting namespace %q, error: %s\n", name, delNsErr) - combinedErrors = errors.Join(combinedErrors, delNsErr) + combinedErrors = errors.Join(combinedErrors, fmt.Errorf("error deleting namespace: %w", err)) } } @@ -339,7 +288,7 @@ func (rm *ResourceManager) DeleteNamespaces(names []string) error { true, /* poll immediately */ func(ctx context.Context) (bool, error) { nsList := &core.NamespaceList{} - if err := rm.K8sClient.List(ctx, nsList); err != nil { + if err := rm.List(ctx, nsList, nil); err != nil { return false, nil //nolint:nilerr // retry on error } @@ -363,9 +312,7 @@ func (rm *ResourceManager) DeleteFromFiles(files []string, namespace string) err ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteTimeout) defer cancel() - if err := rm.K8sClient.Delete(ctx, &obj); err != nil && !apierrors.IsNotFound(err) { - GinkgoWriter.Printf("ERROR occurred during deleting resource from file, error: %s\n", err) - + if err := rm.Delete(ctx, &obj, nil); err != nil && !apierrors.IsNotFound(err) { return err } @@ -375,8 +322,6 @@ func (rm *ResourceManager) DeleteFromFiles(files []string, namespace string) err for _, file := range files { data, err := rm.GetFileContents(file) if err != nil { - GinkgoWriter.Printf("ERROR occurred during getting file contents for file %q, error: %s\n", file, err) - return err } @@ -419,34 +364,49 @@ func (rm *ResourceManager) readAndHandleObject( // path or an https:// URL to YAML manifests and provides the contents. func (rm *ResourceManager) GetFileContents(file string) (*bytes.Buffer, error) { if strings.HasPrefix(file, "http://") { - return nil, fmt.Errorf("data can't be retrieved from %s: http is not supported, use https", file) + err := fmt.Errorf("data can't be retrieved from %s: http is not supported, use https", file) + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + + return nil, err } else if strings.HasPrefix(file, "https://") { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.ManifestFetchTimeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, file, nil) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + return nil, err } resp, err := http.DefaultClient.Do(req) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%d response when getting %s file contents", resp.StatusCode, file) + err = fmt.Errorf("%d response when getting %s file contents", resp.StatusCode, file) + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + + return nil, err } manifests := new(bytes.Buffer) count, err := manifests.ReadFrom(resp.Body) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + return nil, err } if resp.ContentLength != -1 && count != resp.ContentLength { - return nil, fmt.Errorf("received %d bytes from %s, expected %d", count, file, resp.ContentLength) + err = fmt.Errorf("received %d bytes from %s, expected %d", count, file, resp.ContentLength) + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + + return nil, err } return manifests, nil } @@ -457,6 +417,8 @@ func (rm *ResourceManager) GetFileContents(file string) (*bytes.Buffer, error) { b, err := rm.FS.ReadFile(file) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting file contents for file %q, error: %s\n", file, err) + return nil, err } @@ -465,18 +427,18 @@ func (rm *ResourceManager) GetFileContents(file string) (*bytes.Buffer, error) { // WaitForAppsToBeReady waits for all apps in the specified namespace to be ready, // or until the ctx timeout is reached. -func (rm *ResourceManager) WaitForAppsToBeReady(namespace string) error { +func (rm *ResourceManager) WaitForAppsToBeReady(namespace string, opts ...Option) error { GinkgoWriter.Printf("Waiting for apps to be ready in namespace %q\n", namespace) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) defer cancel() - return rm.WaitForAppsToBeReadyWithCtx(ctx, namespace) + return rm.WaitForAppsToBeReadyWithCtx(ctx, namespace, opts...) } // WaitForAppsToBeReadyWithCtx waits for all apps in the specified namespace to be ready or // until the provided context is canceled. -func (rm *ResourceManager) WaitForAppsToBeReadyWithCtx(ctx context.Context, namespace string) error { - if err := rm.WaitForPodsToBeReady(ctx, namespace); err != nil { +func (rm *ResourceManager) WaitForAppsToBeReadyWithCtx(ctx context.Context, namespace string, opts ...Option) error { + if err := rm.WaitForPodsToBeReady(ctx, namespace, opts...); err != nil { GinkgoWriter.Printf("ERROR occurred during waiting for pods to be ready, error: %s\n", err) return err @@ -504,14 +466,23 @@ func (rm *ResourceManager) WaitForAppsToBeReadyWithCtx(ctx context.Context, name // WaitForPodsToBeReady waits for all Pods in the specified namespace to be ready or // until the provided context is canceled. -func (rm *ResourceManager) WaitForPodsToBeReady(ctx context.Context, namespace string) error { - return wait.PollUntilContextCancel( +func (rm *ResourceManager) WaitForPodsToBeReady( + ctx context.Context, + namespace string, + opts ...Option, +) error { + options := LogOptions(opts...) + waitingErr := wait.PollUntilContextCancel( ctx, 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { var podList core.PodList - if err := rm.K8sClient.List(ctx, &podList, client.InNamespace(namespace)); err != nil { + if err := rm.List( + ctx, + &podList, + client.InNamespace(namespace), + ); err != nil { return false, err } @@ -523,10 +494,22 @@ func (rm *ResourceManager) WaitForPodsToBeReady(ctx context.Context, namespace s } } } + if options.logEnabled { + GinkgoWriter.Printf("Pods ready: %d out of %d in namespace %q\n", podsReady, len(podList.Items), namespace) + } return podsReady == len(podList.Items), nil }, ) + if waitingErr != nil { + GinkgoWriter.Printf( + "ERROR occurred during waiting for Pods to be ready in namespace %q, error: %s\n", + namespace, + waitingErr, + ) + } + + return waitingErr } func (rm *ResourceManager) waitForGatewaysToBeReady(ctx context.Context, namespace string) error { @@ -536,7 +519,11 @@ func (rm *ResourceManager) waitForGatewaysToBeReady(ctx context.Context, namespa true, /* poll immediately */ func(ctx context.Context) (bool, error) { var gatewayList v1.GatewayList - if err := rm.K8sClient.List(ctx, &gatewayList, client.InNamespace(namespace)); err != nil { + if err := rm.List( + ctx, + &gatewayList, + client.InNamespace(namespace), + ); err != nil { return false, err } @@ -554,13 +541,18 @@ func (rm *ResourceManager) waitForGatewaysToBeReady(ctx context.Context, namespa } func (rm *ResourceManager) waitForHTTPRoutesToBeReady(ctx context.Context, namespace string) error { + GinkgoWriter.Printf("Waiting for HTTPRoutes to be ready in namespace %q\n", namespace) return wait.PollUntilContextCancel( ctx, 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { var routeList v1.HTTPRouteList - if err := rm.K8sClient.List(ctx, &routeList, client.InNamespace(namespace)); err != nil { + if err := rm.List( + ctx, + &routeList, + client.InNamespace(namespace), + ); err != nil { return false, err } @@ -576,10 +568,13 @@ func (rm *ResourceManager) waitForHTTPRoutesToBeReady(ctx context.Context, names } func (rm *ResourceManager) waitForGRPCRoutesToBeReady(ctx context.Context, namespace string) error { + GinkgoWriter.Printf("Waiting for GRPCRoutes to be ready in namespace %q\n", namespace) // First, check if grpcroute even exists for v1. If not, ignore. var routeList v1.GRPCRouteList - err := rm.K8sClient.List(ctx, &routeList, client.InNamespace(namespace)) + err := rm.List(ctx, &routeList, client.InNamespace(namespace)) if err != nil && strings.Contains(err.Error(), "no matches for kind") { + GinkgoWriter.Printf("No GRPCRoute resources found in namespace %q, skipping wait\n", namespace) + return nil } @@ -589,7 +584,11 @@ func (rm *ResourceManager) waitForGRPCRoutesToBeReady(ctx context.Context, names true, /* poll immediately */ func(ctx context.Context) (bool, error) { var routeList v1.GRPCRouteList - if err := rm.K8sClient.List(ctx, &routeList, client.InNamespace(namespace)); err != nil { + if err := rm.List( + ctx, + &routeList, + client.InNamespace(namespace), + ); err != nil { return false, err } @@ -612,10 +611,10 @@ func (rm *ResourceManager) GetLBIPAddress(namespace string) (string, error) { var serviceList core.ServiceList var address string - if err := rm.K8sClient.List(ctx, &serviceList, client.InNamespace(namespace)); err != nil { - GinkgoWriter.Printf("ERROR occurred during getting list of services in namespace %q, error: %s\n", - namespace, err) - + if err := rm.List( + ctx, &serviceList, + client.InNamespace(namespace), + ); err != nil { return "", err } var nsName types.NamespacedName @@ -624,14 +623,7 @@ func (rm *ResourceManager) GetLBIPAddress(namespace string) (string, error) { if svc.Spec.Type == core.ServiceTypeLoadBalancer { nsName = types.NamespacedName{Namespace: svc.GetNamespace(), Name: svc.GetName()} if err := rm.waitForLBStatusToBeReady(ctx, nsName); err != nil { - lbStatusErr := fmt.Errorf("error getting status from LoadBalancer service: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during waiting for LoadBalancer service in namespace %q to be ready, error: %s\n", - nsName, - err, - ) - - return "", lbStatusErr + return "", fmt.Errorf("error getting status from LoadBalancer service: %w", err) } } } @@ -639,14 +631,8 @@ func (rm *ResourceManager) GetLBIPAddress(namespace string) (string, error) { if nsName.Name != "" { var lbService core.Service - if err := rm.K8sClient.Get(ctx, nsName, &lbService); err != nil { - getLBStatusErr := fmt.Errorf("error getting LoadBalancer service: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting LoadBalancer service in namespace %q, error: %s\n", - nsName, - err, - ) - - return "", getLBStatusErr + if err := rm.Get(ctx, nsName, &lbService); err != nil { + return "", fmt.Errorf("error getting LoadBalancer service: %w", err) } if lbService.Status.LoadBalancer.Ingress[0].IP != "" { address = lbService.Status.LoadBalancer.Ingress[0].IP @@ -665,7 +651,7 @@ func (rm *ResourceManager) waitForLBStatusToBeReady(ctx context.Context, svcNsNa true, /* poll immediately */ func(ctx context.Context) (bool, error) { var svc core.Service - if err := rm.K8sClient.Get(ctx, svcNsName, &svc); err != nil { + if err := rm.Get(ctx, svcNsName, &svc); err != nil { return false, err } if len(svc.Status.LoadBalancer.Ingress) > 0 { @@ -679,19 +665,14 @@ func (rm *ResourceManager) waitForLBStatusToBeReady(ctx context.Context, svcNsNa // GetClusterInfo retrieves node info and Kubernetes version from the cluster. func (rm *ResourceManager) GetClusterInfo() (ClusterInfo, error) { - GinkgoWriter.Printf("Getting cluster info\n") + GinkgoWriter.Printf("Getting cluster info|nodes\n") ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) defer cancel() var nodes core.NodeList ci := &ClusterInfo{} - if err := rm.K8sClient.List(ctx, &nodes); err != nil { - getNodesErr := fmt.Errorf("error getting nodes: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting nodes in cluster, error: %s\n", - getNodesErr, - ) - - return *ci, getNodesErr + if err := rm.List(ctx, &nodes); err != nil { + return *ci, fmt.Errorf("error getting nodes: %w", err) } ci.NodeCount = len(nodes.Items) @@ -712,13 +693,8 @@ func (rm *ResourceManager) GetClusterInfo() (ClusterInfo, error) { var ns core.Namespace key := types.NamespacedName{Name: "kube-system"} - if err := rm.K8sClient.Get(ctx, key, &ns); err != nil { - getK8sNamespaceErr := fmt.Errorf("error getting kube-system namespace: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting kube-system namespace, error: %s\n", - getK8sNamespaceErr, - ) - - return *ci, getK8sNamespaceErr + if err := rm.Get(ctx, key, &ns); err != nil { + return *ci, fmt.Errorf("error getting kube-system namespace: %w", err) } ci.ID = string(ns.UID) @@ -733,19 +709,13 @@ func (rm *ResourceManager) GetPodNames(namespace string, labels client.MatchingL defer cancel() var podList core.PodList - if err := rm.K8sClient.List( + if err := rm.List( ctx, &podList, client.InNamespace(namespace), labels, ); err != nil { - getPodsErr := fmt.Errorf("error getting list of Pods: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting list of Pods in namespace %q, error: %s\n", - namespace, - getPodsErr, - ) - - return nil, getPodsErr + return nil, fmt.Errorf("error getting list of Pods: %w", err) } names := make([]string, 0, len(podList.Items)) @@ -753,7 +723,7 @@ func (rm *ResourceManager) GetPodNames(namespace string, labels client.MatchingL for _, pod := range podList.Items { names = append(names, pod.Name) } - GinkgoWriter.Printf("Found pod names in namespace %q: %v\n", namespace, names) + GinkgoWriter.Printf("Found pod name in namespace %q: %v\n", namespace, names) return names, nil } @@ -765,19 +735,13 @@ func (rm *ResourceManager) GetPods(namespace string, labels client.MatchingLabel defer cancel() var podList core.PodList - if err := rm.K8sClient.List( + if err := rm.List( ctx, &podList, client.InNamespace(namespace), labels, ); err != nil { - getPodsErr := fmt.Errorf("error getting list of Pods: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting list of Pods in namespace %q, error: %s\n", - namespace, - getPodsErr, - ) - - return nil, getPodsErr + return nil, fmt.Errorf("error getting list of Pods: %w", err) } GinkgoWriter.Printf("Found %d pods in namespace %q\n", len(podList.Items), namespace) @@ -791,15 +755,8 @@ func (rm *ResourceManager) GetPod(namespace, name string) (*core.Pod, error) { defer cancel() var pod core.Pod - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &pod); err != nil { - getPodErr := fmt.Errorf("error getting Pod: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting Pod %q in namespace %q, error: %s\n", - name, - namespace, - getPodErr, - ) - - return nil, getPodErr + if err := rm.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &pod); err != nil { + return nil, fmt.Errorf("error getting Pod: %w", err) } GinkgoWriter.Printf("Found pod %q in namespace %q\n", name, namespace) @@ -849,7 +806,7 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app var deployments apps.DeploymentList - if err := rm.K8sClient.List( + if err := rm.List( ctx, &deployments, client.InNamespace(namespace), @@ -857,14 +814,7 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app "app.kubernetes.io/instance": releaseName, }, ); err != nil { - getDeploymentsErr := fmt.Errorf("error getting list of Deployments: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting list of Deployments in namespace %q with release %q, error: %s\n", - namespace, - releaseName, - getDeploymentsErr, - ) - - return nil, getDeploymentsErr + return nil, fmt.Errorf("error getting list of Deployments: %w", err) } if len(deployments.Items) != 1 { @@ -899,13 +849,7 @@ func (rm *ResourceManager) getGatewayClassNginxProxy( var proxy ngfAPIv1alpha2.NginxProxy proxyName := releaseName + "-proxy-config" - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: proxyName}, &proxy); err != nil { - GinkgoWriter.Printf("ERROR occurred during getting NginxProxy %q in namespace %q, error: %s\n", - proxyName, - namespace, - err, - ) - + if err := rm.Get(ctx, types.NamespacedName{Namespace: namespace, Name: proxyName}, &proxy); err != nil { return nil, err } GinkgoWriter.Printf("Successfully found NginxProxy %q in namespace %q\n", proxyName, namespace) @@ -939,15 +883,8 @@ func (rm *ResourceManager) ScaleNginxDeployment(namespace, releaseName string, r proxy.Spec.Kubernetes.Deployment.Replicas = &replicas - if err = rm.K8sClient.Update(ctx, proxy); err != nil { - updateNginxProxyErr := fmt.Errorf("error updating NginxProxy: %w", err) - GinkgoWriter.Printf("ERROR occurred during updating NginxProxy in namespace %q with release name %q, error: %s\n", - namespace, - releaseName, - updateNginxProxyErr, - ) - - return updateNginxProxyErr + if err = rm.Update(ctx, proxy, nil); err != nil { + return fmt.Errorf("error updating NginxProxy: %w", err) } GinkgoWriter.Printf("Successfully scaled Nginx Deployment in namespace %q with release name %q to %d replicas\n", @@ -966,18 +903,12 @@ func (rm *ResourceManager) GetEvents(namespace string) (*core.EventList, error) defer cancel() var eventList core.EventList - if err := rm.K8sClient.List( + if err := rm.List( ctx, &eventList, client.InNamespace(namespace), ); err != nil { - getEventsListErr := fmt.Errorf("error getting list of Events: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting Events in namespace %q, error: %s\n", - namespace, - getEventsListErr, - ) - - return &core.EventList{}, getEventsListErr + return &core.EventList{}, fmt.Errorf("error getting list of Events: %w", err) } GinkgoWriter.Printf("Successfully found %d Events in namespace %q\n", len(eventList.Items), namespace) @@ -991,27 +922,13 @@ func (rm *ResourceManager) ScaleDeployment(namespace, name string, replicas int3 defer cancel() var deployment apps.Deployment - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &deployment); err != nil { - getDeploymentErr := fmt.Errorf("error getting Deployment: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting Deployment in namespace %q with name %q, error: %s\n", - namespace, - name, - getDeploymentErr, - ) - - return getDeploymentErr + if err := rm.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &deployment); err != nil { + return fmt.Errorf("error getting Deployment: %w", err) } deployment.Spec.Replicas = &replicas - if err := rm.K8sClient.Update(ctx, &deployment); err != nil { - updateDeploymentErr := fmt.Errorf("error updating Deployment: %w", err) - GinkgoWriter.Printf("ERROR occurred during updating Deployment in namespace %q with name %q, error: %s\n", - namespace, - name, - updateDeploymentErr, - ) - - return updateDeploymentErr + if err := rm.Update(ctx, &deployment, nil); err != nil { + return fmt.Errorf("error updating Deployment: %w", err) } GinkgoWriter.Printf("Successfully scaled Deployment %q in namespace %q to %d replicas\n", name, namespace, replicas) @@ -1019,13 +936,16 @@ func (rm *ResourceManager) ScaleDeployment(namespace, name string, replicas int3 } // GetReadyNGFPodNames returns the name(s) of the NGF Pod(s). -func GetReadyNGFPodNames( - k8sClient client.Client, +func (rm *ResourceManager) GetReadyNGFPodNames( namespace, releaseName string, timeout time.Duration, + opts ...Option, ) ([]string, error) { - GinkgoWriter.Printf("Getting ready NGF Pod names in namespace %q with release name %q\n", namespace, releaseName) + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Getting ready NGF Pod names in namespace %q with release name %q\n", namespace, releaseName) + } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -1037,7 +957,7 @@ func GetReadyNGFPodNames( true, // poll immediately func(ctx context.Context) (bool, error) { var podList core.PodList - if err := k8sClient.List( + if err := rm.List( ctx, &podList, client.InNamespace(namespace), @@ -1048,38 +968,45 @@ func GetReadyNGFPodNames( return false, fmt.Errorf("error getting list of NGF Pods: %w", err) } - ngfPodNames = getReadyPodNames(podList) + ngfPodNames = getReadyPodNames(podList, opts...) return len(ngfPodNames) > 0, nil }, ) if err != nil { waitingPodsErr := fmt.Errorf("timed out waiting for NGF Pods to be ready: %w", err) + if options.logEnabled { + GinkgoWriter.Printf( + "ERROR occurred during waiting for NGF Pods to be ready in namespace %q with release name %q, error: %s\n", + namespace, + releaseName, + waitingPodsErr, + ) + } + + return nil, waitingPodsErr + } + if options.logEnabled { GinkgoWriter.Printf( - "ERROR occurred during waiting for NGF Pods to be ready in namespace %q with release name %q, error: %s\n", + "Successfully found ready NGF Pod names in namespace %q with release name %q: %v\n", namespace, releaseName, - waitingPodsErr, + ngfPodNames, ) - - return nil, waitingPodsErr } - GinkgoWriter.Printf( - "Successfully found ready NGF Pod names in namespace %q with release name %q: %v\n", - namespace, - releaseName, - ngfPodNames, - ) return ngfPodNames, nil } // GetReadyNginxPodNames returns the name(s) of the NGINX Pod(s). -func GetReadyNginxPodNames( - k8sClient client.Client, +func (rm *ResourceManager) GetReadyNginxPodNames( namespace string, timeout time.Duration, + opts ...Option, ) ([]string, error) { - GinkgoWriter.Printf("Getting ready NGINX Pod names in namespace %q\n", namespace) + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Getting ready NGINX Pod names in namespace %q\n", namespace) + } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -1091,7 +1018,7 @@ func GetReadyNginxPodNames( true, // poll immediately func(ctx context.Context) (bool, error) { var podList core.PodList - if err := k8sClient.List( + if err := rm.List( ctx, &podList, client.InNamespace(namespace), @@ -1100,29 +1027,33 @@ func GetReadyNginxPodNames( return false, fmt.Errorf("error getting list of NGINX Pods: %w", err) } - nginxPodNames = getReadyPodNames(podList) + nginxPodNames = getReadyPodNames(podList, opts...) return len(nginxPodNames) > 0, nil }, ) if err != nil { waitingPodsErr := fmt.Errorf("timed out waiting for NGINX Pods to be ready: %w", err) - GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n", - namespace, - waitingPodsErr, - ) + if options.logEnabled { + GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n", + namespace, + waitingPodsErr, + ) + } return nil, waitingPodsErr } - GinkgoWriter.Printf( - "Successfully found ready NGINX Pod names in namespace %q: %v\n", - namespace, - nginxPodNames, - ) + if options.logEnabled { + GinkgoWriter.Printf( + "Successfully found ready NGINX Pod name(s) in namespace %q: %v\n", + namespace, + nginxPodNames, + ) + } return nginxPodNames, nil } -func getReadyPodNames(podList core.PodList) []string { +func getReadyPodNames(podList core.PodList, opts ...Option) []string { var names []string for _, pod := range podList.Items { for _, cond := range pod.Status.Conditions { @@ -1131,7 +1062,10 @@ func getReadyPodNames(podList core.PodList) []string { } } } - GinkgoWriter.Printf("Found %d ready pod names: %v\n", len(names), names) + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Found %d ready pod names: %v\n", len(names), names) + } return names } @@ -1153,15 +1087,26 @@ func countNumberOfReadyParents(parents []v1.RouteParentStatus) int { // WaitForPodsToBeReadyWithCount waits for all Pods in the specified namespace to be ready or // until the provided context is canceled. -func (rm *ResourceManager) WaitForPodsToBeReadyWithCount(ctx context.Context, namespace string, count int) error { +func (rm *ResourceManager) WaitForPodsToBeReadyWithCount( + ctx context.Context, + namespace string, + count int, + opts ...Option, +) error { + options := LogOptions(opts...) GinkgoWriter.Printf("Waiting for %d pods to be ready in namespace %q\n", count, namespace) + return wait.PollUntilContextCancel( ctx, 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { var podList core.PodList - if err := rm.K8sClient.List(ctx, &podList, client.InNamespace(namespace)); err != nil { + if err := rm.List( + ctx, + &podList, + client.InNamespace(namespace), + ); err != nil { return false, err } @@ -1173,7 +1118,9 @@ func (rm *ResourceManager) WaitForPodsToBeReadyWithCount(ctx context.Context, na } } } - GinkgoWriter.Printf("Found %d/%d ready pods in namespace %q\n", podsReady, count, namespace) + if options.logEnabled { + GinkgoWriter.Printf("Found %d/%d ready pods in namespace %q\n", podsReady, count, namespace) + } return podsReady == count, nil }, @@ -1199,7 +1146,7 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration( func(ctx context.Context) (bool, error) { var gw v1.Gateway key := types.NamespacedName{Namespace: namespace, Name: name} - if err := rm.K8sClient.Get(ctx, key, &gw); err != nil { + if err := rm.Get(ctx, key, &gw); err != nil { return false, err } @@ -1216,8 +1163,15 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration( // GetNginxConfig uses crossplane to get the nginx configuration and convert it to JSON. // If the crossplane image is loaded locally on the node, crossplaneImageRepo can be empty. -func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneImageRepo string) (*Payload, error) { +func (rm *ResourceManager) GetNginxConfig( + nginxPodName, + namespace, + crossplaneImageRepo string, + opts ...Option, +) (*Payload, error) { GinkgoWriter.Printf("Getting NGINX config from pod %q in namespace %q\n", nginxPodName, namespace) + options := LogOptions(opts...) + if err := injectCrossplaneContainer( rm.ClientGoClient, rm.TimeoutConfig.UpdateTimeout, @@ -1263,10 +1217,12 @@ func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneIma }, ); err != nil { containerErr := fmt.Errorf("could not connect to ephemeral container: %w", err) - GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n", - namespace, - containerErr, - ) + if options.logEnabled { + GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n", + namespace, + containerErr, + ) + } return nil, containerErr } @@ -1286,3 +1242,99 @@ func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneIma return conf, nil } + +// Get retrieves a resource by key, logging errors if enabled. +func (rm *ResourceManager) Get( + ctx context.Context, + key client.ObjectKey, + obj client.Object, + opts ...Option, +) error { + options := LogOptions(opts...) + err := rm.K8sClient.Get(ctx, key, obj) + if err != nil { + if options.logEnabled { + GinkgoWriter.Printf("Could not get k8s resource %q error: %v\n", obj.GetName(), err) + } + + return err + } + + return nil +} + +// Create adds a new resource, returning an error on failure. +func (rm *ResourceManager) Create( + ctx context.Context, + obj client.Object, +) error { + err := rm.K8sClient.Create(ctx, obj) + if err != nil { + createErr := fmt.Errorf("error creating k8s resource %q: %w", obj.GetName(), err) + GinkgoWriter.Printf("%v\n", createErr) + + return createErr + } + return nil +} + +// Delete removes a resource, returning an error on failure. +func (rm *ResourceManager) Delete( + ctx context.Context, + obj client.Object, + deleteOpts []client.DeleteOption, + opts ...Option, +) error { + options := LogOptions(opts...) + err := rm.K8sClient.Delete(ctx, obj, deleteOpts...) + if err != nil { + deleteErr := fmt.Errorf("error deleting k8s resource %q: %w", obj.GetName(), err) + if options.logEnabled { + GinkgoWriter.Printf("%v\n", deleteErr) + } + + return deleteErr + } + return nil +} + +// Update modifies a resource. +func (rm *ResourceManager) Update( + ctx context.Context, + obj client.Object, + updateOpts []client.UpdateOption, + opts ...Option, +) error { + options := LogOptions(opts...) + if err := rm.K8sClient.Update(ctx, obj, updateOpts...); err != nil { + updateResourceErr := fmt.Errorf("error updating k8s resource: %w", err) + if options.logEnabled { + GinkgoWriter.Printf( + "ERROR occurred during updating k8s resource in namespace %q with name %q, error: %s\n", + obj.GetNamespace(), + obj.GetName(), + updateResourceErr, + ) + } + + return updateResourceErr + } + + return nil +} + +// List retrieves a list of resources, returning an error on failure. +func (rm *ResourceManager) List( + ctx context.Context, + list client.ObjectList, + listOpts ...client.ListOption, +) error { + err := rm.K8sClient.List(ctx, list, listOpts...) + if err != nil { + listErr := fmt.Errorf("error listing k8s resources: %w", err) + GinkgoWriter.Printf("%v\n", listErr) + + return listErr + } + return nil +} diff --git a/tests/framework/results.go b/tests/framework/results.go index 120dbcc3a5..ff3e730d08 100644 --- a/tests/framework/results.go +++ b/tests/framework/results.go @@ -9,12 +9,13 @@ import ( "os/exec" "path/filepath" + . "github.com/onsi/ginkgo/v2" vegeta "github.com/tsenart/vegeta/v12/lib" ) // CreateResultsDir creates and returns the name of the results directory for a test. func CreateResultsDir(testName, version string) (string, error) { - pwd, err := os.Getwd() + pwd, err := GetWorkingDir() if err != nil { return "", err } @@ -23,17 +24,23 @@ func CreateResultsDir(testName, version string) (string, error) { if _, err := os.Stat(dirName); err == nil { if err := os.RemoveAll(dirName); err != nil { - return "", fmt.Errorf("failed to remove existing directory %s: %w", dirName, err) + rmDirErr := fmt.Errorf("failed to remove existing directory %s: %w", dirName, err) + GinkgoWriter.Printf("ERROR occurred during removing existing results directory %q, error: %s\n", dirName, rmDirErr) + + return "", rmDirErr } } - return dirName, os.MkdirAll(dirName, 0o777) + return dirName, MkdirAll(dirName, 0o777) } // CreateResultsFile creates and returns the results file for a test. func CreateResultsFile(filename string) (*os.File, error) { - outFile, err := os.OpenFile(filename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0o644) + GinkgoWriter.Printf("Creating results file %q\n", filename) + outFile, err := OpenFile(filename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0o644) if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating results file %q, error: %s\n", filename, err) + return nil, err } @@ -65,18 +72,24 @@ func WriteSystemInfoToFile(file *os.File, ci ClusterInfo, plus bool) error { plus, commit, date, dirty, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode, ) if _, err := fmt.Fprint(file, text); err != nil { + GinkgoWriter.Printf("ERROR occurred during writing system info to results file, error: %s\n", err) + return err } if ci.IsGKE { if _, err := fmt.Fprintf(file, "- Zone: %s\n- Instance Type: %s\n", ci.GkeZone, ci.GkeInstanceType); err != nil { + GinkgoWriter.Printf("ERROR occurred during writing GKE info to results file, error: %s\n", err) + return err } } + GinkgoWriter.Printf("Wrote system info to results file\n") + return nil } func generatePNG(resultsDir, inputFilename, outputFilename, configFilename string) error { - pwd, err := os.Getwd() + pwd, err := GetWorkingDir() if err != nil { return err } @@ -89,6 +102,13 @@ func generatePNG(resultsDir, inputFilename, outputFilename, configFilename strin output, err := cmd.CombinedOutput() if err != nil { + GinkgoWriter.Printf( + "ERROR occurred during generating PNG %q using gnuplot, error: %s, output: %s\n", + outputFilename, + err, + string(output), + ) + return fmt.Errorf("failed to generate PNG: %w; output: %s", err, string(output)) } @@ -118,19 +138,116 @@ func GenerateMemoryPNG(resultsDir, inputFilename, outputFilename string) error { // WriteMetricsResults writes the metrics results to the results file in text format. func WriteMetricsResults(resultsFile *os.File, metrics *Metrics) error { reporter := vegeta.NewTextReporter(&metrics.Metrics) + reporterErr := reporter.Report(resultsFile) + if reporterErr != nil { + GinkgoWriter.Printf("ERROR occurred during writing metrics results to results file, error: %s\n", reporterErr) + } + GinkgoWriter.Printf("Wrote metrics results to results file %q\n", resultsFile.Name()) - return reporter.Report(resultsFile) + return reporterErr } // WriteContent writes basic content to the results file. func WriteContent(resultsFile *os.File, content string) error { if _, err := fmt.Fprintln(resultsFile, content); err != nil { + GinkgoWriter.Printf("ERROR occurred during writing content to results file, error: %s\n", err) + return err } return nil } +// GetWorkingDir returns the current working directory. +func GetWorkingDir() (string, error) { + pwd, err := os.Getwd() + if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting current working directory %q, error: %s\n", pwd, err) + } + + return pwd, err +} + +// CreateFile creates a new file with the given name. +func CreateFile(fileName string) (*os.File, error) { + file, err := os.Create(fileName) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating file %q, error: %s\n", fileName, err) + } + + return file, err +} + +// OpenFile opens an existing file with the given name. +func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { + file, err := os.OpenFile(name, flag, perm) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during openning results file %q, error: %s\n", name, err) + } + + return file, err +} + +// MkdirAll creates a directory with the specified permissions. +func MkdirAll(path string, perm os.FileMode) error { + err := os.MkdirAll(path, perm) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating directory %q, error: %s\n", path, err) + } + + return err +} + +// ReadFile reads the contents of a file. +func ReadFile(file string) ([]byte, error) { + result, err := os.ReadFile(file) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during reading file %q, error: %s\n", file, err) + } + + return result, err +} + +// WriteString writes a string to the given file. +func WriteString(file *os.File, content string) (int, error) { + result, err := io.WriteString(file, content) + if err != nil { + GinkgoWriter.Printf("ERROR writing error log file: %v\n", err) + } + return result, err +} + +// WriteCSVRecord writes a CSV record using the given writer. +func WriteCSVRecord(writer *csv.Writer, record []string) error { + err := writer.Write(record) + if err != nil { + GinkgoWriter.Printf("ERROR writing CSV record: %v\n", err) + } + + return err +} + +// UserHomeDir returns the user's home directory. +func UserHomeDir() (string, error) { + dir, err := os.UserHomeDir() + if err != nil { + GinkgoWriter.Printf("ERROR getting user home directory, error: %s\n", err) + } + GinkgoWriter.Printf("User home directory is %q\n", dir) + + return dir, err +} + +// Remove removes the specified file or empty directory. +func Remove(name string) error { + err := os.Remove(name) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during removing %q, error: %s\n", name, err) + } + + return err +} + // NewVegetaCSVEncoder returns a vegeta CSV encoder. func NewVegetaCSVEncoder(w io.Writer) vegeta.Encoder { return vegeta.NewCSVEncoder(w) @@ -138,18 +255,18 @@ func NewVegetaCSVEncoder(w io.Writer) vegeta.Encoder { // NewCSVResultsWriter creates and returns a CSV results file and writer. func NewCSVResultsWriter(resultsDir, fileName string, resultHeaders ...string) (*os.File, *csv.Writer, error) { - if err := os.MkdirAll(resultsDir, 0o750); err != nil { + if err := MkdirAll(resultsDir, 0o750); err != nil { return nil, nil, err } - file, err := os.OpenFile(filepath.Join(resultsDir, fileName), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) + file, err := OpenFile(filepath.Join(resultsDir, fileName), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) if err != nil { return nil, nil, err } writer := csv.NewWriter(file) - if err = writer.Write(resultHeaders); err != nil { + if err = WriteCSVRecord(writer, resultHeaders); err != nil { return nil, nil, err } diff --git a/tests/suite/advanced_routing_test.go b/tests/suite/advanced_routing_test.go index 1163089cc5..2497e63feb 100644 --- a/tests/suite/advanced_routing_test.go +++ b/tests/suite/advanced_routing_test.go @@ -38,9 +38,12 @@ var _ = Describe("AdvancedRouting", Ordered, Label("functional", "routing"), fun Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) diff --git a/tests/suite/client_settings_test.go b/tests/suite/client_settings_test.go index 53762e58bc..e02283ea99 100644 --- a/tests/suite/client_settings_test.go +++ b/tests/suite/client_settings_test.go @@ -47,7 +47,10 @@ var _ = Describe("ClientSettingsPolicy", Ordered, Label("functional", "cspolicy" Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -393,7 +396,7 @@ func waitForClientSettingsAncestorStatus( func(ctx context.Context) (bool, error) { var pol ngfAPI.ClientSettingsPolicy - if err := k8sClient.Get(ctx, policyNsname, &pol); err != nil { + if err := resourceManager.Get(ctx, policyNsname, &pol); err != nil { return false, err } diff --git a/tests/suite/dataplane_perf_test.go b/tests/suite/dataplane_perf_test.go index cce009ab03..c86ed74b2f 100644 --- a/tests/suite/dataplane_perf_test.go +++ b/tests/suite/dataplane_perf_test.go @@ -64,9 +64,13 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"), Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -88,7 +92,7 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"), }) AfterAll(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace, framework.WithLoggingDisabled()) cleanUpPortForward() Expect(resourceManager.DeleteFromFiles(files, namespace)).To(Succeed()) diff --git a/tests/suite/graceful_recovery_test.go b/tests/suite/graceful_recovery_test.go index 01cfed2652..f85eb966b8 100644 --- a/tests/suite/graceful_recovery_test.go +++ b/tests/suite/graceful_recovery_test.go @@ -74,7 +74,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra defer cancel() var pod core.Pod - if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: podName}, &pod); err != nil { + if err := resourceManager.Get( + ctx, + types.NamespacedName{Namespace: namespace, Name: podName}, + &pod, + ); err != nil { return 0, fmt.Errorf("error retrieving Pod: %w", err) } @@ -95,8 +99,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra } if restartCount != currentRestartCount+1 { - return fmt.Errorf("expected current restart count: %d to match incremented restart count: %d", + restartErr := fmt.Errorf("expected current restart count: %d to match incremented restart count: %d", restartCount, currentRestartCount+1) + GinkgoWriter.Printf("%s\n", restartErr) + + return restartErr } return nil @@ -107,7 +114,7 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra defer cancel() var nodes core.NodeList - if err := k8sClient.List(ctx, &nodes); err != nil { + if err := resourceManager.List(ctx, &nodes); err != nil { return nil, fmt.Errorf("error listing nodes: %w", err) } @@ -125,26 +132,39 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra defer cancel() var nginxPod core.Pod - if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: nginxPodName}, &nginxPod); err != nil { + if err := resourceManager.Get( + ctx, + types.NamespacedName{Namespace: ns.Name, Name: nginxPodName}, + &nginxPod, + ); err != nil { return nil, fmt.Errorf("error retrieving nginx Pod: %w", err) } b, err := resourceManager.GetFileContents("graceful-recovery/node-debugger-job.yaml") if err != nil { - return nil, fmt.Errorf("error processing node debugger job file: %w", err) + debugErr := fmt.Errorf("error processing node debugger job file: %w", err) + GinkgoWriter.Printf("%s\n", debugErr) + + return nil, debugErr } job := &v1.Job{} if err = yaml.Unmarshal(b.Bytes(), job); err != nil { - return nil, fmt.Errorf("error with yaml unmarshal: %w", err) + yamlErr := fmt.Errorf("error with yaml unmarshal: %w", err) + GinkgoWriter.Printf("%s\n", yamlErr) + + return nil, yamlErr } job.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"] = nginxPod.Spec.NodeName if len(job.Spec.Template.Spec.Containers) != 1 { - return nil, fmt.Errorf( + containerErr := fmt.Errorf( "expected node debugger job to contain one container, actual number: %d", len(job.Spec.Template.Spec.Containers), ) + GinkgoWriter.Printf("ERROR: %s\n", containerErr) + + return nil, containerErr } job.Namespace = ns.Name @@ -174,7 +194,7 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra // default propagation policy is metav1.DeletePropagationOrphan which does not delete the underlying // pod created through the job after the job is deleted. Setting it to metav1.DeletePropagationBackground // deletes the underlying pod after the job is deleted. - Expect(resourceManager.Delete( + Expect(resourceManager.DeleteResources( []client.Object{job}, client.PropagationPolicy(metav1.DeletePropagationBackground), )).To(Succeed()) @@ -207,7 +227,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -281,11 +305,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra var podNames []string Eventually( func() bool { - podNames, err = framework.GetReadyNGFPodNames( - k8sClient, + podNames, err = resourceManager.GetReadyNGFPodNames( ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), ) return len(podNames) == 1 && err == nil }). @@ -307,7 +331,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra var nginxPodNames []string Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout * 2). @@ -352,12 +380,15 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra var lease coordination.Lease key := types.NamespacedName{Name: "ngf-test-nginx-gateway-fabric-leader-election", Namespace: ngfNamespace} - if err := k8sClient.Get(ctx, key, &lease); err != nil { + if err := resourceManager.Get(ctx, key, &lease); err != nil { return "", errors.New("could not retrieve leader election lease") } if *lease.Spec.HolderIdentity == "" { - return "", errors.New("leader election lease holder identity is empty") + leaderErr := errors.New("leader election lease holder identity is empty") + GinkgoWriter.Printf("ERROR: %s\n", leaderErr) + + return "", leaderErr } return *lease.Spec.HolderIdentity, nil @@ -381,7 +412,12 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra } BeforeAll(func() { - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout) + podNames, err := resourceManager.GetReadyNGFPodNames( + ngfNamespace, + releaseName, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) @@ -397,7 +433,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -431,7 +471,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra It("recovers when nginx container is restarted", func() { restartNginxContainer(activeNginxPodName, ns.Name, nginxContainerName) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) activeNginxPodName = nginxPodNames[0] @@ -460,16 +504,16 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout) defer cancel() - Expect(k8sClient.Delete(ctx, ngfPod)).To(Succeed()) + Expect(resourceManager.Delete(ctx, ngfPod, nil)).To(Succeed()) var newNGFPodNames []string Eventually( func() bool { - newNGFPodNames, err = framework.GetReadyNGFPodNames( - k8sClient, + newNGFPodNames, err = resourceManager.GetReadyNGFPodNames( ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), ) return len(newNGFPodNames) == 1 && err == nil }). diff --git a/tests/suite/longevity_test.go b/tests/suite/longevity_test.go index 8734ffa5bc..6b0461bcdd 100644 --- a/tests/suite/longevity_test.go +++ b/tests/suite/longevity_test.go @@ -56,7 +56,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.ApplyFromFiles(promFile, ngfNamespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) }) It("collects results", Label("longevity-teardown"), func() { @@ -75,14 +75,14 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu Expect(framework.WriteSystemInfoToFile(resultsFile, clusterInfo, *plusEnabled)).To(Succeed()) // gather wrk output - homeDir, err := os.UserHomeDir() + homeDir, err := framework.UserHomeDir() Expect(err).ToNot(HaveOccurred()) Expect(framework.WriteContent(resultsFile, "\n## Traffic\n")).To(Succeed()) Expect(writeTrafficResults(resultsFile, homeDir, "coffee.txt", "HTTP")).To(Succeed()) Expect(writeTrafficResults(resultsFile, homeDir, "tea.txt", "HTTPS")).To(Succeed()) - framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) + framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name, framework.WithLoggingDisabled()) Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed()) }) @@ -90,7 +90,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu func writeTrafficResults(resultsFile *os.File, homeDir, filename, testname string) error { file := fmt.Sprintf("%s/%s", homeDir, filename) - content, err := os.ReadFile(file) + content, err := framework.ReadFile(file) if err != nil { return err } diff --git a/tests/suite/nginxgateway_test.go b/tests/suite/nginxgateway_test.go index 3518c2ed69..4813661964 100644 --- a/tests/suite/nginxgateway_test.go +++ b/tests/suite/nginxgateway_test.go @@ -13,7 +13,6 @@ import ( "k8s.io/apimachinery/pkg/types" ngfAPI "github.com/nginx/nginx-gateway-fabric/v2/apis/v1alpha1" - "github.com/nginx/nginx-gateway-fabric/v2/tests/framework" ) var _ = Describe("NginxGateway", Ordered, Label("functional", "nginxGateway"), func() { @@ -34,15 +33,8 @@ var _ = Describe("NginxGateway", Ordered, Label("functional", "nginxGateway"), f var nginxGateway ngfAPI.NginxGateway - if err := k8sClient.Get(ctx, nsname, &nginxGateway); err != nil { - gatewayErr := fmt.Errorf("failed to get nginxGateway: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during getting NGINX Gateway in namespace %q: %v\n", - nsname.Namespace, - gatewayErr, - ) - - return nginxGateway, gatewayErr + if err := resourceManager.Get(ctx, nsname, &nginxGateway); err != nil { + return nginxGateway, fmt.Errorf("failed to get nginxGateway: %w", err) } return nginxGateway, nil @@ -115,8 +107,7 @@ var _ = Describe("NginxGateway", Ordered, Label("functional", "nginxGateway"), f } getNGFPodName := func() (string, error) { - podNames, err := framework.GetReadyNGFPodNames( - k8sClient, + podNames, err := resourceManager.GetReadyNGFPodNames( ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout, diff --git a/tests/suite/reconfig_test.go b/tests/suite/reconfig_test.go index ab990e1808..020f86c9b3 100644 --- a/tests/suite/reconfig_test.go +++ b/tests/suite/reconfig_test.go @@ -66,7 +66,8 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r k8sConfig := ctlr.GetConfigOrDie() if !clusterInfo.IsGKE { - Expect(promInstance.PortForward(k8sConfig, promPortForwardStopCh)).To(Succeed()) + pfErr := promInstance.PortForward(k8sConfig, promPortForwardStopCh) + Expect(pfErr).NotTo(HaveOccurred()) } }) @@ -85,6 +86,8 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r }) createUniqueResources := func(resourceCount int, fileName string) error { + GinkgoWriter.Printf("Creating %d unique resources from %s\n", resourceCount, fileName) + var appliedResources []string for i := 1; i <= resourceCount; i++ { namespace := "namespace" + strconv.Itoa(i) @@ -98,11 +101,25 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r fileString = strings.ReplaceAll(fileString, "tea", "tea"+namespace) data := bytes.NewBufferString(fileString) - - if err := resourceManager.ApplyFromBuffer(data, namespace); err != nil { - return fmt.Errorf("error processing manifest file: %w", err) + appliedResources = append(appliedResources, namespace) + + if err := resourceManager.ApplyFromBuffer(data, namespace, framework.WithLoggingDisabled()); err != nil { + manifestErr := fmt.Errorf("error processing manifest file: %w", err) + GinkgoWriter.Printf( + "ERROR on creating and applying unique resources, could proceed %v\n the error happened on %q: %v\n", + appliedResources, + namespace, + manifestErr, + ) + return manifestErr } } + GinkgoWriter.Printf( + "Successfully created %d unique resources from %s: %v\n", + resourceCount, + fileName, + appliedResources, + ) return nil } @@ -117,7 +134,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r Name: "namespace" + strconv.Itoa(i), }, } - Expect(k8sClient.Create(ctx, &ns)).To(Succeed()) + Expect(resourceManager.Create(ctx, &ns)).To(Succeed()) } Expect(resourceManager.Apply([]client.Object{&reconfigNamespace})).To(Succeed()) @@ -126,7 +143,9 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r "reconfig/cafe-secret.yaml", "reconfig/reference-grant.yaml", }, - reconfigNamespace.Name)).To(Succeed()) + reconfigNamespace.Name, + framework.WithLoggingDisabled(), + )).To(Succeed()) Expect(createUniqueResources(resourceCount, "manifests/reconfig/cafe.yaml")).To(Succeed()) @@ -138,7 +157,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r Name: "namespace" + strconv.Itoa(i), }, } - Expect(resourceManager.WaitForPodsToBeReady(ctx, ns.Name)).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReady(ctx, ns.Name, framework.WithLoggingDisabled())).To(Succeed()) } } @@ -147,21 +166,24 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r defer cancel() var namespaces core.NamespaceList - if err := k8sClient.List(ctx, &namespaces); err != nil { + if err := resourceManager.List(ctx, &namespaces, nil); err != nil { return fmt.Errorf("error getting namespaces: %w", err) } + GinkgoWriter.Printf("Found %d namespaces, expected at least%d\n", len(namespaces.Items), resourceCount) Expect(len(namespaces.Items)).To(BeNumerically(">=", resourceCount)) var routes v1.HTTPRouteList - if err := k8sClient.List(ctx, &routes); err != nil { + if err := resourceManager.List(ctx, &routes, nil); err != nil { return fmt.Errorf("error getting HTTPRoutes: %w", err) } + GinkgoWriter.Printf("Found %d HTTPRoutes, expected %d\n", len(routes.Items), resourceCount*3) Expect(routes.Items).To(HaveLen(resourceCount * 3)) var pods core.PodList - if err := k8sClient.List(ctx, &pods); err != nil { + if err := resourceManager.List(ctx, &pods, nil); err != nil { return fmt.Errorf("error getting Pods: %w", err) } + GinkgoWriter.Printf("Found %d Pods, expected at least %d\n", len(pods.Items), resourceCount*2) Expect(len(pods.Items)).To(BeNumerically(">=", resourceCount*2)) return nil @@ -175,7 +197,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r namespaces[i] = "namespace" + strconv.Itoa(i+1) } - err = resourceManager.DeleteNamespaces(namespaces) + err = resourceManager.DeleteNamespaces(namespaces, framework.WithLoggingDisabled()) Expect(resourceManager.DeleteNamespace(reconfigNamespace.Name)).To(Succeed()) return err @@ -196,7 +218,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r } // each call to ValidateNginxFieldExists takes about 1ms - if err := framework.ValidateNginxFieldExists(conf, expUpstream); err != nil { + if err := framework.ValidateNginxFieldExists(conf, expUpstream, framework.WithLoggingDisabled()); err != nil { select { case <-ctx.Done(): return fmt.Errorf("error validating nginx conf was generated in "+namespace+": %w", err.Error()) @@ -222,6 +244,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r if stringTimeToReadyTotal == "0" { stringTimeToReadyTotal = "< 1" } + GinkgoWriter.Printf("Calculated time to ready total for %q: %s\n", nginxPodName, stringTimeToReadyTotal) return stringTimeToReadyTotal } @@ -314,7 +337,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r cfg.nfr = true setup(cfg) - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + podNames, err := resourceManager.GetReadyNGFPodNames(ngfNamespace, releaseName, timeoutConfig.GetTimeout) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) ngfPodName := podNames[0] @@ -323,10 +346,10 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r var nginxPodNames []string Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames( - k8sClient, + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( reconfigNamespace.Name, timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -371,7 +394,12 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r cfg.nfr = true setup(cfg) - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + podNames, err := resourceManager.GetReadyNGFPodNames( + ngfNamespace, + releaseName, + timeoutConfig.GetTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) ngfPodName := podNames[0] @@ -382,10 +410,10 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r var nginxPodNames []string Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames( - k8sClient, + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( reconfigNamespace.Name, timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -406,7 +434,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r File: "http.conf", } - return framework.ValidateNginxFieldExists(conf, defaultUpstream) == nil + return framework.ValidateNginxFieldExists(conf, defaultUpstream, framework.WithLoggingDisabled()) == nil }). WithTimeout(timeoutConfig.CreateTimeout). Should(BeTrue()) @@ -439,7 +467,11 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r }) AfterEach(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, reconfigNamespace.Name) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + reconfigNamespace.Name, + framework.WithLoggingDisabled(), + ) Expect(cleanupResources()).Should(Succeed()) teardown(releaseName) diff --git a/tests/suite/sample_test.go b/tests/suite/sample_test.go index a79dda5a7d..f133cc6acd 100644 --- a/tests/suite/sample_test.go +++ b/tests/suite/sample_test.go @@ -38,7 +38,10 @@ var _ = Describe("Basic test example", Label("functional"), func() { Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 526b27049c..74ac30ce9f 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -112,9 +112,14 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { Name: namespace, }, } - Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{ns}, framework.WithLoggingDisabled())).To(Succeed()) - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + podNames, err := resourceManager.GetReadyNGFPodNames( + ngfNamespace, + releaseName, + timeoutConfig.GetTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) ngfPodName = podNames[0] @@ -159,6 +164,8 @@ The logs are attached only if there are errors. writeScaleResults := func(dest io.Writer, results scaleTestResults) error { tmpl, err := template.New("results").Parse(scaleResultTemplate) if err != nil { + GinkgoWriter.Printf("ERROR creating results template: %v\n", err) + return err } @@ -198,12 +205,12 @@ The logs are attached only if there are errors. // attach error logs if len(errors) > 0 { - f, err := os.Create(fileName) + f, err := framework.CreateFile(fileName) Expect(err).ToNot(HaveOccurred()) defer f.Close() for _, e := range errors { - _, err = io.WriteString(f, fmt.Sprintf("%s\n", e)) + _, err = framework.WriteString(f, fmt.Sprintf("%s\n", e)) Expect(err).ToNot(HaveOccurred()) } } @@ -239,14 +246,19 @@ The logs are attached only if there are errors. q, getStartTime, modifyStartTime, + framework.WithLoggingDisabled(), ), ).WithTimeout(metricExistTimeout).WithPolling(metricExistPolling).Should(Succeed()) } test() - // We sleep for 2 scape intervals to ensure that Prometheus scrapes the metrics after the test() finishes + // We sleep for 2 scrape intervals to ensure that Prometheus scrapes the metrics after the test() finishes // before endTime, so that we don't lose any metric values like reloads. + GinkgoWriter.Printf( + "Sleeping for %v to ensure Prometheus scrapes the metrics after the test finishes\n", + 2*scrapeInterval, + ) time.Sleep(2 * scrapeInterval) endTime := time.Now() @@ -282,6 +294,7 @@ The logs are attached only if there are errors. q, getEndTime, noOpModifier, + framework.WithLoggingDisabled(), ), ).WithTimeout(metricExistTimeout).WithPolling(metricExistPolling).Should(Succeed()) } @@ -307,7 +320,7 @@ The logs are attached only if there are errors. framework.GenerateMemoryPNG(testResultsDir, memCSV, memPNG), ).To(Succeed()) - Expect(os.Remove(memCSV)).To(Succeed()) + Expect(framework.Remove(memCSV)).To(Succeed()) result, err = promInstance.QueryRange( fmt.Sprintf(`rate(container_cpu_usage_seconds_total{pod="%s",container="nginx-gateway"}[2m])`, ngfPodName), @@ -327,7 +340,7 @@ The logs are attached only if there are errors. framework.GenerateCPUPNG(testResultsDir, cpuCSV, cpuPNG), ).To(Succeed()) - Expect(os.Remove(cpuCSV)).To(Succeed()) + Expect(framework.Remove(cpuCSV)).To(Succeed()) eventsCount, err := framework.GetEventsCountWithStartTime(promInstance, ngfPodName, startTime) Expect(err).ToNot(HaveOccurred()) @@ -349,7 +362,11 @@ The logs are attached only if there are errors. []string{`"logger":"usageReporter`}, // ignore usageReporter errors ) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -375,10 +392,15 @@ The logs are attached only if there are errors. findRestarts := func(containerName string, pod *core.Pod) int { for _, containerStatus := range pod.Status.ContainerStatuses { if containerStatus.Name == containerName { + GinkgoWriter.Printf("INFO: container %q had %d restarts\n", containerName, containerStatus.RestartCount) + return int(containerStatus.RestartCount) } } - Fail(fmt.Sprintf("container %s not found", containerName)) + fail := fmt.Sprintf("container %s not found", containerName) + GinkgoWriter.Printf("FAIL: %v\n", fail) + + Fail(fail) return 0 } @@ -417,16 +439,23 @@ The logs are attached only if there are errors. ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace, framework.WithLoggingDisabled())).To(Succeed()) for i := range len(objects.ScaleIterationGroups) { - Expect(resourceManager.Apply(objects.ScaleIterationGroups[i])).To(Succeed()) + Expect(resourceManager.Apply( + objects.ScaleIterationGroups[i], + framework.WithLoggingDisabled(), + )).To(Succeed()) if i == 0 { var nginxPodNames []string Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -463,7 +492,7 @@ The logs are attached only if there are errors. seconds := ttr.Seconds() record := []string{strconv.Itoa(i + 1), strconv.FormatFloat(seconds, 'f', -1, 64)} - Expect(writer.Write(record)).To(Succeed()) + Expect(framework.WriteCSVRecord(writer, record)).To(Succeed()) } writer.Flush() @@ -474,7 +503,7 @@ The logs are attached only if there are errors. framework.GenerateTTRPNG(testResultsDir, ttrCsvFile.Name(), ttrPNG), ).To(Succeed()) - Expect(os.Remove(ttrCsvFile.Name())).To(Succeed()) + Expect(framework.Remove(ttrCsvFile.Name())).To(Succeed()) } runScaleUpstreams := func() { @@ -489,7 +518,11 @@ The logs are attached only if there are errors. var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -508,7 +541,12 @@ The logs are attached only if there are errors. } Eventually( - framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), + framework.CreateResponseChecker( + url, + address, + timeoutConfig.RequestTimeout, + framework.WithLoggingDisabled(), + ), ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) Expect( @@ -518,10 +556,15 @@ The logs are attached only if there are errors. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace, framework.WithLoggingDisabled())).To(Succeed()) Eventually( - framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), + framework.CreateResponseChecker( + url, + address, + timeoutConfig.RequestTimeout, + framework.WithLoggingDisabled(), + ), ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) } @@ -543,7 +586,7 @@ The logs are attached only if there are errors. const testName = "TestScale_Listeners" testResultsDir := filepath.Join(resultsDir, testName) - Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) objects, err := framework.GenerateScaleListenerObjects(httpListenerCount, false /*non-tls*/) Expect(err).ToNot(HaveOccurred()) @@ -567,7 +610,7 @@ The logs are attached only if there are errors. const testName = "TestScale_HTTPSListeners" testResultsDir := filepath.Join(resultsDir, testName) - Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) objects, err := framework.GenerateScaleListenerObjects(httpsListenerCount, true /*tls*/) Expect(err).ToNot(HaveOccurred()) @@ -591,7 +634,7 @@ The logs are attached only if there are errors. const testName = "TestScale_HTTPRoutes" testResultsDir := filepath.Join(resultsDir, testName) - Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) objects, err := framework.GenerateScaleHTTPRouteObjects(httpRouteCount) Expect(err).ToNot(HaveOccurred()) @@ -618,7 +661,7 @@ The logs are attached only if there are errors. const testName = "TestScale_UpstreamServers" testResultsDir := filepath.Join(resultsDir, testName) - Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) runTestWithMetricsAndLogs( testName, @@ -633,13 +676,17 @@ The logs are attached only if there are errors. const testName = "TestScale_HTTPMatches" Expect(resourceManager.ApplyFromFiles(matchesManifests, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) var nginxPodNames []string var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -702,7 +749,11 @@ The logs are attached only if there are errors. }) AfterEach(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + namespace, + framework.WithLoggingDisabled(), + ) cleanUpPortForward() Expect(resourceManager.DeleteNamespace(namespace)).To(Succeed()) teardown(releaseName) @@ -899,13 +950,17 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) var nginxPodNames []string var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -921,7 +976,11 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim }) AfterAll(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + ns.Name, + framework.WithLoggingDisabled(), + ) cleanUpPortForward() teardown(releaseName) @@ -961,7 +1020,12 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.UpdateTimeout) - Expect(resourceManager.WaitForPodsToBeReadyWithCount(ctx, ns.Name, i+numCoffeeAndTeaPods)).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReadyWithCount( + ctx, + ns.Name, + i+numCoffeeAndTeaPods, + framework.WithLoggingDisabled()), + ).To(Succeed()) Expect(resourceManager.WaitForGatewayObservedGeneration(ctx, ns.Name, "gateway", i)).To(Succeed()) cancel() @@ -1037,8 +1101,11 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim var gw v1.Gateway key := types.NamespacedName{Namespace: ns.Name, Name: "gateway"} - if err := resourceManager.K8sClient.Get(ctx, key, &gw); err != nil { - return fmt.Errorf("failed to get gateway: %w", err) + if err := resourceManager.Get(ctx, key, &gw); err != nil { + gatewayErr := fmt.Errorf("failed to get gateway: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", gatewayErr) + + return gatewayErr } if len(gw.Status.Listeners) != num { diff --git a/tests/suite/snippets_filter_test.go b/tests/suite/snippets_filter_test.go index 78e0a592e8..625ea75e7e 100644 --- a/tests/suite/snippets_filter_test.go +++ b/tests/suite/snippets_filter_test.go @@ -40,9 +40,12 @@ var _ = Describe("SnippetsFilter", Ordered, Label("functional", "snippets-filter Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -270,9 +273,7 @@ func checkHTTPRouteToHaveGatewayNotProgrammedCond(httpRouteNsName types.Namespac var hr v1.HTTPRoute var err error - if err = k8sClient.Get(ctx, httpRouteNsName, &hr); err != nil { - GinkgoWriter.Printf("ERROR: failed to get HTTPRoute: %v\n", err) - + if err = resourceManager.Get(ctx, httpRouteNsName, &hr); err != nil { return err } @@ -328,9 +329,7 @@ func checkForSnippetsFilterToBeAccepted(snippetsFilterNsNames types.NamespacedNa var sf ngfAPI.SnippetsFilter var err error - if err = k8sClient.Get(ctx, snippetsFilterNsNames, &sf); err != nil { - GinkgoWriter.Printf("ERROR: failed to get SnippetsFilter: %v\n", err) - + if err = resourceManager.Get(ctx, snippetsFilterNsNames, &sf); err != nil { return err } diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index 71d610bf5a..1576af0361 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -124,7 +124,6 @@ func setup(cfg setupConfig, extraInstallArgs ...string) { options := client.Options{ Scheme: scheme, } - var err error k8sClient, err = client.New(k8sConfig, options) Expect(err).ToNot(HaveOccurred()) @@ -179,8 +178,7 @@ func setup(cfg setupConfig, extraInstallArgs ...string) { installCfg := createNGFInstallConfig(cfg, extraInstallArgs...) - podNames, err := framework.GetReadyNGFPodNames( - k8sClient, + podNames, err := resourceManager.GetReadyNGFPodNames( installCfg.Namespace, installCfg.ReleaseName, timeoutConfig.CreateTimeout, @@ -237,6 +235,7 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo installCfg.NgfImageRepository = *ngfImageRepository installCfg.NginxImageRepository = *nginxImageRepository if *plusEnabled && cfg.nfr { + GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = *nginxPlusImageRepository } installCfg.ImageTag = *imageTag @@ -246,9 +245,11 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo chartVersion = "0.0.0-edge" installCfg.ChartVersion = chartVersion if *plusEnabled && cfg.nfr { + GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = fmt.Sprintf(formatNginxPlusEdgeImagePath, *gkeProject) } case *plusEnabled && cfg.nfr: + GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = fmt.Sprintf(formatNginxPlusEdgeImagePath, *gkeProject) } @@ -263,7 +264,7 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo } if *plusEnabled { - Expect(framework.CreateLicenseSecret(k8sClient, ngfNamespace, *plusLicenseFileName)).To(Succeed()) + Expect(framework.CreateLicenseSecret(resourceManager, ngfNamespace, *plusLicenseFileName)).To(Succeed()) } output, err = framework.InstallNGF(installCfg, extraInstallArgs...) @@ -278,7 +279,7 @@ func teardown(relName string) { Namespace: ngfNamespace, } - output, err := framework.UninstallNGF(cfg, k8sClient) + output, err := framework.UninstallNGF(cfg, resourceManager) Expect(err).ToNot(HaveOccurred(), string(output)) output, err = framework.UninstallGatewayAPI(*gatewayAPIVersion) @@ -293,7 +294,11 @@ func teardown(relName string) { true, /* poll immediately */ func(ctx context.Context) (bool, error) { key := k8sTypes.NamespacedName{Name: ngfNamespace} - if err := k8sClient.Get(ctx, key, &core.Namespace{}); err != nil && apierrors.IsNotFound(err) { + if err := resourceManager.Get( + ctx, + key, + &core.Namespace{}, + ); err != nil && apierrors.IsNotFound(err) { return true, nil } diff --git a/tests/suite/tracing_test.go b/tests/suite/tracing_test.go index bdc1abda54..9b31d47be0 100644 --- a/tests/suite/tracing_test.go +++ b/tests/suite/tracing_test.go @@ -52,11 +52,11 @@ var _ = Describe("Tracing", FlakeAttempts(2), Ordered, Label("functional", "trac key := types.NamespacedName{Name: "ngf-test-proxy-config", Namespace: "nginx-gateway"} var nginxProxy ngfAPIv1alpha2.NginxProxy - Expect(k8sClient.Get(ctx, key, &nginxProxy)).To(Succeed()) + Expect(resourceManager.Get(ctx, key, &nginxProxy)).To(Succeed()) nginxProxy.Spec.Telemetry = &telemetry - Expect(k8sClient.Update(ctx, &nginxProxy)).To(Succeed()) + Expect(resourceManager.Update(ctx, &nginxProxy, nil)).To(Succeed()) } BeforeAll(func() { @@ -92,7 +92,10 @@ var _ = Describe("Tracing", FlakeAttempts(2), Ordered, Label("functional", "trac Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -240,9 +243,7 @@ func verifyGatewayClassResolvedRefs() error { defer cancel() var gc gatewayv1.GatewayClass - if err := k8sClient.Get(ctx, types.NamespacedName{Name: gatewayClassName}, &gc); err != nil { - GinkgoWriter.Printf("ERROR getting GatewayClass %s: %v\n", gatewayClassName, err) - + if err := resourceManager.Get(ctx, types.NamespacedName{Name: gatewayClassName}, &gc); err != nil { return err } @@ -272,9 +273,7 @@ func verifyPolicyStatus() error { var pol ngfAPIv1alpha2.ObservabilityPolicy key := types.NamespacedName{Name: "test-observability-policy", Namespace: "helloworld"} - if err := k8sClient.Get(ctx, key, &pol); err != nil { - GinkgoWriter.Printf("ERROR getting ObservabilityPolicy %q in namespace %q: %v\n", key.Name, key.Namespace, err) - + if err := resourceManager.Get(ctx, key, &pol); err != nil { return err } diff --git a/tests/suite/upgrade_test.go b/tests/suite/upgrade_test.go index cc864ff990..3e8e5fa297 100644 --- a/tests/suite/upgrade_test.go +++ b/tests/suite/upgrade_test.go @@ -67,7 +67,11 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -83,7 +87,11 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { }) AfterEach(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + ns.Name, + framework.WithLoggingDisabled(), + ) cleanUpPortForward() Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) @@ -204,7 +212,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { Expect(resourceManager.ApplyFromFiles([]string{"ngf-upgrade/gateway-updated.yaml"}, ns.Name)).To(Succeed()) - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + podNames, err := resourceManager.GetReadyNGFPodNames(ngfNamespace, releaseName, timeoutConfig.GetTimeout) Expect(err).ToNot(HaveOccurred()) Expect(podNames).ToNot(BeEmpty()) @@ -220,7 +228,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { true, /* poll immediately */ func(_ context.Context) (bool, error) { defer GinkgoRecover() - Expect(k8sClient.Get(leaseCtx, key, &lease)).To(Succeed()) + Expect(resourceManager.Get(leaseCtx, key, &lease)).To(Succeed()) if lease.Spec.HolderIdentity != nil { for _, podName := range podNames { @@ -245,7 +253,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { - Expect(k8sClient.Get(ctx, key, &gw)).To(Succeed()) + Expect(resourceManager.Get(ctx, key, &gw)).To(Succeed()) expListenerName := "http-new" for _, listener := range gw.Status.Listeners { if listener.Name == v1.SectionName(expListenerName) { diff --git a/tests/suite/upstream_settings_test.go b/tests/suite/upstream_settings_test.go index cd70604ef0..8d598e19b1 100644 --- a/tests/suite/upstream_settings_test.go +++ b/tests/suite/upstream_settings_test.go @@ -51,7 +51,10 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -138,7 +141,7 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic DescribeTable("are set properly for", func(expCfgs []framework.ExpectedNginxField) { for _, expCfg := range expCfgs { - Expect(framework.ValidateNginxFieldExists(conf, expCfg)).To(Succeed()) + Expect(framework.ValidateNginxFieldExists(conf, expCfg, framework.WithLoggingDisabled())).To(Succeed()) } }, Entry("HTTP upstreams", []framework.ExpectedNginxField{ @@ -318,7 +321,7 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic DescribeTable("are set properly for", func(expCfgs []framework.ExpectedNginxField) { for _, expCfg := range expCfgs { - Expect(framework.ValidateNginxFieldExists(conf, expCfg)).To(Succeed()) + Expect(framework.ValidateNginxFieldExists(conf, expCfg, framework.WithLoggingDisabled())).To(Succeed()) } }, Entry("Coffee upstream", []framework.ExpectedNginxField{ @@ -424,7 +427,7 @@ func usPolicyHasNoAncestors(usPolicyNsName types.NamespacedName) bool { defer cancel() var usPolicy ngfAPI.UpstreamSettingsPolicy - if err := k8sClient.Get(ctx, usPolicyNsName, &usPolicy); err != nil { + if err := resourceManager.Get(ctx, usPolicyNsName, &usPolicy); err != nil { GinkgoWriter.Printf("ERROR: Failed to get UpstreamSettingsPolicy %q: %s", usPolicyNsName, err.Error()) return false } @@ -464,9 +467,7 @@ func waitForUSPolicyStatus( var usPolicy ngfAPI.UpstreamSettingsPolicy var err error - if err := k8sClient.Get(ctx, usPolicyNsName, &usPolicy); err != nil { - GinkgoWriter.Printf("ERROR: Failed to get UpstreamSettingsPolicy %q: %s", usPolicyNsName, err.Error()) - + if err := resourceManager.Get(ctx, usPolicyNsName, &usPolicy); err != nil { return false, err } From 255adbbc2f3bcad009a28e213082165dd641b44b Mon Sep 17 00:00:00 2001 From: Tina Usova Date: Wed, 10 Sep 2025 14:52:36 +0100 Subject: [PATCH 05/11] Fix reconfig tests --- tests/suite/reconfig_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/suite/reconfig_test.go b/tests/suite/reconfig_test.go index 020f86c9b3..d0a1d2bbaf 100644 --- a/tests/suite/reconfig_test.go +++ b/tests/suite/reconfig_test.go @@ -166,21 +166,21 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r defer cancel() var namespaces core.NamespaceList - if err := resourceManager.List(ctx, &namespaces, nil); err != nil { + if err := resourceManager.List(ctx, &namespaces); err != nil { return fmt.Errorf("error getting namespaces: %w", err) } GinkgoWriter.Printf("Found %d namespaces, expected at least%d\n", len(namespaces.Items), resourceCount) Expect(len(namespaces.Items)).To(BeNumerically(">=", resourceCount)) var routes v1.HTTPRouteList - if err := resourceManager.List(ctx, &routes, nil); err != nil { + if err := resourceManager.List(ctx, &routes); err != nil { return fmt.Errorf("error getting HTTPRoutes: %w", err) } GinkgoWriter.Printf("Found %d HTTPRoutes, expected %d\n", len(routes.Items), resourceCount*3) Expect(routes.Items).To(HaveLen(resourceCount * 3)) var pods core.PodList - if err := resourceManager.List(ctx, &pods, nil); err != nil { + if err := resourceManager.List(ctx, &pods); err != nil { return fmt.Errorf("error getting Pods: %w", err) } GinkgoWriter.Printf("Found %d Pods, expected at least %d\n", len(pods.Items), resourceCount*2) From 8d1dd951e2cc93abf7c0bc11af6553f6441c5bfc Mon Sep 17 00:00:00 2001 From: Tina Usova Date: Wed, 10 Sep 2025 15:38:47 +0100 Subject: [PATCH 06/11] Remove os methods wrappers --- tests/framework/results.go | 105 +++------------------------------- tests/suite/longevity_test.go | 4 +- tests/suite/scale_test.go | 20 +++---- 3 files changed, 19 insertions(+), 110 deletions(-) diff --git a/tests/framework/results.go b/tests/framework/results.go index ff3e730d08..87bba0a0b4 100644 --- a/tests/framework/results.go +++ b/tests/framework/results.go @@ -15,7 +15,7 @@ import ( // CreateResultsDir creates and returns the name of the results directory for a test. func CreateResultsDir(testName, version string) (string, error) { - pwd, err := GetWorkingDir() + pwd, err := os.Getwd() if err != nil { return "", err } @@ -31,13 +31,12 @@ func CreateResultsDir(testName, version string) (string, error) { } } - return dirName, MkdirAll(dirName, 0o777) + return dirName, os.MkdirAll(dirName, 0o777) } // CreateResultsFile creates and returns the results file for a test. func CreateResultsFile(filename string) (*os.File, error) { - GinkgoWriter.Printf("Creating results file %q\n", filename) - outFile, err := OpenFile(filename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0o644) + outFile, err := os.OpenFile(filename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0o644) if err != nil { GinkgoWriter.Printf("ERROR occurred during creating results file %q, error: %s\n", filename, err) @@ -89,7 +88,7 @@ func WriteSystemInfoToFile(file *os.File, ci ClusterInfo, plus bool) error { } func generatePNG(resultsDir, inputFilename, outputFilename, configFilename string) error { - pwd, err := GetWorkingDir() + pwd, err := os.Getwd() if err != nil { return err } @@ -158,96 +157,6 @@ func WriteContent(resultsFile *os.File, content string) error { return nil } -// GetWorkingDir returns the current working directory. -func GetWorkingDir() (string, error) { - pwd, err := os.Getwd() - if err != nil { - GinkgoWriter.Printf("ERROR occurred during getting current working directory %q, error: %s\n", pwd, err) - } - - return pwd, err -} - -// CreateFile creates a new file with the given name. -func CreateFile(fileName string) (*os.File, error) { - file, err := os.Create(fileName) - if err != nil { - GinkgoWriter.Printf("ERROR occurred during creating file %q, error: %s\n", fileName, err) - } - - return file, err -} - -// OpenFile opens an existing file with the given name. -func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { - file, err := os.OpenFile(name, flag, perm) - if err != nil { - GinkgoWriter.Printf("ERROR occurred during openning results file %q, error: %s\n", name, err) - } - - return file, err -} - -// MkdirAll creates a directory with the specified permissions. -func MkdirAll(path string, perm os.FileMode) error { - err := os.MkdirAll(path, perm) - if err != nil { - GinkgoWriter.Printf("ERROR occurred during creating directory %q, error: %s\n", path, err) - } - - return err -} - -// ReadFile reads the contents of a file. -func ReadFile(file string) ([]byte, error) { - result, err := os.ReadFile(file) - if err != nil { - GinkgoWriter.Printf("ERROR occurred during reading file %q, error: %s\n", file, err) - } - - return result, err -} - -// WriteString writes a string to the given file. -func WriteString(file *os.File, content string) (int, error) { - result, err := io.WriteString(file, content) - if err != nil { - GinkgoWriter.Printf("ERROR writing error log file: %v\n", err) - } - return result, err -} - -// WriteCSVRecord writes a CSV record using the given writer. -func WriteCSVRecord(writer *csv.Writer, record []string) error { - err := writer.Write(record) - if err != nil { - GinkgoWriter.Printf("ERROR writing CSV record: %v\n", err) - } - - return err -} - -// UserHomeDir returns the user's home directory. -func UserHomeDir() (string, error) { - dir, err := os.UserHomeDir() - if err != nil { - GinkgoWriter.Printf("ERROR getting user home directory, error: %s\n", err) - } - GinkgoWriter.Printf("User home directory is %q\n", dir) - - return dir, err -} - -// Remove removes the specified file or empty directory. -func Remove(name string) error { - err := os.Remove(name) - if err != nil { - GinkgoWriter.Printf("ERROR occurred during removing %q, error: %s\n", name, err) - } - - return err -} - // NewVegetaCSVEncoder returns a vegeta CSV encoder. func NewVegetaCSVEncoder(w io.Writer) vegeta.Encoder { return vegeta.NewCSVEncoder(w) @@ -255,18 +164,18 @@ func NewVegetaCSVEncoder(w io.Writer) vegeta.Encoder { // NewCSVResultsWriter creates and returns a CSV results file and writer. func NewCSVResultsWriter(resultsDir, fileName string, resultHeaders ...string) (*os.File, *csv.Writer, error) { - if err := MkdirAll(resultsDir, 0o750); err != nil { + if err := os.MkdirAll(resultsDir, 0o750); err != nil { return nil, nil, err } - file, err := OpenFile(filepath.Join(resultsDir, fileName), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) + file, err := os.OpenFile(filepath.Join(resultsDir, fileName), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) if err != nil { return nil, nil, err } writer := csv.NewWriter(file) - if err = WriteCSVRecord(writer, resultHeaders); err != nil { + if err = writer.Write(resultHeaders); err != nil { return nil, nil, err } diff --git a/tests/suite/longevity_test.go b/tests/suite/longevity_test.go index 6b0461bcdd..0a758b5b49 100644 --- a/tests/suite/longevity_test.go +++ b/tests/suite/longevity_test.go @@ -75,7 +75,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu Expect(framework.WriteSystemInfoToFile(resultsFile, clusterInfo, *plusEnabled)).To(Succeed()) // gather wrk output - homeDir, err := framework.UserHomeDir() + homeDir, err := os.UserHomeDir() Expect(err).ToNot(HaveOccurred()) Expect(framework.WriteContent(resultsFile, "\n## Traffic\n")).To(Succeed()) @@ -90,7 +90,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu func writeTrafficResults(resultsFile *os.File, homeDir, filename, testname string) error { file := fmt.Sprintf("%s/%s", homeDir, filename) - content, err := framework.ReadFile(file) + content, err := os.ReadFile(file) if err != nil { return err } diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 74ac30ce9f..3d26514026 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -205,12 +205,12 @@ The logs are attached only if there are errors. // attach error logs if len(errors) > 0 { - f, err := framework.CreateFile(fileName) + f, err := os.Create(fileName) Expect(err).ToNot(HaveOccurred()) defer f.Close() for _, e := range errors { - _, err = framework.WriteString(f, fmt.Sprintf("%s\n", e)) + _, err = io.WriteString(f, fmt.Sprintf("%s\n", e)) Expect(err).ToNot(HaveOccurred()) } } @@ -320,7 +320,7 @@ The logs are attached only if there are errors. framework.GenerateMemoryPNG(testResultsDir, memCSV, memPNG), ).To(Succeed()) - Expect(framework.Remove(memCSV)).To(Succeed()) + Expect(os.Remove(memCSV)).To(Succeed()) result, err = promInstance.QueryRange( fmt.Sprintf(`rate(container_cpu_usage_seconds_total{pod="%s",container="nginx-gateway"}[2m])`, ngfPodName), @@ -340,7 +340,7 @@ The logs are attached only if there are errors. framework.GenerateCPUPNG(testResultsDir, cpuCSV, cpuPNG), ).To(Succeed()) - Expect(framework.Remove(cpuCSV)).To(Succeed()) + Expect(os.Remove(cpuCSV)).To(Succeed()) eventsCount, err := framework.GetEventsCountWithStartTime(promInstance, ngfPodName, startTime) Expect(err).ToNot(HaveOccurred()) @@ -492,7 +492,7 @@ The logs are attached only if there are errors. seconds := ttr.Seconds() record := []string{strconv.Itoa(i + 1), strconv.FormatFloat(seconds, 'f', -1, 64)} - Expect(framework.WriteCSVRecord(writer, record)).To(Succeed()) + Expect(writer.Write(record)).To(Succeed()) } writer.Flush() @@ -503,7 +503,7 @@ The logs are attached only if there are errors. framework.GenerateTTRPNG(testResultsDir, ttrCsvFile.Name(), ttrPNG), ).To(Succeed()) - Expect(framework.Remove(ttrCsvFile.Name())).To(Succeed()) + Expect(os.Remove(ttrCsvFile.Name())).To(Succeed()) } runScaleUpstreams := func() { @@ -586,7 +586,7 @@ The logs are attached only if there are errors. const testName = "TestScale_Listeners" testResultsDir := filepath.Join(resultsDir, testName) - Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) objects, err := framework.GenerateScaleListenerObjects(httpListenerCount, false /*non-tls*/) Expect(err).ToNot(HaveOccurred()) @@ -610,7 +610,7 @@ The logs are attached only if there are errors. const testName = "TestScale_HTTPSListeners" testResultsDir := filepath.Join(resultsDir, testName) - Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) objects, err := framework.GenerateScaleListenerObjects(httpsListenerCount, true /*tls*/) Expect(err).ToNot(HaveOccurred()) @@ -634,7 +634,7 @@ The logs are attached only if there are errors. const testName = "TestScale_HTTPRoutes" testResultsDir := filepath.Join(resultsDir, testName) - Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) objects, err := framework.GenerateScaleHTTPRouteObjects(httpRouteCount) Expect(err).ToNot(HaveOccurred()) @@ -661,7 +661,7 @@ The logs are attached only if there are errors. const testName = "TestScale_UpstreamServers" testResultsDir := filepath.Join(resultsDir, testName) - Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) runTestWithMetricsAndLogs( testName, From fde9fdf57cb67694f737ffad6ddaa373ab283474 Mon Sep 17 00:00:00 2001 From: Tina Usova Date: Thu, 11 Sep 2025 09:18:39 +0100 Subject: [PATCH 07/11] fix delete err --- tests/framework/prometheus.go | 2 -- tests/framework/resourcemanager.go | 9 ++++----- tests/suite/dataplane_perf_test.go | 2 +- tests/suite/longevity_test.go | 2 +- tests/suite/reconfig_test.go | 6 ++++-- tests/suite/scale_test.go | 12 ++++++------ tests/suite/upgrade_test.go | 2 +- 7 files changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/framework/prometheus.go b/tests/framework/prometheus.go index d03ca4e44f..37d72e1c49 100644 --- a/tests/framework/prometheus.go +++ b/tests/framework/prometheus.go @@ -191,7 +191,6 @@ func (ins *PrometheusInstance) PortForward(config *rest.Config, stopCh <-chan st } func (ins *PrometheusInstance) getAPIClient() (v1.API, error) { - GinkgoWriter.Printf("Creating Prometheus API client for pod %q in namespace %q\n", ins.podName, ins.podNamespace) var endpoint string if ins.portForward { endpoint = fmt.Sprintf("http://localhost:%d", PrometheusPortForwardPort) @@ -231,7 +230,6 @@ func (ins *PrometheusInstance) ensureAPIClient() error { // Query sends a query to Prometheus. func (ins *PrometheusInstance) Query(query string) (model.Value, error) { - GinkgoWriter.Printf("Querying Prometheus with query: %q\n", query) ctx, cancel := context.WithTimeout(context.Background(), ins.queryTimeout) defer cancel() diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go index d795877f11..cc012e0cda 100644 --- a/tests/framework/resourcemanager.go +++ b/tests/framework/resourcemanager.go @@ -265,7 +265,7 @@ func (rm *ResourceManager) DeleteNamespace(name string, opts ...Option) error { } func (rm *ResourceManager) DeleteNamespaces(names []string, opts ...Option) error { - GinkgoWriter.Printf("Deleting namespaces: %v\n", names) + GinkgoWriter.Printf("Deleting %d namespaces\n", len(names)) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteNamespaceTimeout*2) defer cancel() @@ -288,7 +288,7 @@ func (rm *ResourceManager) DeleteNamespaces(names []string, opts ...Option) erro true, /* poll immediately */ func(ctx context.Context) (bool, error) { nsList := &core.NamespaceList{} - if err := rm.List(ctx, nsList, nil); err != nil { + if err := rm.List(ctx, nsList); err != nil { return false, nil //nolint:nilerr // retry on error } @@ -1288,12 +1288,11 @@ func (rm *ResourceManager) Delete( options := LogOptions(opts...) err := rm.K8sClient.Delete(ctx, obj, deleteOpts...) if err != nil { - deleteErr := fmt.Errorf("error deleting k8s resource %q: %w", obj.GetName(), err) if options.logEnabled { - GinkgoWriter.Printf("%v\n", deleteErr) + GinkgoWriter.Printf("Could not delete k8s resource %q: %w\n", obj.GetName(), err) } - return deleteErr + return err } return nil } diff --git a/tests/suite/dataplane_perf_test.go b/tests/suite/dataplane_perf_test.go index c86ed74b2f..d085779644 100644 --- a/tests/suite/dataplane_perf_test.go +++ b/tests/suite/dataplane_perf_test.go @@ -63,7 +63,7 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"), } Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) - Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) + Expect(resourceManager.ApplyFromFiles(files, ns.Name, framework.WithLoggingDisabled())).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) nginxPodNames, err := resourceManager.GetReadyNginxPodNames( diff --git a/tests/suite/longevity_test.go b/tests/suite/longevity_test.go index 0a758b5b49..fb917eed8a 100644 --- a/tests/suite/longevity_test.go +++ b/tests/suite/longevity_test.go @@ -84,7 +84,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name, framework.WithLoggingDisabled()) Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) - Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed()) + Expect(resourceManager.DeleteNamespace(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) }) }) diff --git a/tests/suite/reconfig_test.go b/tests/suite/reconfig_test.go index d0a1d2bbaf..7e46a23373 100644 --- a/tests/suite/reconfig_test.go +++ b/tests/suite/reconfig_test.go @@ -137,7 +137,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r Expect(resourceManager.Create(ctx, &ns)).To(Succeed()) } - Expect(resourceManager.Apply([]client.Object{&reconfigNamespace})).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{&reconfigNamespace}, framework.WithLoggingDisabled())).To(Succeed()) Expect(resourceManager.ApplyFromFiles( []string{ "reconfig/cafe-secret.yaml", @@ -271,6 +271,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r q, getStartTime, modifyStartTime, + framework.WithLoggingDisabled(), ), ).WithTimeout(metricExistTimeout).WithPolling(metricExistPolling).Should(Succeed()) } @@ -299,6 +300,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r q, getEndTime, noOpModifier, + framework.WithLoggingDisabled(), ), ).WithTimeout(metricExistTimeout).WithPolling(metricExistPolling).Should(Succeed()) } @@ -404,7 +406,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r Expect(podNames).To(HaveLen(1)) ngfPodName := podNames[0] - Expect(resourceManager.Apply([]client.Object{&reconfigNamespace})).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{&reconfigNamespace}, framework.WithLoggingDisabled())).To(Succeed()) Expect(resourceManager.ApplyFromFiles([]string{"reconfig/gateway.yaml"}, reconfigNamespace.Name)).To(Succeed()) var nginxPodNames []string diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 3d26514026..7273e88088 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -507,12 +507,12 @@ The logs are attached only if there are errors. } runScaleUpstreams := func() { - Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) // apply HTTPRoute after upstreams are ready - Expect(resourceManager.ApplyFromFiles(httpRouteManifests, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + Expect(resourceManager.ApplyFromFiles(httpRouteManifests, namespace, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) var nginxPodNames []string var err error @@ -755,7 +755,7 @@ The logs are attached only if there are errors. framework.WithLoggingDisabled(), ) cleanUpPortForward() - Expect(resourceManager.DeleteNamespace(namespace)).To(Succeed()) + Expect(resourceManager.DeleteNamespace(namespace, framework.WithLoggingDisabled())).To(Succeed()) teardown(releaseName) }) @@ -948,7 +948,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim cfg.nfr = true setup(cfg, "--values", test.valuesFile) - Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{&ns}, framework.WithLoggingDisabled())).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) diff --git a/tests/suite/upgrade_test.go b/tests/suite/upgrade_test.go index 3e8e5fa297..9f7c4f38da 100644 --- a/tests/suite/upgrade_test.go +++ b/tests/suite/upgrade_test.go @@ -63,7 +63,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { }, } - Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{&ns}, framework.WithLoggingDisabled())).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) From 8d501ecfec05ee44b734fb0b0f6529ed593db10e Mon Sep 17 00:00:00 2001 From: Tina Usova Date: Thu, 11 Sep 2025 11:12:32 +0100 Subject: [PATCH 08/11] Fix review comments --- tests/framework/resourcemanager.go | 19 +++++--------- tests/suite/advanced_routing_test.go | 2 +- tests/suite/dataplane_perf_test.go | 7 +++-- tests/suite/graceful_recovery_test.go | 7 ----- tests/suite/longevity_test.go | 4 +-- tests/suite/reconfig_test.go | 12 +++------ tests/suite/scale_test.go | 38 +++++++++------------------ tests/suite/snippets_filter_test.go | 2 +- tests/suite/system_suite_test.go | 3 --- tests/suite/upgrade_test.go | 4 +-- tests/suite/upstream_settings_test.go | 4 +-- 11 files changed, 33 insertions(+), 69 deletions(-) diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go index cc012e0cda..903e8b1cfa 100644 --- a/tests/framework/resourcemanager.go +++ b/tests/framework/resourcemanager.go @@ -165,6 +165,7 @@ func (rm *ResourceManager) ApplyFromFiles(files []string, namespace string, opts func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace string, opts ...Option) error { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) defer cancel() + options := LogOptions(opts...) if options.logEnabled { GinkgoWriter.Printf("Applying resources from buffer to namespace %q\n", namespace) @@ -198,9 +199,7 @@ func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace strin }) if err != nil { retryErr := fmt.Errorf("error updating resource: %w", err) - GinkgoWriter.Printf("%s\n", - retryErr, - ) + GinkgoWriter.Printf("%s\n", retryErr) return retryErr } @@ -723,7 +722,7 @@ func (rm *ResourceManager) GetPodNames(namespace string, labels client.MatchingL for _, pod := range podList.Items { names = append(names, pod.Name) } - GinkgoWriter.Printf("Found pod name in namespace %q: %v\n", namespace, names) + GinkgoWriter.Printf("Found pod names in namespace %q: %v\n", namespace, names) return names, nil } @@ -1251,8 +1250,7 @@ func (rm *ResourceManager) Get( opts ...Option, ) error { options := LogOptions(opts...) - err := rm.K8sClient.Get(ctx, key, obj) - if err != nil { + if err := rm.K8sClient.Get(ctx, key, obj); err != nil { if options.logEnabled { GinkgoWriter.Printf("Could not get k8s resource %q error: %v\n", obj.GetName(), err) } @@ -1268,8 +1266,7 @@ func (rm *ResourceManager) Create( ctx context.Context, obj client.Object, ) error { - err := rm.K8sClient.Create(ctx, obj) - if err != nil { + if err := rm.K8sClient.Create(ctx, obj); err != nil { createErr := fmt.Errorf("error creating k8s resource %q: %w", obj.GetName(), err) GinkgoWriter.Printf("%v\n", createErr) @@ -1286,8 +1283,7 @@ func (rm *ResourceManager) Delete( opts ...Option, ) error { options := LogOptions(opts...) - err := rm.K8sClient.Delete(ctx, obj, deleteOpts...) - if err != nil { + if err := rm.K8sClient.Delete(ctx, obj, deleteOpts...); err != nil { if options.logEnabled { GinkgoWriter.Printf("Could not delete k8s resource %q: %w\n", obj.GetName(), err) } @@ -1328,8 +1324,7 @@ func (rm *ResourceManager) List( list client.ObjectList, listOpts ...client.ListOption, ) error { - err := rm.K8sClient.List(ctx, list, listOpts...) - if err != nil { + if err := rm.K8sClient.List(ctx, list, listOpts...); err != nil { listErr := fmt.Errorf("error listing k8s resources: %w", err) GinkgoWriter.Printf("%v\n", listErr) diff --git a/tests/suite/advanced_routing_test.go b/tests/suite/advanced_routing_test.go index 2497e63feb..944a9f0832 100644 --- a/tests/suite/advanced_routing_test.go +++ b/tests/suite/advanced_routing_test.go @@ -38,7 +38,7 @@ var _ = Describe("AdvancedRouting", Ordered, Label("functional", "routing"), fun Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) nginxPodNames, err := resourceManager.GetReadyNginxPodNames( namespace, diff --git a/tests/suite/dataplane_perf_test.go b/tests/suite/dataplane_perf_test.go index d085779644..45bd8ef295 100644 --- a/tests/suite/dataplane_perf_test.go +++ b/tests/suite/dataplane_perf_test.go @@ -63,13 +63,12 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"), } Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) - Expect(resourceManager.ApplyFromFiles(files, ns.Name, framework.WithLoggingDisabled())).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) nginxPodNames, err := resourceManager.GetReadyNginxPodNames( namespace, timeoutConfig.GetTimeout, - framework.WithLoggingDisabled(), ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -92,7 +91,7 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"), }) AfterAll(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, namespace, framework.WithLoggingDisabled()) + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) cleanUpPortForward() Expect(resourceManager.DeleteFromFiles(files, namespace)).To(Succeed()) diff --git a/tests/suite/graceful_recovery_test.go b/tests/suite/graceful_recovery_test.go index f85eb966b8..a5d3348958 100644 --- a/tests/suite/graceful_recovery_test.go +++ b/tests/suite/graceful_recovery_test.go @@ -230,7 +230,6 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra nginxPodNames, err = resourceManager.GetReadyNginxPodNames( ns.Name, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -309,7 +308,6 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) return len(podNames) == 1 && err == nil }). @@ -334,7 +332,6 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra nginxPodNames, err = resourceManager.GetReadyNginxPodNames( ns.Name, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -416,7 +413,6 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) @@ -436,7 +432,6 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra nginxPodNames, err := resourceManager.GetReadyNginxPodNames( ns.Name, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -474,7 +469,6 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra nginxPodNames, err := resourceManager.GetReadyNginxPodNames( ns.Name, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -513,7 +507,6 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) return len(newNGFPodNames) == 1 && err == nil }). diff --git a/tests/suite/longevity_test.go b/tests/suite/longevity_test.go index fb917eed8a..6d9cc2e37e 100644 --- a/tests/suite/longevity_test.go +++ b/tests/suite/longevity_test.go @@ -82,9 +82,9 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu Expect(writeTrafficResults(resultsFile, homeDir, "coffee.txt", "HTTP")).To(Succeed()) Expect(writeTrafficResults(resultsFile, homeDir, "tea.txt", "HTTPS")).To(Succeed()) - framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name, framework.WithLoggingDisabled()) + framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) - Expect(resourceManager.DeleteNamespace(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed()) }) }) diff --git a/tests/suite/reconfig_test.go b/tests/suite/reconfig_test.go index 7e46a23373..398d62510f 100644 --- a/tests/suite/reconfig_test.go +++ b/tests/suite/reconfig_test.go @@ -64,10 +64,8 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r Expect(err).ToNot(HaveOccurred()) k8sConfig := ctlr.GetConfigOrDie() - if !clusterInfo.IsGKE { - pfErr := promInstance.PortForward(k8sConfig, promPortForwardStopCh) - Expect(pfErr).NotTo(HaveOccurred()) + Expect(promInstance.PortForward(k8sConfig, promPortForwardStopCh)).To(Succeed()) } }) @@ -103,7 +101,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r data := bytes.NewBufferString(fileString) appliedResources = append(appliedResources, namespace) - if err := resourceManager.ApplyFromBuffer(data, namespace, framework.WithLoggingDisabled()); err != nil { + if err := resourceManager.ApplyFromBuffer(data, namespace); err != nil { manifestErr := fmt.Errorf("error processing manifest file: %w", err) GinkgoWriter.Printf( "ERROR on creating and applying unique resources, could proceed %v\n the error happened on %q: %v\n", @@ -137,14 +135,13 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r Expect(resourceManager.Create(ctx, &ns)).To(Succeed()) } - Expect(resourceManager.Apply([]client.Object{&reconfigNamespace}, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{&reconfigNamespace})).To(Succeed()) Expect(resourceManager.ApplyFromFiles( []string{ "reconfig/cafe-secret.yaml", "reconfig/reference-grant.yaml", }, reconfigNamespace.Name, - framework.WithLoggingDisabled(), )).To(Succeed()) Expect(createUniqueResources(resourceCount, "manifests/reconfig/cafe.yaml")).To(Succeed()) @@ -157,7 +154,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r Name: "namespace" + strconv.Itoa(i), }, } - Expect(resourceManager.WaitForPodsToBeReady(ctx, ns.Name, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReady(ctx, ns.Name)).To(Succeed()) } } @@ -271,7 +268,6 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r q, getStartTime, modifyStartTime, - framework.WithLoggingDisabled(), ), ).WithTimeout(metricExistTimeout).WithPolling(metricExistPolling).Should(Succeed()) } diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 7273e88088..70becef9f3 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -112,13 +112,12 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { Name: namespace, }, } - Expect(resourceManager.Apply([]client.Object{ns}, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) podNames, err := resourceManager.GetReadyNGFPodNames( ngfNamespace, releaseName, timeoutConfig.GetTimeout, - framework.WithLoggingDisabled(), ) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) @@ -246,7 +245,6 @@ The logs are attached only if there are errors. q, getStartTime, modifyStartTime, - framework.WithLoggingDisabled(), ), ).WithTimeout(metricExistTimeout).WithPolling(metricExistPolling).Should(Succeed()) } @@ -294,7 +292,6 @@ The logs are attached only if there are errors. q, getEndTime, noOpModifier, - framework.WithLoggingDisabled(), ), ).WithTimeout(metricExistTimeout).WithPolling(metricExistPolling).Should(Succeed()) } @@ -365,7 +362,6 @@ The logs are attached only if there are errors. nginxPodNames, err := resourceManager.GetReadyNginxPodNames( namespace, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -439,12 +435,11 @@ The logs are attached only if there are errors. ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) for i := range len(objects.ScaleIterationGroups) { Expect(resourceManager.Apply( objects.ScaleIterationGroups[i], - framework.WithLoggingDisabled(), )).To(Succeed()) if i == 0 { @@ -454,7 +449,6 @@ The logs are attached only if there are errors. nginxPodNames, err = resourceManager.GetReadyNginxPodNames( namespace, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -507,12 +501,12 @@ The logs are attached only if there are errors. } runScaleUpstreams := func() { - Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace, framework.WithLoggingDisabled())).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) // apply HTTPRoute after upstreams are ready - Expect(resourceManager.ApplyFromFiles(httpRouteManifests, namespace, framework.WithLoggingDisabled())).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.ApplyFromFiles(httpRouteManifests, namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) var nginxPodNames []string var err error @@ -521,7 +515,6 @@ The logs are attached only if there are errors. nginxPodNames, err = resourceManager.GetReadyNginxPodNames( namespace, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -545,7 +538,6 @@ The logs are attached only if there are errors. url, address, timeoutConfig.RequestTimeout, - framework.WithLoggingDisabled(), ), ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) @@ -556,14 +548,13 @@ The logs are attached only if there are errors. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) Eventually( framework.CreateResponseChecker( url, address, timeoutConfig.RequestTimeout, - framework.WithLoggingDisabled(), ), ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) } @@ -676,7 +667,7 @@ The logs are attached only if there are errors. const testName = "TestScale_HTTPMatches" Expect(resourceManager.ApplyFromFiles(matchesManifests, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) var nginxPodNames []string var err error @@ -685,7 +676,6 @@ The logs are attached only if there are errors. nginxPodNames, err = resourceManager.GetReadyNginxPodNames( namespace, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -752,10 +742,9 @@ The logs are attached only if there are errors. framework.AddNginxLogsAndEventsToReport( resourceManager, namespace, - framework.WithLoggingDisabled(), ) cleanUpPortForward() - Expect(resourceManager.DeleteNamespace(namespace, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.DeleteNamespace(namespace)).To(Succeed()) teardown(releaseName) }) @@ -948,9 +937,9 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim cfg.nfr = true setup(cfg, "--values", test.valuesFile) - Expect(resourceManager.Apply([]client.Object{&ns}, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) var nginxPodNames []string var err error @@ -959,7 +948,6 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim nginxPodNames, err = resourceManager.GetReadyNginxPodNames( ns.Name, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -979,7 +967,6 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim framework.AddNginxLogsAndEventsToReport( resourceManager, ns.Name, - framework.WithLoggingDisabled(), ) cleanUpPortForward() @@ -1023,8 +1010,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim Expect(resourceManager.WaitForPodsToBeReadyWithCount( ctx, ns.Name, - i+numCoffeeAndTeaPods, - framework.WithLoggingDisabled()), + i+numCoffeeAndTeaPods), ).To(Succeed()) Expect(resourceManager.WaitForGatewayObservedGeneration(ctx, ns.Name, "gateway", i)).To(Succeed()) diff --git a/tests/suite/snippets_filter_test.go b/tests/suite/snippets_filter_test.go index 625ea75e7e..af67646a7d 100644 --- a/tests/suite/snippets_filter_test.go +++ b/tests/suite/snippets_filter_test.go @@ -40,7 +40,7 @@ var _ = Describe("SnippetsFilter", Ordered, Label("functional", "snippets-filter Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) nginxPodNames, err := resourceManager.GetReadyNginxPodNames( namespace, diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index 1576af0361..31ce49c0d0 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -235,7 +235,6 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo installCfg.NgfImageRepository = *ngfImageRepository installCfg.NginxImageRepository = *nginxImageRepository if *plusEnabled && cfg.nfr { - GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = *nginxPlusImageRepository } installCfg.ImageTag = *imageTag @@ -245,11 +244,9 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo chartVersion = "0.0.0-edge" installCfg.ChartVersion = chartVersion if *plusEnabled && cfg.nfr { - GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = fmt.Sprintf(formatNginxPlusEdgeImagePath, *gkeProject) } case *plusEnabled && cfg.nfr: - GinkgoWriter.Printf("Using NGINX Plus image repository %q\n", *nginxPlusImageRepository) installCfg.NginxImageRepository = fmt.Sprintf(formatNginxPlusEdgeImagePath, *gkeProject) } diff --git a/tests/suite/upgrade_test.go b/tests/suite/upgrade_test.go index 9f7c4f38da..1636893003 100644 --- a/tests/suite/upgrade_test.go +++ b/tests/suite/upgrade_test.go @@ -63,14 +63,13 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { }, } - Expect(resourceManager.Apply([]client.Object{&ns}, framework.WithLoggingDisabled())).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) nginxPodNames, err := resourceManager.GetReadyNginxPodNames( ns.Name, timeoutConfig.GetStatusTimeout, - framework.WithLoggingDisabled(), ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -90,7 +89,6 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { framework.AddNginxLogsAndEventsToReport( resourceManager, ns.Name, - framework.WithLoggingDisabled(), ) cleanUpPortForward() diff --git a/tests/suite/upstream_settings_test.go b/tests/suite/upstream_settings_test.go index 8d598e19b1..2d53aac520 100644 --- a/tests/suite/upstream_settings_test.go +++ b/tests/suite/upstream_settings_test.go @@ -141,7 +141,7 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic DescribeTable("are set properly for", func(expCfgs []framework.ExpectedNginxField) { for _, expCfg := range expCfgs { - Expect(framework.ValidateNginxFieldExists(conf, expCfg, framework.WithLoggingDisabled())).To(Succeed()) + Expect(framework.ValidateNginxFieldExists(conf, expCfg)).To(Succeed()) } }, Entry("HTTP upstreams", []framework.ExpectedNginxField{ @@ -321,7 +321,7 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic DescribeTable("are set properly for", func(expCfgs []framework.ExpectedNginxField) { for _, expCfg := range expCfgs { - Expect(framework.ValidateNginxFieldExists(conf, expCfg, framework.WithLoggingDisabled())).To(Succeed()) + Expect(framework.ValidateNginxFieldExists(conf, expCfg)).To(Succeed()) } }, Entry("Coffee upstream", []framework.ExpectedNginxField{ From d4b76f42d84fe109fe48d732265b85337906e9e4 Mon Sep 17 00:00:00 2001 From: Tina Usova Date: Thu, 11 Sep 2025 12:01:26 +0100 Subject: [PATCH 09/11] Fix misprit --- tests/framework/resourcemanager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go index 903e8b1cfa..f8bc54acaa 100644 --- a/tests/framework/resourcemanager.go +++ b/tests/framework/resourcemanager.go @@ -1063,7 +1063,7 @@ func getReadyPodNames(podList core.PodList, opts ...Option) []string { } options := LogOptions(opts...) if options.logEnabled { - GinkgoWriter.Printf("Found %d ready pod names: %v\n", len(names), names) + GinkgoWriter.Printf("Found %d ready pod name(s): %v\n", len(names), names) } return names From e6c42355b0546fc3af09ee9615d7275cf3fd99a5 Mon Sep 17 00:00:00 2001 From: Tina Usova Date: Thu, 11 Sep 2025 12:32:12 +0100 Subject: [PATCH 10/11] return disable logging for scale and reconfig tests --- tests/suite/reconfig_test.go | 6 +++++- tests/suite/scale_test.go | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/suite/reconfig_test.go b/tests/suite/reconfig_test.go index 398d62510f..51816d0736 100644 --- a/tests/suite/reconfig_test.go +++ b/tests/suite/reconfig_test.go @@ -101,7 +101,11 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r data := bytes.NewBufferString(fileString) appliedResources = append(appliedResources, namespace) - if err := resourceManager.ApplyFromBuffer(data, namespace); err != nil { + if err := resourceManager.ApplyFromBuffer( + data, + namespace, + framework.WithLoggingDisabled(), // disable logging to avoid huge logs for 150 resources + ); err != nil { manifestErr := fmt.Errorf("error processing manifest file: %w", err) GinkgoWriter.Printf( "ERROR on creating and applying unique resources, could proceed %v\n the error happened on %q: %v\n", diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 70becef9f3..adecea7e5c 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -440,6 +440,7 @@ The logs are attached only if there are errors. for i := range len(objects.ScaleIterationGroups) { Expect(resourceManager.Apply( objects.ScaleIterationGroups[i], + framework.WithLoggingDisabled(), // disable logging to avoid huge log )).To(Succeed()) if i == 0 { @@ -477,7 +478,7 @@ The logs are attached only if there are errors. url, address, timeoutConfig.RequestTimeout, - framework.WithLoggingDisabled(), // disable logging to avoid huge logs + framework.WithLoggingDisabled(), // disable logging to avoid huge log for 1000 requests ), ).WithTimeout(6 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) From d7571435664996802806309dbc413621df9d16fc Mon Sep 17 00:00:00 2001 From: Tina Usova Date: Fri, 12 Sep 2025 11:09:47 +0100 Subject: [PATCH 11/11] Remove installation logs duplication cused by bad rebasing --- tests/framework/ngf.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/framework/ngf.go b/tests/framework/ngf.go index 27eddcd0de..0bb98d1956 100644 --- a/tests/framework/ngf.go +++ b/tests/framework/ngf.go @@ -41,7 +41,6 @@ type InstallationConfig struct { func InstallGatewayAPI(apiVersion string) ([]byte, error) { apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion) GinkgoWriter.Printf("Installing Gateway API version %q at API path %q\n", apiVersion, apiPath) - GinkgoWriter.Printf("Installing Gateway API version %q at API path %q\n", apiVersion, apiPath) cmd := exec.CommandContext( context.Background(), @@ -62,18 +61,14 @@ func InstallGatewayAPI(apiVersion string) ([]byte, error) { func UninstallGatewayAPI(apiVersion string) ([]byte, error) { apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion) GinkgoWriter.Printf("Uninstalling Gateway API version %q at API path %q\n", apiVersion, apiPath) - GinkgoWriter.Printf("Uninstalling Gateway API version %q at API path %q\n", apiVersion, apiPath) output, err := exec.CommandContext(context.Background(), "kubectl", "delete", "-f", apiPath).CombinedOutput() if err != nil && !strings.Contains(string(output), "not found") { GinkgoWriter.Printf("Error uninstalling Gateway API version %q: %v\n", apiVersion, err) - GinkgoWriter.Printf("Error uninstalling Gateway API version %q: %v\n", apiVersion, err) - return output, err } GinkgoWriter.Printf("Successfully uninstalled Gateway API version %q\n", apiVersion) - GinkgoWriter.Printf("Successfully uninstalled Gateway API version %q\n", apiVersion) return nil, nil }