diff --git a/tests/framework/crossplane.go b/tests/framework/crossplane.go index 02a16b6cb5..90b50faef4 100644 --- a/tests/framework/crossplane.go +++ b/tests/framework/crossplane.go @@ -8,6 +8,7 @@ import ( "strings" "time" + . "github.com/onsi/ginkgo/v2" core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -42,10 +43,13 @@ const crossplaneImageName = "nginx-crossplane:latest" // ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field, // and returns whether or not that field exists where it should. -func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) error { +func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField, opts ...Option) error { b, err := json.Marshal(conf) if err != nil { - return fmt.Errorf("error marshaling nginx config: %w", err) + marshalErr := fmt.Errorf("error marshaling nginx config: %w", err) + GinkgoWriter.Printf("%v\n", marshalErr) + + return marshalErr } for _, config := range conf.Config { @@ -55,7 +59,7 @@ func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) err for _, directive := range config.Parsed { if expFieldCfg.Server == "" && expFieldCfg.Upstream == "" { - if expFieldCfg.fieldFound(directive) { + if expFieldCfg.fieldFound(directive, opts...) { return nil } continue @@ -65,13 +69,15 @@ func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) err return nil } - if expFieldCfg.Upstream != "" && fieldExistsInUpstream(expFieldCfg, *directive) { + if expFieldCfg.Upstream != "" && fieldExistsInUpstream(expFieldCfg, *directive, opts...) { return nil } } } + directiveErr := fmt.Errorf("directive %s not found in: nginx config %s", expFieldCfg.Directive, string(b)) + GinkgoWriter.Printf("ERROR: %v\n", directiveErr) - return fmt.Errorf("directive %s not found in: nginx config %s", expFieldCfg.Directive, string(b)) + return directiveErr } func fieldExistsInServer( @@ -94,7 +100,16 @@ func fieldExistsInServer( func fieldExistsInUpstream( expFieldCfg ExpectedNginxField, directive Directive, + opts ...Option, ) bool { + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf( + "Checking upstream for directive %q with value %q\n", + expFieldCfg.Directive, + expFieldCfg.Value, + ) + } if directive.Directive == "upstream" && directive.Args[0] == expFieldCfg.Upstream { for _, directive := range directive.Block { if expFieldCfg.fieldFound(directive) { @@ -115,7 +130,8 @@ func getServerName(serverBlock Directives) string { return "" } -func (e ExpectedNginxField) fieldFound(directive *Directive) bool { +func (e ExpectedNginxField) fieldFound(directive *Directive, opts ...Option) bool { + options := LogOptions(opts...) arg := strings.Join(directive.Args, " ") valueMatch := arg == e.Value @@ -123,7 +139,20 @@ func (e ExpectedNginxField) fieldFound(directive *Directive) bool { valueMatch = strings.Contains(arg, e.Value) } - return directive.Directive == e.Directive && valueMatch + if directive.Directive == e.Directive && valueMatch { + if options.logEnabled { + GinkgoWriter.Printf( + "Found field %q with value %q in field %q with value %q\n", + e.Directive, + e.Value, + directive.Directive, + arg, + ) + } + return true + } + + return false } func fieldExistsInLocation(locationDirective *Directive, expFieldCfg ExpectedNginxField) bool { @@ -201,7 +230,10 @@ func injectCrossplaneContainer( podClient := k8sClient.CoreV1().Pods(namespace) if _, err := podClient.UpdateEphemeralContainers(ctx, ngfPodName, pod, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("error adding ephemeral container: %w", err) + containerErr := fmt.Errorf("error adding ephemeral container: %w", err) + GinkgoWriter.Printf("%v\n", containerErr) + + return containerErr } return nil @@ -231,7 +263,10 @@ func createCrossplaneExecutor( exec, err := remotecommand.NewSPDYExecutor(k8sConfig, http.MethodPost, req.URL()) if err != nil { - return nil, fmt.Errorf("error creating executor: %w", err) + executorErr := fmt.Errorf("error creating executor: %w", err) + GinkgoWriter.Printf("%v\n", executorErr) + + return nil, executorErr } return exec, nil diff --git a/tests/framework/generate_manifests.go b/tests/framework/generate_manifests.go index c7a00e234a..8e5d436fd6 100644 --- a/tests/framework/generate_manifests.go +++ b/tests/framework/generate_manifests.go @@ -7,7 +7,6 @@ import ( "io" "text/template" - . "github.com/onsi/ginkgo/v2" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" @@ -222,7 +221,6 @@ func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, err } func generateSecrets(secrets []string) ([]client.Object, error) { - GinkgoWriter.Printf("Generating secrets\n") objects := make([]client.Object, 0, len(secrets)) for _, secret := range secrets { @@ -239,7 +237,6 @@ func generateSecrets(secrets []string) ([]client.Object, error) { objects = append(objects, objs...) } - GinkgoWriter.Printf("Generated %d secrets\n", len(objects)) return objects, nil } diff --git a/tests/framework/info.go b/tests/framework/info.go index c485edc9aa..8f36929bc5 100644 --- a/tests/framework/info.go +++ b/tests/framework/info.go @@ -84,11 +84,11 @@ func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) { } // AddNginxLogsAndEventsToReport adds nginx logs and events from the namespace to the report if the spec failed. -func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string) { +func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string, opts ...Option) { if CurrentSpecReport().Failed() { var returnLogs string - nginxPodNames, _ := GetReadyNginxPodNames(rm.K8sClient, namespace, rm.TimeoutConfig.GetStatusTimeout) + nginxPodNames, _ := rm.GetReadyNginxPodNames(namespace, rm.TimeoutConfig.GetStatusTimeout, opts...) for _, nginxPodName := range nginxPodNames { returnLogs += fmt.Sprintf("Logs for Nginx Pod %s:\n", nginxPodName) diff --git a/tests/framework/load.go b/tests/framework/load.go index c687f275b1..d6caf312b5 100644 --- a/tests/framework/load.go +++ b/tests/framework/load.go @@ -7,6 +7,7 @@ import ( "net/http" "time" + . "github.com/onsi/ginkgo/v2" vegeta "github.com/tsenart/vegeta/v12/lib" ) @@ -49,6 +50,7 @@ type Metrics struct { // RunLoadTest uses Vegeta to send traffic to the provided Targets at the given rate for the given duration and writes // the results to the provided file. func RunLoadTest(cfg LoadTestConfig) (vegeta.Results, Metrics) { + GinkgoWriter.Printf("Running load test: %s\n", cfg.Description) vegTargets := convertTargetToVegetaTarget(cfg.Targets) targeter := vegeta.NewStaticTargeter(vegTargets...) @@ -61,7 +63,12 @@ func RunLoadTest(cfg LoadTestConfig) (vegeta.Results, Metrics) { Timeout: vegeta.DefaultTimeout, Transport: &http.Transport{ DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { - return dialer.DialContext(ctx, network, cfg.Proxy) + conn, err := dialer.DialContext(ctx, network, cfg.Proxy) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during dialing %q in %q network, error: %s\n", cfg.Proxy, network, err) + } + + return conn, err }, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, //nolint:gosec // self-signed cert for testing diff --git a/tests/framework/logging.go b/tests/framework/logging.go index aaf9f54a48..ddc02672a3 100644 --- a/tests/framework/logging.go +++ b/tests/framework/logging.go @@ -11,3 +11,12 @@ func WithLoggingDisabled() Option { opts.logEnabled = false } } + +func LogOptions(opts ...Option) *Options { + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + + return options +} diff --git a/tests/framework/ngf.go b/tests/framework/ngf.go index 660cb79e8e..0bb98d1956 100644 --- a/tests/framework/ngf.go +++ b/tests/framework/ngf.go @@ -14,7 +14,6 @@ import ( apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -101,7 +100,7 @@ func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) { } // CreateLicenseSecret creates the NGINX Plus JWT secret. -func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) error { +func CreateLicenseSecret(rm ResourceManager, namespace, filename string) error { GinkgoWriter.Printf("Creating NGINX Plus license secret in namespace %q from file %q\n", namespace, filename) conf, err := os.ReadFile(filename) @@ -121,11 +120,8 @@ func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) er }, } - if err := k8sClient.Create(ctx, ns); err != nil && !apierrors.IsAlreadyExists(err) { - createNSErr := fmt.Errorf("error creating namespace: %w", err) - GinkgoWriter.Printf("%v\n", createNSErr) - - return createNSErr + if err := rm.Create(ctx, ns); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("error creating namespace: %w", err) } secret := &core.Secret{ @@ -138,7 +134,7 @@ func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) er }, } - if err := k8sClient.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) { + if err := rm.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) { createSecretErr := fmt.Errorf("error creating secret: %w", err) GinkgoWriter.Printf("%v\n", createSecretErr) @@ -185,7 +181,7 @@ func UpgradeNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) { } // UninstallNGF uninstalls NGF. -func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, error) { +func UninstallNGF(cfg InstallationConfig, rm ResourceManager) ([]byte, error) { args := []string{ "uninstall", cfg.ReleaseName, "--namespace", cfg.Namespace, } @@ -199,20 +195,20 @@ func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, erro ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err = k8sClient.Delete(ctx, &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cfg.Namespace}}) + err = rm.Delete(ctx, &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cfg.Namespace}}, nil) if err != nil && !apierrors.IsNotFound(err) { return nil, err } var crList apiext.CustomResourceDefinitionList - if err := k8sClient.List(ctx, &crList); err != nil { + if err := rm.List(ctx, &crList); err != nil { return nil, err } for _, cr := range crList.Items { if strings.Contains(cr.Spec.Group, "gateway.nginx.org") { cr := cr - if err := k8sClient.Delete(ctx, &cr); err != nil && !apierrors.IsNotFound(err) { + if err := rm.Delete(ctx, &cr, nil); err != nil && !apierrors.IsNotFound(err) { return nil, err } } diff --git a/tests/framework/prometheus.go b/tests/framework/prometheus.go index 6ce8a300bb..37d72e1c49 100644 --- a/tests/framework/prometheus.go +++ b/tests/framework/prometheus.go @@ -10,6 +10,7 @@ import ( "os/exec" "time" + . "github.com/onsi/ginkgo/v2" "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" @@ -49,7 +50,10 @@ func InstallPrometheus( "https://prometheus-community.github.io/helm-charts", ).CombinedOutput() if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to add Prometheus helm repo: %w; output: %s", err, string(output)) + prometheusErr := fmt.Errorf("failed to add Prometheus helm repo: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", prometheusErr) + + return PrometheusInstance{}, prometheusErr } output, err = exec.CommandContext( @@ -59,7 +63,10 @@ func InstallPrometheus( "update", ).CombinedOutput() if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to update helm repos: %w; output: %s", err, string(output)) + helmReposErr := fmt.Errorf("failed to update helm repos: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", helmReposErr) + + return PrometheusInstance{}, helmReposErr } scrapeInterval := fmt.Sprintf("%ds", int(cfg.ScrapeInterval.Seconds())) @@ -77,24 +84,36 @@ func InstallPrometheus( "--wait", ).CombinedOutput() if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to install Prometheus: %w; output: %s", err, string(output)) + prometheusInstallationErr := fmt.Errorf("failed to install Prometheus: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", prometheusInstallationErr) + + return PrometheusInstance{}, prometheusInstallationErr } pods, err := rm.GetPods(prometheusNamespace, client.MatchingLabels{ "app.kubernetes.io/name": "prometheus", }) if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to get Prometheus pods: %w", err) + podsErr := fmt.Errorf("failed to get Prometheus pods: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", podsErr) + + return PrometheusInstance{}, podsErr } if len(pods) != 1 { - return PrometheusInstance{}, fmt.Errorf("expected one Prometheus pod, found %d", len(pods)) + manyPodsErr := fmt.Errorf("expected one Prometheus pod, found %d", len(pods)) + GinkgoWriter.Printf("ERROR: %v\n", manyPodsErr) + + return PrometheusInstance{}, manyPodsErr } pod := pods[0] if pod.Status.PodIP == "" { - return PrometheusInstance{}, errors.New("the Prometheus pod has no IP") + podIPErr := errors.New("the Prometheus pod has no IP") + GinkgoWriter.Printf("ERROR: %v\n", podIPErr) + + return PrometheusInstance{}, podIPErr } var queryTimeout time.Duration @@ -114,6 +133,7 @@ func InstallPrometheus( // UninstallPrometheus uninstalls Prometheus from the cluster. func UninstallPrometheus(rm ResourceManager) error { + GinkgoWriter.Printf("Uninstalling Prometheus from namespace %q\n", prometheusNamespace) output, err := exec.CommandContext( context.Background(), "helm", @@ -122,11 +142,17 @@ func UninstallPrometheus(rm ResourceManager) error { "-n", prometheusNamespace, ).CombinedOutput() if err != nil { - return fmt.Errorf("failed to uninstall Prometheus: %w; output: %s", err, string(output)) + uninstallErr := fmt.Errorf("failed to uninstall Prometheus: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", uninstallErr) + + return uninstallErr } if err := rm.DeleteNamespace(prometheusNamespace); err != nil { - return fmt.Errorf("failed to delete Prometheus namespace: %w", err) + deleteNSErr := fmt.Errorf("failed to delete Prometheus namespace: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", deleteNSErr) + + return deleteNSErr } return nil @@ -150,8 +176,12 @@ type PrometheusInstance struct { // PortForward starts port forwarding to the Prometheus instance. func (ins *PrometheusInstance) PortForward(config *rest.Config, stopCh <-chan struct{}) error { + GinkgoWriter.Printf("Starting port forwarding to Prometheus pod %q in namespace %q\n", ins.podName, ins.podNamespace) if ins.portForward { - panic("port forwarding already started") + infoMsg := "port forwarding already started" + GinkgoWriter.Printf("INFO: %s\n", infoMsg) + + panic(infoMsg) } ins.portForward = true @@ -175,6 +205,8 @@ func (ins *PrometheusInstance) getAPIClient() (v1.API, error) { c, err := api.NewClient(cfg) if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating Prometheus API client: %v\n", err) + return nil, err } @@ -185,7 +217,10 @@ func (ins *PrometheusInstance) ensureAPIClient() error { if ins.apiClient == nil { ac, err := ins.getAPIClient() if err != nil { - return fmt.Errorf("failed to get Prometheus API client: %w", err) + apiClientErr := fmt.Errorf("failed to get Prometheus API client: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", apiClientErr) + + return apiClientErr } ins.apiClient = ac } @@ -209,10 +244,14 @@ func (ins *PrometheusInstance) QueryWithCtx(ctx context.Context, query string) ( result, warnings, err := ins.apiClient.Query(ctx, query, time.Time{}) if err != nil { - return nil, fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", queryErr) + + return nil, queryErr } if len(warnings) > 0 { + GinkgoWriter.Printf("WARNING: Prometheus query returned warnings: %v\n", warnings) slog.InfoContext(context.Background(), "Prometheus query returned warnings", "query", query, @@ -235,16 +274,23 @@ func (ins *PrometheusInstance) QueryRange(query string, promRange v1.Range) (mod func (ins *PrometheusInstance) QueryRangeWithCtx(ctx context.Context, query string, promRange v1.Range, ) (model.Value, error) { + GinkgoWriter.Printf("Querying Prometheus with range query: %q\n", query) if err := ins.ensureAPIClient(); err != nil { + GinkgoWriter.Printf("ERROR during ensureAPIClient for prometheus: %v\n", err) + return nil, err } result, warnings, err := ins.apiClient.QueryRange(ctx, query, promRange) if err != nil { - return nil, fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", queryErr) + + return nil, queryErr } if len(warnings) > 0 { + GinkgoWriter.Printf("WARNING: Prometheus range query returned warnings: %v\n", warnings) slog.InfoContext(context.Background(), "Prometheus range query returned warnings", "query", query, @@ -260,11 +306,17 @@ func (ins *PrometheusInstance) QueryRangeWithCtx(ctx context.Context, func GetFirstValueOfPrometheusVector(val model.Value) (float64, error) { res, ok := val.(model.Vector) if !ok { - return 0, fmt.Errorf("expected a vector, got %T", val) + valueErr := fmt.Errorf("expected a vector, got %T", val) + GinkgoWriter.Printf("ERROR: %v\n", valueErr) + + return 0, valueErr } if len(res) == 0 { - return 0, errors.New("empty vector") + vectorErr := errors.New("empty vector") + GinkgoWriter.Printf("ERROR: %v\n", vectorErr) + + return 0, vectorErr } return float64(res[0].Value), nil @@ -272,8 +324,11 @@ func GetFirstValueOfPrometheusVector(val model.Value) (float64, error) { // WritePrometheusMatrixToCSVFile writes a Prometheus matrix to a CSV file. func WritePrometheusMatrixToCSVFile(fileName string, value model.Value) error { + GinkgoWriter.Printf("Writing Prometheus matrix to CSV file %q\n", fileName) file, err := os.Create(fileName) if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating file %q: %v\n", fileName, err) + return err } defer file.Close() @@ -282,13 +337,18 @@ func WritePrometheusMatrixToCSVFile(fileName string, value model.Value) error { matrix, ok := value.(model.Matrix) if !ok { - return fmt.Errorf("expected a matrix, got %T", value) + matrixErr := fmt.Errorf("expected a matrix, got %T", value) + GinkgoWriter.Printf("ERROR: %v\n", matrixErr) + + return matrixErr } for _, sample := range matrix { for _, pair := range sample.Values { record := []string{fmt.Sprint(pair.Timestamp.Unix()), pair.Value.String()} if err := csvWriter.Write(record); err != nil { + GinkgoWriter.Printf("ERROR: %v\n", err) + return err } } @@ -409,18 +469,30 @@ func CreateMetricExistChecker( query string, getTime func() time.Time, modifyTime func(), + opts ...Option, ) func() error { return func() error { queryWithTimestamp := fmt.Sprintf("%s @ %d", query, getTime().Unix()) + options := LogOptions(opts...) result, err := promInstance.Query(queryWithTimestamp) if err != nil { - return fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating metric existence checker: %v\n", queryErr) + } + + return queryErr } if result.String() == "" { modifyTime() - return errors.New("empty result") + emptyResultErr := errors.New("empty result") + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating metric existence checker: %v\n", emptyResultErr) + } + + return emptyResultErr } return nil @@ -436,6 +508,7 @@ func CreateEndTimeFinder( endTime *time.Time, queryRangeStep time.Duration, ) func() error { + GinkgoWriter.Printf("Creating end time finder with start time %v and initial end time %v\n", startTime, endTime) return func() error { result, err := promInstance.QueryRange(query, v1.Range{ Start: startTime, @@ -443,12 +516,18 @@ func CreateEndTimeFinder( Step: queryRangeStep, }) if err != nil { - return fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + GinkgoWriter.Printf("ERROR during creating end time finder: %v\n", queryErr) + + return queryErr } if result.String() == "" { *endTime = time.Now() - return errors.New("empty result") + emptyResultsErr := errors.New("empty result") + GinkgoWriter.Printf("ERROR during creating end time finder: %v\n", emptyResultsErr) + + return emptyResultsErr } return nil @@ -457,14 +536,29 @@ func CreateEndTimeFinder( // CreateResponseChecker returns a function that checks if there is a successful response from a url. func CreateResponseChecker(url, address string, requestTimeout time.Duration, opts ...Option) func() error { + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Starting checking response for url %q and address %q\n", url, address) + } + return func() error { status, _, err := Get(url, address, requestTimeout, nil, nil, opts...) if err != nil { - return fmt.Errorf("bad response: %w", err) + badReqErr := fmt.Errorf("bad response: %w", err) + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating response checker: %v\n", badReqErr) + } + + return badReqErr } if status != 200 { - return fmt.Errorf("unexpected status code: %d", status) + statusErr := fmt.Errorf("unexpected status code: %d", status) + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating response checker: %v\n", statusErr) + } + + return statusErr } return nil @@ -474,11 +568,15 @@ func CreateResponseChecker(url, address string, requestTimeout time.Duration, op func getFirstValueOfVector(query string, promInstance PrometheusInstance) (float64, error) { result, err := promInstance.Query(query) if err != nil { + GinkgoWriter.Printf("ERROR querying Prometheus during getting first value of vector: %v\n", err) + return 0, err } val, err := GetFirstValueOfPrometheusVector(result) if err != nil { + GinkgoWriter.Printf("ERROR getting first value of Prometheus vector: %v\n", err) + return 0, err } @@ -488,12 +586,17 @@ func getFirstValueOfVector(query string, promInstance PrometheusInstance) (float func getBuckets(query string, promInstance PrometheusInstance) ([]Bucket, error) { result, err := promInstance.Query(query) if err != nil { + GinkgoWriter.Printf("ERROR querying Prometheus during getting buckets: %v\n", err) + return nil, err } res, ok := result.(model.Vector) if !ok { - return nil, errors.New("could not convert result to vector") + convertationErr := errors.New("could not convert result to vector") + GinkgoWriter.Printf("ERROR during getting buckets: %v\n", convertationErr) + + return nil, convertationErr } buckets := make([]Bucket, 0, len(res)) diff --git a/tests/framework/request.go b/tests/framework/request.go index a04d8ccc66..bd8663b8a3 100644 --- a/tests/framework/request.go +++ b/tests/framework/request.go @@ -24,10 +24,7 @@ func Get( headers, queryParams map[string]string, opts ...Option, ) (int, string, error) { - options := &Options{logEnabled: true} - for _, opt := range opts { - opt(options) - } + options := LogOptions(opts...) resp, err := makeRequest(http.MethodGet, url, address, nil, timeout, headers, queryParams, opts...) if err != nil { @@ -99,11 +96,7 @@ func makeRequest( ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - options := &Options{logEnabled: true} - - for _, opt := range opts { - opt(options) - } + options := LogOptions(opts...) if options.logEnabled { requestDetails := fmt.Sprintf( "Method: %s, URL: %s, Address: %s, Headers: %v, QueryParams: %v\n", diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go index f51f17920a..f8bc54acaa 100644 --- a/tests/framework/resourcemanager.go +++ b/tests/framework/resourcemanager.go @@ -76,8 +76,11 @@ type ClusterInfo struct { } // Apply creates or updates Kubernetes resources defined as Go objects. -func (rm *ResourceManager) Apply(resources []client.Object) error { - GinkgoWriter.Printf("Applying resources defined as Go objects\n") +func (rm *ResourceManager) Apply(resources []client.Object, opts ...Option) error { + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Applying resources defined as Go objects\n") + } ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) defer cancel() @@ -101,25 +104,13 @@ func (rm *ResourceManager) Apply(resources []client.Object) error { } } - if err := rm.K8sClient.Get(ctx, client.ObjectKeyFromObject(resource), obj); err != nil { + if err := rm.Get(ctx, client.ObjectKeyFromObject(resource), obj, opts...); err != nil { if !apierrors.IsNotFound(err) { - notFoundErr := fmt.Errorf("error getting resource: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during getting Kubernetes resources: %s\n", - notFoundErr, - ) - - return notFoundErr + return err } - if err := rm.K8sClient.Create(ctx, resource); err != nil { - creatingResourceErr := fmt.Errorf("error creating resource: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during applying creates Kubernetes resources: %s\n", - creatingResourceErr, - ) - - return creatingResourceErr + if err := rm.Create(ctx, resource); err != nil { + return fmt.Errorf("error creating resource: %w", err) } continue @@ -128,83 +119,69 @@ func (rm *ResourceManager) Apply(resources []client.Object) error { // Some tests modify resources that are also modified by NGF (to update their status), so conflicts are possible // For example, a Gateway resource. err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - if err := rm.K8sClient.Get(ctx, client.ObjectKeyFromObject(resource), obj); err != nil { - GinkgoWriter.Printf( - "ERROR occurred during getting Kubernetes resources on retries: %s\n", - err, - ) - + if err := rm.Get(ctx, client.ObjectKeyFromObject(resource), obj); err != nil { return err } resource.SetResourceVersion(obj.GetResourceVersion()) - updateErr := rm.K8sClient.Update(ctx, resource) - if updateErr != nil { - GinkgoWriter.Printf( - "ERROR occurred during updating Kubernetes resources on retries: %s\n", - updateErr, - ) - } - return updateErr + return rm.Update(ctx, resource, nil) }) if err != nil { retryErr := fmt.Errorf("error updating resource: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during retries: %s\n", - retryErr, - ) + GinkgoWriter.Printf("%s\n", retryErr) return retryErr } } - GinkgoWriter.Printf("Resources defined as Go objects applied successfully\n") - + if options.logEnabled { + GinkgoWriter.Printf("Resources defined as Go objects applied successfully\n") + } return nil } // ApplyFromFiles creates or updates Kubernetes resources defined within the provided YAML files. -func (rm *ResourceManager) ApplyFromFiles(files []string, namespace string) error { +func (rm *ResourceManager) ApplyFromFiles(files []string, namespace string, opts ...Option) error { + options := LogOptions(opts...) for _, file := range files { - GinkgoWriter.Printf("Applying resources from file: %q to namespace %q\n", file, namespace) + if options.logEnabled { + GinkgoWriter.Printf("\nApplying resources from file: %q to namespace %q\n", file, namespace) + } data, err := rm.GetFileContents(file) if err != nil { - GinkgoWriter.Printf("ERROR occurred during getting file contents for file %q, error: %s\n", file, err) - return err } if err = rm.ApplyFromBuffer(data, namespace); err != nil { - GinkgoWriter.Printf("ERROR occurred during applying resources from file %q, error: %s\n", file, err) - return err } } - GinkgoWriter.Printf("Resources from files applied successfully to namespace %q,\n", namespace) + if options.logEnabled { + GinkgoWriter.Printf("Resources from files applied successfully to namespace %q,\n", namespace) + } return nil } -func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace string) error { +func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace string, opts ...Option) error { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) defer cancel() + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Applying resources from buffer to namespace %q\n", namespace) + } + handlerFunc := func(obj unstructured.Unstructured) error { obj.SetNamespace(namespace) nsName := types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()} fetchedObj := obj.DeepCopy() - if err := rm.K8sClient.Get(ctx, nsName, fetchedObj); err != nil { + if err := rm.Get(ctx, nsName, fetchedObj, opts...); err != nil { if !apierrors.IsNotFound(err) { - getResourceErr := fmt.Errorf("error getting resource: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting resource from buffer, error: %s\n", getResourceErr) - - return getResourceErr + return err } - if err := rm.K8sClient.Create(ctx, &obj); err != nil { - createResourceErr := fmt.Errorf("error creating resource: %w", err) - GinkgoWriter.Printf("ERROR occurred during creating resource from buffer, error: %s\n", createResourceErr) - - return createResourceErr + if err := rm.Create(ctx, &obj); err != nil { + return fmt.Errorf("error creating resource: %w", err) } return nil @@ -213,28 +190,16 @@ func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace strin // Some tests modify resources that are also modified by NGF (to update their status), so conflicts are possible // For example, a Gateway resource. err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - if err := rm.K8sClient.Get(ctx, nsName, fetchedObj); err != nil { - GinkgoWriter.Printf( - "ERROR occurred during getting resource from buffer on retries, error: %s\n", - err, - ) - + if err := rm.Get(ctx, nsName, fetchedObj); err != nil { return err } obj.SetResourceVersion(fetchedObj.GetResourceVersion()) - updateErr := rm.K8sClient.Update(ctx, &obj) - if updateErr != nil { - GinkgoWriter.Printf("ERROR occurred during updating resource from buffer, error: %s\n", updateErr) - } - return updateErr + return rm.Update(ctx, &obj, nil) }) if err != nil { retryErr := fmt.Errorf("error updating resource: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during retries, while update from buffer error: %s\n", - retryErr, - ) + GinkgoWriter.Printf("%s\n", retryErr) return retryErr } @@ -246,17 +211,14 @@ func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace strin } // Delete deletes Kubernetes resources defined as Go objects. -func (rm *ResourceManager) Delete(resources []client.Object, opts ...client.DeleteOption) error { +func (rm *ResourceManager) DeleteResources(resources []client.Object, opts ...client.DeleteOption) error { GinkgoWriter.Printf("Deleting resources\n") ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteTimeout) defer cancel() for _, resource := range resources { - if err := rm.K8sClient.Delete(ctx, resource, opts...); err != nil && !apierrors.IsNotFound(err) { - delErr := fmt.Errorf("error deleting resource: %w", err) - GinkgoWriter.Printf("ERROR occurred during deleting resource, error: %s\n", delErr) - - return delErr + if err := rm.Delete(ctx, resource, opts); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("error deleting resource: %w", err) } } GinkgoWriter.Printf("Resources deleted successfully\n") @@ -264,29 +226,22 @@ func (rm *ResourceManager) Delete(resources []client.Object, opts ...client.Dele return nil } -func (rm *ResourceManager) DeleteNamespace(name string) error { +func (rm *ResourceManager) DeleteNamespace(name string, opts ...Option) error { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteNamespaceTimeout) GinkgoWriter.Printf("Deleting namespace %q\n", name) defer cancel() ns := &core.Namespace{} - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Name: name}, ns); err != nil { + if err := rm.Get(ctx, types.NamespacedName{Name: name}, ns, opts...); err != nil { if apierrors.IsNotFound(err) { - GinkgoWriter.Printf("Namespace %q not found, nothing to delete\n", name) - return nil } - getNsErr := fmt.Errorf("error getting namespace: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting namespace, error: %s\n", getNsErr) - return getNsErr + return fmt.Errorf("error getting namespace: %w", err) } - if err := rm.K8sClient.Delete(ctx, ns); err != nil { - delErr := fmt.Errorf("error deleting namespace: %w", err) - GinkgoWriter.Printf("ERROR occurred during deleting namespace, error: %s\n", delErr) - - return delErr + if err := rm.Delete(ctx, ns, nil, opts...); err != nil { + return fmt.Errorf("error deleting namespace: %w", err) } GinkgoWriter.Printf("Waiting for namespace %q to be deleted\n", name) @@ -296,24 +251,20 @@ func (rm *ResourceManager) DeleteNamespace(name string) error { 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Name: name}, ns); err != nil { + if err := rm.Get(ctx, types.NamespacedName{Name: name}, ns, opts...); err != nil { if apierrors.IsNotFound(err) { - GinkgoWriter.Printf("Namespace %q not found (deleted)\n", name) - return true, nil } - getNsErr := fmt.Errorf("error getting namespace: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting namespace, error: %s\n", getNsErr) - return false, getNsErr + return false, fmt.Errorf("error getting namespace: %w", err) } return false, nil }) } -func (rm *ResourceManager) DeleteNamespaces(names []string) error { - GinkgoWriter.Printf("Deleting namespaces: %v\n", names) +func (rm *ResourceManager) DeleteNamespaces(names []string, opts ...Option) error { + GinkgoWriter.Printf("Deleting %d namespaces\n", len(names)) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteNamespaceTimeout*2) defer cancel() @@ -321,15 +272,12 @@ func (rm *ResourceManager) DeleteNamespaces(names []string) error { for _, name := range names { ns := &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}} - if err := rm.K8sClient.Delete(ctx, ns); err != nil { + if err := rm.Delete(ctx, ns, nil, opts...); err != nil { if apierrors.IsNotFound(err) { - GinkgoWriter.Printf("Namespace %q not found, nothing to delete\n", name) continue } - delNsErr := fmt.Errorf("error deleting namespace: %w", err) - GinkgoWriter.Printf("ERROR occurred during deleting namespace %q, error: %s\n", name, delNsErr) - combinedErrors = errors.Join(combinedErrors, delNsErr) + combinedErrors = errors.Join(combinedErrors, fmt.Errorf("error deleting namespace: %w", err)) } } @@ -339,7 +287,7 @@ func (rm *ResourceManager) DeleteNamespaces(names []string) error { true, /* poll immediately */ func(ctx context.Context) (bool, error) { nsList := &core.NamespaceList{} - if err := rm.K8sClient.List(ctx, nsList); err != nil { + if err := rm.List(ctx, nsList); err != nil { return false, nil //nolint:nilerr // retry on error } @@ -363,9 +311,7 @@ func (rm *ResourceManager) DeleteFromFiles(files []string, namespace string) err ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteTimeout) defer cancel() - if err := rm.K8sClient.Delete(ctx, &obj); err != nil && !apierrors.IsNotFound(err) { - GinkgoWriter.Printf("ERROR occurred during deleting resource from file, error: %s\n", err) - + if err := rm.Delete(ctx, &obj, nil); err != nil && !apierrors.IsNotFound(err) { return err } @@ -375,8 +321,6 @@ func (rm *ResourceManager) DeleteFromFiles(files []string, namespace string) err for _, file := range files { data, err := rm.GetFileContents(file) if err != nil { - GinkgoWriter.Printf("ERROR occurred during getting file contents for file %q, error: %s\n", file, err) - return err } @@ -419,34 +363,49 @@ func (rm *ResourceManager) readAndHandleObject( // path or an https:// URL to YAML manifests and provides the contents. func (rm *ResourceManager) GetFileContents(file string) (*bytes.Buffer, error) { if strings.HasPrefix(file, "http://") { - return nil, fmt.Errorf("data can't be retrieved from %s: http is not supported, use https", file) + err := fmt.Errorf("data can't be retrieved from %s: http is not supported, use https", file) + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + + return nil, err } else if strings.HasPrefix(file, "https://") { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.ManifestFetchTimeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, file, nil) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + return nil, err } resp, err := http.DefaultClient.Do(req) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%d response when getting %s file contents", resp.StatusCode, file) + err = fmt.Errorf("%d response when getting %s file contents", resp.StatusCode, file) + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + + return nil, err } manifests := new(bytes.Buffer) count, err := manifests.ReadFrom(resp.Body) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + return nil, err } if resp.ContentLength != -1 && count != resp.ContentLength { - return nil, fmt.Errorf("received %d bytes from %s, expected %d", count, file, resp.ContentLength) + err = fmt.Errorf("received %d bytes from %s, expected %d", count, file, resp.ContentLength) + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + + return nil, err } return manifests, nil } @@ -457,6 +416,8 @@ func (rm *ResourceManager) GetFileContents(file string) (*bytes.Buffer, error) { b, err := rm.FS.ReadFile(file) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting file contents for file %q, error: %s\n", file, err) + return nil, err } @@ -465,18 +426,18 @@ func (rm *ResourceManager) GetFileContents(file string) (*bytes.Buffer, error) { // WaitForAppsToBeReady waits for all apps in the specified namespace to be ready, // or until the ctx timeout is reached. -func (rm *ResourceManager) WaitForAppsToBeReady(namespace string) error { +func (rm *ResourceManager) WaitForAppsToBeReady(namespace string, opts ...Option) error { GinkgoWriter.Printf("Waiting for apps to be ready in namespace %q\n", namespace) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) defer cancel() - return rm.WaitForAppsToBeReadyWithCtx(ctx, namespace) + return rm.WaitForAppsToBeReadyWithCtx(ctx, namespace, opts...) } // WaitForAppsToBeReadyWithCtx waits for all apps in the specified namespace to be ready or // until the provided context is canceled. -func (rm *ResourceManager) WaitForAppsToBeReadyWithCtx(ctx context.Context, namespace string) error { - if err := rm.WaitForPodsToBeReady(ctx, namespace); err != nil { +func (rm *ResourceManager) WaitForAppsToBeReadyWithCtx(ctx context.Context, namespace string, opts ...Option) error { + if err := rm.WaitForPodsToBeReady(ctx, namespace, opts...); err != nil { GinkgoWriter.Printf("ERROR occurred during waiting for pods to be ready, error: %s\n", err) return err @@ -504,14 +465,23 @@ func (rm *ResourceManager) WaitForAppsToBeReadyWithCtx(ctx context.Context, name // WaitForPodsToBeReady waits for all Pods in the specified namespace to be ready or // until the provided context is canceled. -func (rm *ResourceManager) WaitForPodsToBeReady(ctx context.Context, namespace string) error { - return wait.PollUntilContextCancel( +func (rm *ResourceManager) WaitForPodsToBeReady( + ctx context.Context, + namespace string, + opts ...Option, +) error { + options := LogOptions(opts...) + waitingErr := wait.PollUntilContextCancel( ctx, 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { var podList core.PodList - if err := rm.K8sClient.List(ctx, &podList, client.InNamespace(namespace)); err != nil { + if err := rm.List( + ctx, + &podList, + client.InNamespace(namespace), + ); err != nil { return false, err } @@ -523,10 +493,22 @@ func (rm *ResourceManager) WaitForPodsToBeReady(ctx context.Context, namespace s } } } + if options.logEnabled { + GinkgoWriter.Printf("Pods ready: %d out of %d in namespace %q\n", podsReady, len(podList.Items), namespace) + } return podsReady == len(podList.Items), nil }, ) + if waitingErr != nil { + GinkgoWriter.Printf( + "ERROR occurred during waiting for Pods to be ready in namespace %q, error: %s\n", + namespace, + waitingErr, + ) + } + + return waitingErr } func (rm *ResourceManager) waitForGatewaysToBeReady(ctx context.Context, namespace string) error { @@ -536,7 +518,11 @@ func (rm *ResourceManager) waitForGatewaysToBeReady(ctx context.Context, namespa true, /* poll immediately */ func(ctx context.Context) (bool, error) { var gatewayList v1.GatewayList - if err := rm.K8sClient.List(ctx, &gatewayList, client.InNamespace(namespace)); err != nil { + if err := rm.List( + ctx, + &gatewayList, + client.InNamespace(namespace), + ); err != nil { return false, err } @@ -554,13 +540,18 @@ func (rm *ResourceManager) waitForGatewaysToBeReady(ctx context.Context, namespa } func (rm *ResourceManager) waitForHTTPRoutesToBeReady(ctx context.Context, namespace string) error { + GinkgoWriter.Printf("Waiting for HTTPRoutes to be ready in namespace %q\n", namespace) return wait.PollUntilContextCancel( ctx, 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { var routeList v1.HTTPRouteList - if err := rm.K8sClient.List(ctx, &routeList, client.InNamespace(namespace)); err != nil { + if err := rm.List( + ctx, + &routeList, + client.InNamespace(namespace), + ); err != nil { return false, err } @@ -576,10 +567,13 @@ func (rm *ResourceManager) waitForHTTPRoutesToBeReady(ctx context.Context, names } func (rm *ResourceManager) waitForGRPCRoutesToBeReady(ctx context.Context, namespace string) error { + GinkgoWriter.Printf("Waiting for GRPCRoutes to be ready in namespace %q\n", namespace) // First, check if grpcroute even exists for v1. If not, ignore. var routeList v1.GRPCRouteList - err := rm.K8sClient.List(ctx, &routeList, client.InNamespace(namespace)) + err := rm.List(ctx, &routeList, client.InNamespace(namespace)) if err != nil && strings.Contains(err.Error(), "no matches for kind") { + GinkgoWriter.Printf("No GRPCRoute resources found in namespace %q, skipping wait\n", namespace) + return nil } @@ -589,7 +583,11 @@ func (rm *ResourceManager) waitForGRPCRoutesToBeReady(ctx context.Context, names true, /* poll immediately */ func(ctx context.Context) (bool, error) { var routeList v1.GRPCRouteList - if err := rm.K8sClient.List(ctx, &routeList, client.InNamespace(namespace)); err != nil { + if err := rm.List( + ctx, + &routeList, + client.InNamespace(namespace), + ); err != nil { return false, err } @@ -612,10 +610,10 @@ func (rm *ResourceManager) GetLBIPAddress(namespace string) (string, error) { var serviceList core.ServiceList var address string - if err := rm.K8sClient.List(ctx, &serviceList, client.InNamespace(namespace)); err != nil { - GinkgoWriter.Printf("ERROR occurred during getting list of services in namespace %q, error: %s\n", - namespace, err) - + if err := rm.List( + ctx, &serviceList, + client.InNamespace(namespace), + ); err != nil { return "", err } var nsName types.NamespacedName @@ -624,14 +622,7 @@ func (rm *ResourceManager) GetLBIPAddress(namespace string) (string, error) { if svc.Spec.Type == core.ServiceTypeLoadBalancer { nsName = types.NamespacedName{Namespace: svc.GetNamespace(), Name: svc.GetName()} if err := rm.waitForLBStatusToBeReady(ctx, nsName); err != nil { - lbStatusErr := fmt.Errorf("error getting status from LoadBalancer service: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during waiting for LoadBalancer service in namespace %q to be ready, error: %s\n", - nsName, - err, - ) - - return "", lbStatusErr + return "", fmt.Errorf("error getting status from LoadBalancer service: %w", err) } } } @@ -639,14 +630,8 @@ func (rm *ResourceManager) GetLBIPAddress(namespace string) (string, error) { if nsName.Name != "" { var lbService core.Service - if err := rm.K8sClient.Get(ctx, nsName, &lbService); err != nil { - getLBStatusErr := fmt.Errorf("error getting LoadBalancer service: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting LoadBalancer service in namespace %q, error: %s\n", - nsName, - err, - ) - - return "", getLBStatusErr + if err := rm.Get(ctx, nsName, &lbService); err != nil { + return "", fmt.Errorf("error getting LoadBalancer service: %w", err) } if lbService.Status.LoadBalancer.Ingress[0].IP != "" { address = lbService.Status.LoadBalancer.Ingress[0].IP @@ -665,7 +650,7 @@ func (rm *ResourceManager) waitForLBStatusToBeReady(ctx context.Context, svcNsNa true, /* poll immediately */ func(ctx context.Context) (bool, error) { var svc core.Service - if err := rm.K8sClient.Get(ctx, svcNsName, &svc); err != nil { + if err := rm.Get(ctx, svcNsName, &svc); err != nil { return false, err } if len(svc.Status.LoadBalancer.Ingress) > 0 { @@ -679,19 +664,14 @@ func (rm *ResourceManager) waitForLBStatusToBeReady(ctx context.Context, svcNsNa // GetClusterInfo retrieves node info and Kubernetes version from the cluster. func (rm *ResourceManager) GetClusterInfo() (ClusterInfo, error) { - GinkgoWriter.Printf("Getting cluster info\n") + GinkgoWriter.Printf("Getting cluster info|nodes\n") ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) defer cancel() var nodes core.NodeList ci := &ClusterInfo{} - if err := rm.K8sClient.List(ctx, &nodes); err != nil { - getNodesErr := fmt.Errorf("error getting nodes: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting nodes in cluster, error: %s\n", - getNodesErr, - ) - - return *ci, getNodesErr + if err := rm.List(ctx, &nodes); err != nil { + return *ci, fmt.Errorf("error getting nodes: %w", err) } ci.NodeCount = len(nodes.Items) @@ -712,13 +692,8 @@ func (rm *ResourceManager) GetClusterInfo() (ClusterInfo, error) { var ns core.Namespace key := types.NamespacedName{Name: "kube-system"} - if err := rm.K8sClient.Get(ctx, key, &ns); err != nil { - getK8sNamespaceErr := fmt.Errorf("error getting kube-system namespace: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting kube-system namespace, error: %s\n", - getK8sNamespaceErr, - ) - - return *ci, getK8sNamespaceErr + if err := rm.Get(ctx, key, &ns); err != nil { + return *ci, fmt.Errorf("error getting kube-system namespace: %w", err) } ci.ID = string(ns.UID) @@ -733,19 +708,13 @@ func (rm *ResourceManager) GetPodNames(namespace string, labels client.MatchingL defer cancel() var podList core.PodList - if err := rm.K8sClient.List( + if err := rm.List( ctx, &podList, client.InNamespace(namespace), labels, ); err != nil { - getPodsErr := fmt.Errorf("error getting list of Pods: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting list of Pods in namespace %q, error: %s\n", - namespace, - getPodsErr, - ) - - return nil, getPodsErr + return nil, fmt.Errorf("error getting list of Pods: %w", err) } names := make([]string, 0, len(podList.Items)) @@ -765,19 +734,13 @@ func (rm *ResourceManager) GetPods(namespace string, labels client.MatchingLabel defer cancel() var podList core.PodList - if err := rm.K8sClient.List( + if err := rm.List( ctx, &podList, client.InNamespace(namespace), labels, ); err != nil { - getPodsErr := fmt.Errorf("error getting list of Pods: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting list of Pods in namespace %q, error: %s\n", - namespace, - getPodsErr, - ) - - return nil, getPodsErr + return nil, fmt.Errorf("error getting list of Pods: %w", err) } GinkgoWriter.Printf("Found %d pods in namespace %q\n", len(podList.Items), namespace) @@ -791,15 +754,8 @@ func (rm *ResourceManager) GetPod(namespace, name string) (*core.Pod, error) { defer cancel() var pod core.Pod - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &pod); err != nil { - getPodErr := fmt.Errorf("error getting Pod: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting Pod %q in namespace %q, error: %s\n", - name, - namespace, - getPodErr, - ) - - return nil, getPodErr + if err := rm.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &pod); err != nil { + return nil, fmt.Errorf("error getting Pod: %w", err) } GinkgoWriter.Printf("Found pod %q in namespace %q\n", name, namespace) @@ -849,7 +805,7 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app var deployments apps.DeploymentList - if err := rm.K8sClient.List( + if err := rm.List( ctx, &deployments, client.InNamespace(namespace), @@ -857,14 +813,7 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app "app.kubernetes.io/instance": releaseName, }, ); err != nil { - getDeploymentsErr := fmt.Errorf("error getting list of Deployments: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting list of Deployments in namespace %q with release %q, error: %s\n", - namespace, - releaseName, - getDeploymentsErr, - ) - - return nil, getDeploymentsErr + return nil, fmt.Errorf("error getting list of Deployments: %w", err) } if len(deployments.Items) != 1 { @@ -899,13 +848,7 @@ func (rm *ResourceManager) getGatewayClassNginxProxy( var proxy ngfAPIv1alpha2.NginxProxy proxyName := releaseName + "-proxy-config" - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: proxyName}, &proxy); err != nil { - GinkgoWriter.Printf("ERROR occurred during getting NginxProxy %q in namespace %q, error: %s\n", - proxyName, - namespace, - err, - ) - + if err := rm.Get(ctx, types.NamespacedName{Namespace: namespace, Name: proxyName}, &proxy); err != nil { return nil, err } GinkgoWriter.Printf("Successfully found NginxProxy %q in namespace %q\n", proxyName, namespace) @@ -939,15 +882,8 @@ func (rm *ResourceManager) ScaleNginxDeployment(namespace, releaseName string, r proxy.Spec.Kubernetes.Deployment.Replicas = &replicas - if err = rm.K8sClient.Update(ctx, proxy); err != nil { - updateNginxProxyErr := fmt.Errorf("error updating NginxProxy: %w", err) - GinkgoWriter.Printf("ERROR occurred during updating NginxProxy in namespace %q with release name %q, error: %s\n", - namespace, - releaseName, - updateNginxProxyErr, - ) - - return updateNginxProxyErr + if err = rm.Update(ctx, proxy, nil); err != nil { + return fmt.Errorf("error updating NginxProxy: %w", err) } GinkgoWriter.Printf("Successfully scaled Nginx Deployment in namespace %q with release name %q to %d replicas\n", @@ -966,18 +902,12 @@ func (rm *ResourceManager) GetEvents(namespace string) (*core.EventList, error) defer cancel() var eventList core.EventList - if err := rm.K8sClient.List( + if err := rm.List( ctx, &eventList, client.InNamespace(namespace), ); err != nil { - getEventsListErr := fmt.Errorf("error getting list of Events: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting Events in namespace %q, error: %s\n", - namespace, - getEventsListErr, - ) - - return &core.EventList{}, getEventsListErr + return &core.EventList{}, fmt.Errorf("error getting list of Events: %w", err) } GinkgoWriter.Printf("Successfully found %d Events in namespace %q\n", len(eventList.Items), namespace) @@ -991,27 +921,13 @@ func (rm *ResourceManager) ScaleDeployment(namespace, name string, replicas int3 defer cancel() var deployment apps.Deployment - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &deployment); err != nil { - getDeploymentErr := fmt.Errorf("error getting Deployment: %w", err) - GinkgoWriter.Printf("ERROR occurred during getting Deployment in namespace %q with name %q, error: %s\n", - namespace, - name, - getDeploymentErr, - ) - - return getDeploymentErr + if err := rm.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &deployment); err != nil { + return fmt.Errorf("error getting Deployment: %w", err) } deployment.Spec.Replicas = &replicas - if err := rm.K8sClient.Update(ctx, &deployment); err != nil { - updateDeploymentErr := fmt.Errorf("error updating Deployment: %w", err) - GinkgoWriter.Printf("ERROR occurred during updating Deployment in namespace %q with name %q, error: %s\n", - namespace, - name, - updateDeploymentErr, - ) - - return updateDeploymentErr + if err := rm.Update(ctx, &deployment, nil); err != nil { + return fmt.Errorf("error updating Deployment: %w", err) } GinkgoWriter.Printf("Successfully scaled Deployment %q in namespace %q to %d replicas\n", name, namespace, replicas) @@ -1019,13 +935,16 @@ func (rm *ResourceManager) ScaleDeployment(namespace, name string, replicas int3 } // GetReadyNGFPodNames returns the name(s) of the NGF Pod(s). -func GetReadyNGFPodNames( - k8sClient client.Client, +func (rm *ResourceManager) GetReadyNGFPodNames( namespace, releaseName string, timeout time.Duration, + opts ...Option, ) ([]string, error) { - GinkgoWriter.Printf("Getting ready NGF Pod names in namespace %q with release name %q\n", namespace, releaseName) + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Getting ready NGF Pod names in namespace %q with release name %q\n", namespace, releaseName) + } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -1037,7 +956,7 @@ func GetReadyNGFPodNames( true, // poll immediately func(ctx context.Context) (bool, error) { var podList core.PodList - if err := k8sClient.List( + if err := rm.List( ctx, &podList, client.InNamespace(namespace), @@ -1048,38 +967,45 @@ func GetReadyNGFPodNames( return false, fmt.Errorf("error getting list of NGF Pods: %w", err) } - ngfPodNames = getReadyPodNames(podList) + ngfPodNames = getReadyPodNames(podList, opts...) return len(ngfPodNames) > 0, nil }, ) if err != nil { waitingPodsErr := fmt.Errorf("timed out waiting for NGF Pods to be ready: %w", err) + if options.logEnabled { + GinkgoWriter.Printf( + "ERROR occurred during waiting for NGF Pods to be ready in namespace %q with release name %q, error: %s\n", + namespace, + releaseName, + waitingPodsErr, + ) + } + + return nil, waitingPodsErr + } + if options.logEnabled { GinkgoWriter.Printf( - "ERROR occurred during waiting for NGF Pods to be ready in namespace %q with release name %q, error: %s\n", + "Successfully found ready NGF Pod names in namespace %q with release name %q: %v\n", namespace, releaseName, - waitingPodsErr, + ngfPodNames, ) - - return nil, waitingPodsErr } - GinkgoWriter.Printf( - "Successfully found ready NGF Pod names in namespace %q with release name %q: %v\n", - namespace, - releaseName, - ngfPodNames, - ) return ngfPodNames, nil } // GetReadyNginxPodNames returns the name(s) of the NGINX Pod(s). -func GetReadyNginxPodNames( - k8sClient client.Client, +func (rm *ResourceManager) GetReadyNginxPodNames( namespace string, timeout time.Duration, + opts ...Option, ) ([]string, error) { - GinkgoWriter.Printf("Getting ready NGINX Pod names in namespace %q\n", namespace) + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Getting ready NGINX Pod names in namespace %q\n", namespace) + } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -1091,7 +1017,7 @@ func GetReadyNginxPodNames( true, // poll immediately func(ctx context.Context) (bool, error) { var podList core.PodList - if err := k8sClient.List( + if err := rm.List( ctx, &podList, client.InNamespace(namespace), @@ -1100,29 +1026,33 @@ func GetReadyNginxPodNames( return false, fmt.Errorf("error getting list of NGINX Pods: %w", err) } - nginxPodNames = getReadyPodNames(podList) + nginxPodNames = getReadyPodNames(podList, opts...) return len(nginxPodNames) > 0, nil }, ) if err != nil { waitingPodsErr := fmt.Errorf("timed out waiting for NGINX Pods to be ready: %w", err) - GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n", - namespace, - waitingPodsErr, - ) + if options.logEnabled { + GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n", + namespace, + waitingPodsErr, + ) + } return nil, waitingPodsErr } - GinkgoWriter.Printf( - "Successfully found ready NGINX Pod names in namespace %q: %v\n", - namespace, - nginxPodNames, - ) + if options.logEnabled { + GinkgoWriter.Printf( + "Successfully found ready NGINX Pod name(s) in namespace %q: %v\n", + namespace, + nginxPodNames, + ) + } return nginxPodNames, nil } -func getReadyPodNames(podList core.PodList) []string { +func getReadyPodNames(podList core.PodList, opts ...Option) []string { var names []string for _, pod := range podList.Items { for _, cond := range pod.Status.Conditions { @@ -1131,7 +1061,10 @@ func getReadyPodNames(podList core.PodList) []string { } } } - GinkgoWriter.Printf("Found %d ready pod names: %v\n", len(names), names) + options := LogOptions(opts...) + if options.logEnabled { + GinkgoWriter.Printf("Found %d ready pod name(s): %v\n", len(names), names) + } return names } @@ -1153,15 +1086,26 @@ func countNumberOfReadyParents(parents []v1.RouteParentStatus) int { // WaitForPodsToBeReadyWithCount waits for all Pods in the specified namespace to be ready or // until the provided context is canceled. -func (rm *ResourceManager) WaitForPodsToBeReadyWithCount(ctx context.Context, namespace string, count int) error { +func (rm *ResourceManager) WaitForPodsToBeReadyWithCount( + ctx context.Context, + namespace string, + count int, + opts ...Option, +) error { + options := LogOptions(opts...) GinkgoWriter.Printf("Waiting for %d pods to be ready in namespace %q\n", count, namespace) + return wait.PollUntilContextCancel( ctx, 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { var podList core.PodList - if err := rm.K8sClient.List(ctx, &podList, client.InNamespace(namespace)); err != nil { + if err := rm.List( + ctx, + &podList, + client.InNamespace(namespace), + ); err != nil { return false, err } @@ -1173,7 +1117,9 @@ func (rm *ResourceManager) WaitForPodsToBeReadyWithCount(ctx context.Context, na } } } - GinkgoWriter.Printf("Found %d/%d ready pods in namespace %q\n", podsReady, count, namespace) + if options.logEnabled { + GinkgoWriter.Printf("Found %d/%d ready pods in namespace %q\n", podsReady, count, namespace) + } return podsReady == count, nil }, @@ -1199,7 +1145,7 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration( func(ctx context.Context) (bool, error) { var gw v1.Gateway key := types.NamespacedName{Namespace: namespace, Name: name} - if err := rm.K8sClient.Get(ctx, key, &gw); err != nil { + if err := rm.Get(ctx, key, &gw); err != nil { return false, err } @@ -1216,8 +1162,15 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration( // GetNginxConfig uses crossplane to get the nginx configuration and convert it to JSON. // If the crossplane image is loaded locally on the node, crossplaneImageRepo can be empty. -func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneImageRepo string) (*Payload, error) { +func (rm *ResourceManager) GetNginxConfig( + nginxPodName, + namespace, + crossplaneImageRepo string, + opts ...Option, +) (*Payload, error) { GinkgoWriter.Printf("Getting NGINX config from pod %q in namespace %q\n", nginxPodName, namespace) + options := LogOptions(opts...) + if err := injectCrossplaneContainer( rm.ClientGoClient, rm.TimeoutConfig.UpdateTimeout, @@ -1263,10 +1216,12 @@ func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneIma }, ); err != nil { containerErr := fmt.Errorf("could not connect to ephemeral container: %w", err) - GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n", - namespace, - containerErr, - ) + if options.logEnabled { + GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n", + namespace, + containerErr, + ) + } return nil, containerErr } @@ -1286,3 +1241,94 @@ func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneIma return conf, nil } + +// Get retrieves a resource by key, logging errors if enabled. +func (rm *ResourceManager) Get( + ctx context.Context, + key client.ObjectKey, + obj client.Object, + opts ...Option, +) error { + options := LogOptions(opts...) + if err := rm.K8sClient.Get(ctx, key, obj); err != nil { + if options.logEnabled { + GinkgoWriter.Printf("Could not get k8s resource %q error: %v\n", obj.GetName(), err) + } + + return err + } + + return nil +} + +// Create adds a new resource, returning an error on failure. +func (rm *ResourceManager) Create( + ctx context.Context, + obj client.Object, +) error { + if err := rm.K8sClient.Create(ctx, obj); err != nil { + createErr := fmt.Errorf("error creating k8s resource %q: %w", obj.GetName(), err) + GinkgoWriter.Printf("%v\n", createErr) + + return createErr + } + return nil +} + +// Delete removes a resource, returning an error on failure. +func (rm *ResourceManager) Delete( + ctx context.Context, + obj client.Object, + deleteOpts []client.DeleteOption, + opts ...Option, +) error { + options := LogOptions(opts...) + if err := rm.K8sClient.Delete(ctx, obj, deleteOpts...); err != nil { + if options.logEnabled { + GinkgoWriter.Printf("Could not delete k8s resource %q: %w\n", obj.GetName(), err) + } + + return err + } + return nil +} + +// Update modifies a resource. +func (rm *ResourceManager) Update( + ctx context.Context, + obj client.Object, + updateOpts []client.UpdateOption, + opts ...Option, +) error { + options := LogOptions(opts...) + if err := rm.K8sClient.Update(ctx, obj, updateOpts...); err != nil { + updateResourceErr := fmt.Errorf("error updating k8s resource: %w", err) + if options.logEnabled { + GinkgoWriter.Printf( + "ERROR occurred during updating k8s resource in namespace %q with name %q, error: %s\n", + obj.GetNamespace(), + obj.GetName(), + updateResourceErr, + ) + } + + return updateResourceErr + } + + return nil +} + +// List retrieves a list of resources, returning an error on failure. +func (rm *ResourceManager) List( + ctx context.Context, + list client.ObjectList, + listOpts ...client.ListOption, +) error { + if err := rm.K8sClient.List(ctx, list, listOpts...); err != nil { + listErr := fmt.Errorf("error listing k8s resources: %w", err) + GinkgoWriter.Printf("%v\n", listErr) + + return listErr + } + return nil +} diff --git a/tests/framework/results.go b/tests/framework/results.go index 120dbcc3a5..87bba0a0b4 100644 --- a/tests/framework/results.go +++ b/tests/framework/results.go @@ -9,6 +9,7 @@ import ( "os/exec" "path/filepath" + . "github.com/onsi/ginkgo/v2" vegeta "github.com/tsenart/vegeta/v12/lib" ) @@ -23,7 +24,10 @@ func CreateResultsDir(testName, version string) (string, error) { if _, err := os.Stat(dirName); err == nil { if err := os.RemoveAll(dirName); err != nil { - return "", fmt.Errorf("failed to remove existing directory %s: %w", dirName, err) + rmDirErr := fmt.Errorf("failed to remove existing directory %s: %w", dirName, err) + GinkgoWriter.Printf("ERROR occurred during removing existing results directory %q, error: %s\n", dirName, rmDirErr) + + return "", rmDirErr } } @@ -34,6 +38,8 @@ func CreateResultsDir(testName, version string) (string, error) { func CreateResultsFile(filename string) (*os.File, error) { outFile, err := os.OpenFile(filename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0o644) if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating results file %q, error: %s\n", filename, err) + return nil, err } @@ -65,13 +71,19 @@ func WriteSystemInfoToFile(file *os.File, ci ClusterInfo, plus bool) error { plus, commit, date, dirty, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode, ) if _, err := fmt.Fprint(file, text); err != nil { + GinkgoWriter.Printf("ERROR occurred during writing system info to results file, error: %s\n", err) + return err } if ci.IsGKE { if _, err := fmt.Fprintf(file, "- Zone: %s\n- Instance Type: %s\n", ci.GkeZone, ci.GkeInstanceType); err != nil { + GinkgoWriter.Printf("ERROR occurred during writing GKE info to results file, error: %s\n", err) + return err } } + GinkgoWriter.Printf("Wrote system info to results file\n") + return nil } @@ -89,6 +101,13 @@ func generatePNG(resultsDir, inputFilename, outputFilename, configFilename strin output, err := cmd.CombinedOutput() if err != nil { + GinkgoWriter.Printf( + "ERROR occurred during generating PNG %q using gnuplot, error: %s, output: %s\n", + outputFilename, + err, + string(output), + ) + return fmt.Errorf("failed to generate PNG: %w; output: %s", err, string(output)) } @@ -118,13 +137,20 @@ func GenerateMemoryPNG(resultsDir, inputFilename, outputFilename string) error { // WriteMetricsResults writes the metrics results to the results file in text format. func WriteMetricsResults(resultsFile *os.File, metrics *Metrics) error { reporter := vegeta.NewTextReporter(&metrics.Metrics) + reporterErr := reporter.Report(resultsFile) + if reporterErr != nil { + GinkgoWriter.Printf("ERROR occurred during writing metrics results to results file, error: %s\n", reporterErr) + } + GinkgoWriter.Printf("Wrote metrics results to results file %q\n", resultsFile.Name()) - return reporter.Report(resultsFile) + return reporterErr } // WriteContent writes basic content to the results file. func WriteContent(resultsFile *os.File, content string) error { if _, err := fmt.Fprintln(resultsFile, content); err != nil { + GinkgoWriter.Printf("ERROR occurred during writing content to results file, error: %s\n", err) + return err } diff --git a/tests/suite/advanced_routing_test.go b/tests/suite/advanced_routing_test.go index 1163089cc5..944a9f0832 100644 --- a/tests/suite/advanced_routing_test.go +++ b/tests/suite/advanced_routing_test.go @@ -40,7 +40,10 @@ var _ = Describe("AdvancedRouting", Ordered, Label("functional", "routing"), fun Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) diff --git a/tests/suite/client_settings_test.go b/tests/suite/client_settings_test.go index 53762e58bc..e02283ea99 100644 --- a/tests/suite/client_settings_test.go +++ b/tests/suite/client_settings_test.go @@ -47,7 +47,10 @@ var _ = Describe("ClientSettingsPolicy", Ordered, Label("functional", "cspolicy" Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -393,7 +396,7 @@ func waitForClientSettingsAncestorStatus( func(ctx context.Context) (bool, error) { var pol ngfAPI.ClientSettingsPolicy - if err := k8sClient.Get(ctx, policyNsname, &pol); err != nil { + if err := resourceManager.Get(ctx, policyNsname, &pol); err != nil { return false, err } diff --git a/tests/suite/dataplane_perf_test.go b/tests/suite/dataplane_perf_test.go index cce009ab03..45bd8ef295 100644 --- a/tests/suite/dataplane_perf_test.go +++ b/tests/suite/dataplane_perf_test.go @@ -66,7 +66,10 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"), Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) diff --git a/tests/suite/graceful_recovery_test.go b/tests/suite/graceful_recovery_test.go index 01cfed2652..a5d3348958 100644 --- a/tests/suite/graceful_recovery_test.go +++ b/tests/suite/graceful_recovery_test.go @@ -74,7 +74,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra defer cancel() var pod core.Pod - if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: podName}, &pod); err != nil { + if err := resourceManager.Get( + ctx, + types.NamespacedName{Namespace: namespace, Name: podName}, + &pod, + ); err != nil { return 0, fmt.Errorf("error retrieving Pod: %w", err) } @@ -95,8 +99,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra } if restartCount != currentRestartCount+1 { - return fmt.Errorf("expected current restart count: %d to match incremented restart count: %d", + restartErr := fmt.Errorf("expected current restart count: %d to match incremented restart count: %d", restartCount, currentRestartCount+1) + GinkgoWriter.Printf("%s\n", restartErr) + + return restartErr } return nil @@ -107,7 +114,7 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra defer cancel() var nodes core.NodeList - if err := k8sClient.List(ctx, &nodes); err != nil { + if err := resourceManager.List(ctx, &nodes); err != nil { return nil, fmt.Errorf("error listing nodes: %w", err) } @@ -125,26 +132,39 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra defer cancel() var nginxPod core.Pod - if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: nginxPodName}, &nginxPod); err != nil { + if err := resourceManager.Get( + ctx, + types.NamespacedName{Namespace: ns.Name, Name: nginxPodName}, + &nginxPod, + ); err != nil { return nil, fmt.Errorf("error retrieving nginx Pod: %w", err) } b, err := resourceManager.GetFileContents("graceful-recovery/node-debugger-job.yaml") if err != nil { - return nil, fmt.Errorf("error processing node debugger job file: %w", err) + debugErr := fmt.Errorf("error processing node debugger job file: %w", err) + GinkgoWriter.Printf("%s\n", debugErr) + + return nil, debugErr } job := &v1.Job{} if err = yaml.Unmarshal(b.Bytes(), job); err != nil { - return nil, fmt.Errorf("error with yaml unmarshal: %w", err) + yamlErr := fmt.Errorf("error with yaml unmarshal: %w", err) + GinkgoWriter.Printf("%s\n", yamlErr) + + return nil, yamlErr } job.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"] = nginxPod.Spec.NodeName if len(job.Spec.Template.Spec.Containers) != 1 { - return nil, fmt.Errorf( + containerErr := fmt.Errorf( "expected node debugger job to contain one container, actual number: %d", len(job.Spec.Template.Spec.Containers), ) + GinkgoWriter.Printf("ERROR: %s\n", containerErr) + + return nil, containerErr } job.Namespace = ns.Name @@ -174,7 +194,7 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra // default propagation policy is metav1.DeletePropagationOrphan which does not delete the underlying // pod created through the job after the job is deleted. Setting it to metav1.DeletePropagationBackground // deletes the underlying pod after the job is deleted. - Expect(resourceManager.Delete( + Expect(resourceManager.DeleteResources( []client.Object{job}, client.PropagationPolicy(metav1.DeletePropagationBackground), )).To(Succeed()) @@ -207,7 +227,10 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -281,8 +304,7 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra var podNames []string Eventually( func() bool { - podNames, err = framework.GetReadyNGFPodNames( - k8sClient, + podNames, err = resourceManager.GetReadyNGFPodNames( ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout, @@ -307,7 +329,10 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra var nginxPodNames []string Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout * 2). @@ -352,12 +377,15 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra var lease coordination.Lease key := types.NamespacedName{Name: "ngf-test-nginx-gateway-fabric-leader-election", Namespace: ngfNamespace} - if err := k8sClient.Get(ctx, key, &lease); err != nil { + if err := resourceManager.Get(ctx, key, &lease); err != nil { return "", errors.New("could not retrieve leader election lease") } if *lease.Spec.HolderIdentity == "" { - return "", errors.New("leader election lease holder identity is empty") + leaderErr := errors.New("leader election lease holder identity is empty") + GinkgoWriter.Printf("ERROR: %s\n", leaderErr) + + return "", leaderErr } return *lease.Spec.HolderIdentity, nil @@ -381,7 +409,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra } BeforeAll(func() { - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout) + podNames, err := resourceManager.GetReadyNGFPodNames( + ngfNamespace, + releaseName, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) @@ -397,7 +429,10 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -431,7 +466,10 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra It("recovers when nginx container is restarted", func() { restartNginxContainer(activeNginxPodName, ns.Name, nginxContainerName) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) activeNginxPodName = nginxPodNames[0] @@ -460,13 +498,12 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout) defer cancel() - Expect(k8sClient.Delete(ctx, ngfPod)).To(Succeed()) + Expect(resourceManager.Delete(ctx, ngfPod, nil)).To(Succeed()) var newNGFPodNames []string Eventually( func() bool { - newNGFPodNames, err = framework.GetReadyNGFPodNames( - k8sClient, + newNGFPodNames, err = resourceManager.GetReadyNGFPodNames( ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout, diff --git a/tests/suite/longevity_test.go b/tests/suite/longevity_test.go index 8734ffa5bc..6d9cc2e37e 100644 --- a/tests/suite/longevity_test.go +++ b/tests/suite/longevity_test.go @@ -56,7 +56,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.ApplyFromFiles(promFile, ngfNamespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) }) It("collects results", Label("longevity-teardown"), func() { diff --git a/tests/suite/nginxgateway_test.go b/tests/suite/nginxgateway_test.go index 3518c2ed69..4813661964 100644 --- a/tests/suite/nginxgateway_test.go +++ b/tests/suite/nginxgateway_test.go @@ -13,7 +13,6 @@ import ( "k8s.io/apimachinery/pkg/types" ngfAPI "github.com/nginx/nginx-gateway-fabric/v2/apis/v1alpha1" - "github.com/nginx/nginx-gateway-fabric/v2/tests/framework" ) var _ = Describe("NginxGateway", Ordered, Label("functional", "nginxGateway"), func() { @@ -34,15 +33,8 @@ var _ = Describe("NginxGateway", Ordered, Label("functional", "nginxGateway"), f var nginxGateway ngfAPI.NginxGateway - if err := k8sClient.Get(ctx, nsname, &nginxGateway); err != nil { - gatewayErr := fmt.Errorf("failed to get nginxGateway: %w", err) - GinkgoWriter.Printf( - "ERROR occurred during getting NGINX Gateway in namespace %q: %v\n", - nsname.Namespace, - gatewayErr, - ) - - return nginxGateway, gatewayErr + if err := resourceManager.Get(ctx, nsname, &nginxGateway); err != nil { + return nginxGateway, fmt.Errorf("failed to get nginxGateway: %w", err) } return nginxGateway, nil @@ -115,8 +107,7 @@ var _ = Describe("NginxGateway", Ordered, Label("functional", "nginxGateway"), f } getNGFPodName := func() (string, error) { - podNames, err := framework.GetReadyNGFPodNames( - k8sClient, + podNames, err := resourceManager.GetReadyNGFPodNames( ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout, diff --git a/tests/suite/reconfig_test.go b/tests/suite/reconfig_test.go index ab990e1808..51816d0736 100644 --- a/tests/suite/reconfig_test.go +++ b/tests/suite/reconfig_test.go @@ -64,7 +64,6 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r Expect(err).ToNot(HaveOccurred()) k8sConfig := ctlr.GetConfigOrDie() - if !clusterInfo.IsGKE { Expect(promInstance.PortForward(k8sConfig, promPortForwardStopCh)).To(Succeed()) } @@ -85,6 +84,8 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r }) createUniqueResources := func(resourceCount int, fileName string) error { + GinkgoWriter.Printf("Creating %d unique resources from %s\n", resourceCount, fileName) + var appliedResources []string for i := 1; i <= resourceCount; i++ { namespace := "namespace" + strconv.Itoa(i) @@ -98,11 +99,29 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r fileString = strings.ReplaceAll(fileString, "tea", "tea"+namespace) data := bytes.NewBufferString(fileString) - - if err := resourceManager.ApplyFromBuffer(data, namespace); err != nil { - return fmt.Errorf("error processing manifest file: %w", err) + appliedResources = append(appliedResources, namespace) + + if err := resourceManager.ApplyFromBuffer( + data, + namespace, + framework.WithLoggingDisabled(), // disable logging to avoid huge logs for 150 resources + ); err != nil { + manifestErr := fmt.Errorf("error processing manifest file: %w", err) + GinkgoWriter.Printf( + "ERROR on creating and applying unique resources, could proceed %v\n the error happened on %q: %v\n", + appliedResources, + namespace, + manifestErr, + ) + return manifestErr } } + GinkgoWriter.Printf( + "Successfully created %d unique resources from %s: %v\n", + resourceCount, + fileName, + appliedResources, + ) return nil } @@ -117,7 +136,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r Name: "namespace" + strconv.Itoa(i), }, } - Expect(k8sClient.Create(ctx, &ns)).To(Succeed()) + Expect(resourceManager.Create(ctx, &ns)).To(Succeed()) } Expect(resourceManager.Apply([]client.Object{&reconfigNamespace})).To(Succeed()) @@ -126,7 +145,8 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r "reconfig/cafe-secret.yaml", "reconfig/reference-grant.yaml", }, - reconfigNamespace.Name)).To(Succeed()) + reconfigNamespace.Name, + )).To(Succeed()) Expect(createUniqueResources(resourceCount, "manifests/reconfig/cafe.yaml")).To(Succeed()) @@ -147,21 +167,24 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r defer cancel() var namespaces core.NamespaceList - if err := k8sClient.List(ctx, &namespaces); err != nil { + if err := resourceManager.List(ctx, &namespaces); err != nil { return fmt.Errorf("error getting namespaces: %w", err) } + GinkgoWriter.Printf("Found %d namespaces, expected at least%d\n", len(namespaces.Items), resourceCount) Expect(len(namespaces.Items)).To(BeNumerically(">=", resourceCount)) var routes v1.HTTPRouteList - if err := k8sClient.List(ctx, &routes); err != nil { + if err := resourceManager.List(ctx, &routes); err != nil { return fmt.Errorf("error getting HTTPRoutes: %w", err) } + GinkgoWriter.Printf("Found %d HTTPRoutes, expected %d\n", len(routes.Items), resourceCount*3) Expect(routes.Items).To(HaveLen(resourceCount * 3)) var pods core.PodList - if err := k8sClient.List(ctx, &pods); err != nil { + if err := resourceManager.List(ctx, &pods); err != nil { return fmt.Errorf("error getting Pods: %w", err) } + GinkgoWriter.Printf("Found %d Pods, expected at least %d\n", len(pods.Items), resourceCount*2) Expect(len(pods.Items)).To(BeNumerically(">=", resourceCount*2)) return nil @@ -175,7 +198,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r namespaces[i] = "namespace" + strconv.Itoa(i+1) } - err = resourceManager.DeleteNamespaces(namespaces) + err = resourceManager.DeleteNamespaces(namespaces, framework.WithLoggingDisabled()) Expect(resourceManager.DeleteNamespace(reconfigNamespace.Name)).To(Succeed()) return err @@ -196,7 +219,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r } // each call to ValidateNginxFieldExists takes about 1ms - if err := framework.ValidateNginxFieldExists(conf, expUpstream); err != nil { + if err := framework.ValidateNginxFieldExists(conf, expUpstream, framework.WithLoggingDisabled()); err != nil { select { case <-ctx.Done(): return fmt.Errorf("error validating nginx conf was generated in "+namespace+": %w", err.Error()) @@ -222,6 +245,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r if stringTimeToReadyTotal == "0" { stringTimeToReadyTotal = "< 1" } + GinkgoWriter.Printf("Calculated time to ready total for %q: %s\n", nginxPodName, stringTimeToReadyTotal) return stringTimeToReadyTotal } @@ -276,6 +300,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r q, getEndTime, noOpModifier, + framework.WithLoggingDisabled(), ), ).WithTimeout(metricExistTimeout).WithPolling(metricExistPolling).Should(Succeed()) } @@ -314,7 +339,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r cfg.nfr = true setup(cfg) - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + podNames, err := resourceManager.GetReadyNGFPodNames(ngfNamespace, releaseName, timeoutConfig.GetTimeout) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) ngfPodName := podNames[0] @@ -323,10 +348,10 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r var nginxPodNames []string Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames( - k8sClient, + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( reconfigNamespace.Name, timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -371,21 +396,26 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r cfg.nfr = true setup(cfg) - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + podNames, err := resourceManager.GetReadyNGFPodNames( + ngfNamespace, + releaseName, + timeoutConfig.GetTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) ngfPodName := podNames[0] - Expect(resourceManager.Apply([]client.Object{&reconfigNamespace})).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{&reconfigNamespace}, framework.WithLoggingDisabled())).To(Succeed()) Expect(resourceManager.ApplyFromFiles([]string{"reconfig/gateway.yaml"}, reconfigNamespace.Name)).To(Succeed()) var nginxPodNames []string Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames( - k8sClient, + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( reconfigNamespace.Name, timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -406,7 +436,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r File: "http.conf", } - return framework.ValidateNginxFieldExists(conf, defaultUpstream) == nil + return framework.ValidateNginxFieldExists(conf, defaultUpstream, framework.WithLoggingDisabled()) == nil }). WithTimeout(timeoutConfig.CreateTimeout). Should(BeTrue()) @@ -439,7 +469,11 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r }) AfterEach(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, reconfigNamespace.Name) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + reconfigNamespace.Name, + framework.WithLoggingDisabled(), + ) Expect(cleanupResources()).Should(Succeed()) teardown(releaseName) diff --git a/tests/suite/sample_test.go b/tests/suite/sample_test.go index a79dda5a7d..f133cc6acd 100644 --- a/tests/suite/sample_test.go +++ b/tests/suite/sample_test.go @@ -38,7 +38,10 @@ var _ = Describe("Basic test example", Label("functional"), func() { Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 526b27049c..adecea7e5c 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -114,7 +114,11 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { } Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + podNames, err := resourceManager.GetReadyNGFPodNames( + ngfNamespace, + releaseName, + timeoutConfig.GetTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) ngfPodName = podNames[0] @@ -159,6 +163,8 @@ The logs are attached only if there are errors. writeScaleResults := func(dest io.Writer, results scaleTestResults) error { tmpl, err := template.New("results").Parse(scaleResultTemplate) if err != nil { + GinkgoWriter.Printf("ERROR creating results template: %v\n", err) + return err } @@ -245,8 +251,12 @@ The logs are attached only if there are errors. test() - // We sleep for 2 scape intervals to ensure that Prometheus scrapes the metrics after the test() finishes + // We sleep for 2 scrape intervals to ensure that Prometheus scrapes the metrics after the test() finishes // before endTime, so that we don't lose any metric values like reloads. + GinkgoWriter.Printf( + "Sleeping for %v to ensure Prometheus scrapes the metrics after the test finishes\n", + 2*scrapeInterval, + ) time.Sleep(2 * scrapeInterval) endTime := time.Now() @@ -349,7 +359,10 @@ The logs are attached only if there are errors. []string{`"logger":"usageReporter`}, // ignore usageReporter errors ) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -375,10 +388,15 @@ The logs are attached only if there are errors. findRestarts := func(containerName string, pod *core.Pod) int { for _, containerStatus := range pod.Status.ContainerStatuses { if containerStatus.Name == containerName { + GinkgoWriter.Printf("INFO: container %q had %d restarts\n", containerName, containerStatus.RestartCount) + return int(containerStatus.RestartCount) } } - Fail(fmt.Sprintf("container %s not found", containerName)) + fail := fmt.Sprintf("container %s not found", containerName) + GinkgoWriter.Printf("FAIL: %v\n", fail) + + Fail(fail) return 0 } @@ -420,13 +438,19 @@ The logs are attached only if there are errors. Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) for i := range len(objects.ScaleIterationGroups) { - Expect(resourceManager.Apply(objects.ScaleIterationGroups[i])).To(Succeed()) + Expect(resourceManager.Apply( + objects.ScaleIterationGroups[i], + framework.WithLoggingDisabled(), // disable logging to avoid huge log + )).To(Succeed()) if i == 0 { var nginxPodNames []string Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -454,7 +478,7 @@ The logs are attached only if there are errors. url, address, timeoutConfig.RequestTimeout, - framework.WithLoggingDisabled(), // disable logging to avoid huge logs + framework.WithLoggingDisabled(), // disable logging to avoid huge log for 1000 requests ), ).WithTimeout(6 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) @@ -489,7 +513,10 @@ The logs are attached only if there are errors. var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -508,7 +535,11 @@ The logs are attached only if there are errors. } Eventually( - framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), + framework.CreateResponseChecker( + url, + address, + timeoutConfig.RequestTimeout, + ), ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) Expect( @@ -521,7 +552,11 @@ The logs are attached only if there are errors. Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) Eventually( - framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), + framework.CreateResponseChecker( + url, + address, + timeoutConfig.RequestTimeout, + ), ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) } @@ -639,7 +674,10 @@ The logs are attached only if there are errors. var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -702,7 +740,10 @@ The logs are attached only if there are errors. }) AfterEach(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + namespace, + ) cleanUpPortForward() Expect(resourceManager.DeleteNamespace(namespace)).To(Succeed()) teardown(releaseName) @@ -905,7 +946,10 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -921,7 +965,10 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim }) AfterAll(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + ns.Name, + ) cleanUpPortForward() teardown(releaseName) @@ -961,7 +1008,11 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.UpdateTimeout) - Expect(resourceManager.WaitForPodsToBeReadyWithCount(ctx, ns.Name, i+numCoffeeAndTeaPods)).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReadyWithCount( + ctx, + ns.Name, + i+numCoffeeAndTeaPods), + ).To(Succeed()) Expect(resourceManager.WaitForGatewayObservedGeneration(ctx, ns.Name, "gateway", i)).To(Succeed()) cancel() @@ -1037,8 +1088,11 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim var gw v1.Gateway key := types.NamespacedName{Namespace: ns.Name, Name: "gateway"} - if err := resourceManager.K8sClient.Get(ctx, key, &gw); err != nil { - return fmt.Errorf("failed to get gateway: %w", err) + if err := resourceManager.Get(ctx, key, &gw); err != nil { + gatewayErr := fmt.Errorf("failed to get gateway: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", gatewayErr) + + return gatewayErr } if len(gw.Status.Listeners) != num { diff --git a/tests/suite/snippets_filter_test.go b/tests/suite/snippets_filter_test.go index 78e0a592e8..af67646a7d 100644 --- a/tests/suite/snippets_filter_test.go +++ b/tests/suite/snippets_filter_test.go @@ -42,7 +42,10 @@ var _ = Describe("SnippetsFilter", Ordered, Label("functional", "snippets-filter Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -270,9 +273,7 @@ func checkHTTPRouteToHaveGatewayNotProgrammedCond(httpRouteNsName types.Namespac var hr v1.HTTPRoute var err error - if err = k8sClient.Get(ctx, httpRouteNsName, &hr); err != nil { - GinkgoWriter.Printf("ERROR: failed to get HTTPRoute: %v\n", err) - + if err = resourceManager.Get(ctx, httpRouteNsName, &hr); err != nil { return err } @@ -328,9 +329,7 @@ func checkForSnippetsFilterToBeAccepted(snippetsFilterNsNames types.NamespacedNa var sf ngfAPI.SnippetsFilter var err error - if err = k8sClient.Get(ctx, snippetsFilterNsNames, &sf); err != nil { - GinkgoWriter.Printf("ERROR: failed to get SnippetsFilter: %v\n", err) - + if err = resourceManager.Get(ctx, snippetsFilterNsNames, &sf); err != nil { return err } diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index 71d610bf5a..31ce49c0d0 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -124,7 +124,6 @@ func setup(cfg setupConfig, extraInstallArgs ...string) { options := client.Options{ Scheme: scheme, } - var err error k8sClient, err = client.New(k8sConfig, options) Expect(err).ToNot(HaveOccurred()) @@ -179,8 +178,7 @@ func setup(cfg setupConfig, extraInstallArgs ...string) { installCfg := createNGFInstallConfig(cfg, extraInstallArgs...) - podNames, err := framework.GetReadyNGFPodNames( - k8sClient, + podNames, err := resourceManager.GetReadyNGFPodNames( installCfg.Namespace, installCfg.ReleaseName, timeoutConfig.CreateTimeout, @@ -263,7 +261,7 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo } if *plusEnabled { - Expect(framework.CreateLicenseSecret(k8sClient, ngfNamespace, *plusLicenseFileName)).To(Succeed()) + Expect(framework.CreateLicenseSecret(resourceManager, ngfNamespace, *plusLicenseFileName)).To(Succeed()) } output, err = framework.InstallNGF(installCfg, extraInstallArgs...) @@ -278,7 +276,7 @@ func teardown(relName string) { Namespace: ngfNamespace, } - output, err := framework.UninstallNGF(cfg, k8sClient) + output, err := framework.UninstallNGF(cfg, resourceManager) Expect(err).ToNot(HaveOccurred(), string(output)) output, err = framework.UninstallGatewayAPI(*gatewayAPIVersion) @@ -293,7 +291,11 @@ func teardown(relName string) { true, /* poll immediately */ func(ctx context.Context) (bool, error) { key := k8sTypes.NamespacedName{Name: ngfNamespace} - if err := k8sClient.Get(ctx, key, &core.Namespace{}); err != nil && apierrors.IsNotFound(err) { + if err := resourceManager.Get( + ctx, + key, + &core.Namespace{}, + ); err != nil && apierrors.IsNotFound(err) { return true, nil } diff --git a/tests/suite/tracing_test.go b/tests/suite/tracing_test.go index bdc1abda54..9b31d47be0 100644 --- a/tests/suite/tracing_test.go +++ b/tests/suite/tracing_test.go @@ -52,11 +52,11 @@ var _ = Describe("Tracing", FlakeAttempts(2), Ordered, Label("functional", "trac key := types.NamespacedName{Name: "ngf-test-proxy-config", Namespace: "nginx-gateway"} var nginxProxy ngfAPIv1alpha2.NginxProxy - Expect(k8sClient.Get(ctx, key, &nginxProxy)).To(Succeed()) + Expect(resourceManager.Get(ctx, key, &nginxProxy)).To(Succeed()) nginxProxy.Spec.Telemetry = &telemetry - Expect(k8sClient.Update(ctx, &nginxProxy)).To(Succeed()) + Expect(resourceManager.Update(ctx, &nginxProxy, nil)).To(Succeed()) } BeforeAll(func() { @@ -92,7 +92,10 @@ var _ = Describe("Tracing", FlakeAttempts(2), Ordered, Label("functional", "trac Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -240,9 +243,7 @@ func verifyGatewayClassResolvedRefs() error { defer cancel() var gc gatewayv1.GatewayClass - if err := k8sClient.Get(ctx, types.NamespacedName{Name: gatewayClassName}, &gc); err != nil { - GinkgoWriter.Printf("ERROR getting GatewayClass %s: %v\n", gatewayClassName, err) - + if err := resourceManager.Get(ctx, types.NamespacedName{Name: gatewayClassName}, &gc); err != nil { return err } @@ -272,9 +273,7 @@ func verifyPolicyStatus() error { var pol ngfAPIv1alpha2.ObservabilityPolicy key := types.NamespacedName{Name: "test-observability-policy", Namespace: "helloworld"} - if err := k8sClient.Get(ctx, key, &pol); err != nil { - GinkgoWriter.Printf("ERROR getting ObservabilityPolicy %q in namespace %q: %v\n", key.Name, key.Namespace, err) - + if err := resourceManager.Get(ctx, key, &pol); err != nil { return err } diff --git a/tests/suite/upgrade_test.go b/tests/suite/upgrade_test.go index cc864ff990..1636893003 100644 --- a/tests/suite/upgrade_test.go +++ b/tests/suite/upgrade_test.go @@ -67,7 +67,10 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + ns.Name, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -83,7 +86,10 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { }) AfterEach(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + ns.Name, + ) cleanUpPortForward() Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) @@ -204,7 +210,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { Expect(resourceManager.ApplyFromFiles([]string{"ngf-upgrade/gateway-updated.yaml"}, ns.Name)).To(Succeed()) - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + podNames, err := resourceManager.GetReadyNGFPodNames(ngfNamespace, releaseName, timeoutConfig.GetTimeout) Expect(err).ToNot(HaveOccurred()) Expect(podNames).ToNot(BeEmpty()) @@ -220,7 +226,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { true, /* poll immediately */ func(_ context.Context) (bool, error) { defer GinkgoRecover() - Expect(k8sClient.Get(leaseCtx, key, &lease)).To(Succeed()) + Expect(resourceManager.Get(leaseCtx, key, &lease)).To(Succeed()) if lease.Spec.HolderIdentity != nil { for _, podName := range podNames { @@ -245,7 +251,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { - Expect(k8sClient.Get(ctx, key, &gw)).To(Succeed()) + Expect(resourceManager.Get(ctx, key, &gw)).To(Succeed()) expListenerName := "http-new" for _, listener := range gw.Status.Listeners { if listener.Name == v1.SectionName(expListenerName) { diff --git a/tests/suite/upstream_settings_test.go b/tests/suite/upstream_settings_test.go index cd70604ef0..2d53aac520 100644 --- a/tests/suite/upstream_settings_test.go +++ b/tests/suite/upstream_settings_test.go @@ -51,7 +51,10 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := resourceManager.GetReadyNginxPodNames( + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -424,7 +427,7 @@ func usPolicyHasNoAncestors(usPolicyNsName types.NamespacedName) bool { defer cancel() var usPolicy ngfAPI.UpstreamSettingsPolicy - if err := k8sClient.Get(ctx, usPolicyNsName, &usPolicy); err != nil { + if err := resourceManager.Get(ctx, usPolicyNsName, &usPolicy); err != nil { GinkgoWriter.Printf("ERROR: Failed to get UpstreamSettingsPolicy %q: %s", usPolicyNsName, err.Error()) return false } @@ -464,9 +467,7 @@ func waitForUSPolicyStatus( var usPolicy ngfAPI.UpstreamSettingsPolicy var err error - if err := k8sClient.Get(ctx, usPolicyNsName, &usPolicy); err != nil { - GinkgoWriter.Printf("ERROR: Failed to get UpstreamSettingsPolicy %q: %s", usPolicyNsName, err.Error()) - + if err := resourceManager.Get(ctx, usPolicyNsName, &usPolicy); err != nil { return false, err }