diff --git a/tests/framework/crossplane.go b/tests/framework/crossplane.go index 02a16b6cb5..0a5f4524ed 100644 --- a/tests/framework/crossplane.go +++ b/tests/framework/crossplane.go @@ -8,6 +8,7 @@ import ( "strings" "time" + . "github.com/onsi/ginkgo/v2" core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -42,10 +43,13 @@ const crossplaneImageName = "nginx-crossplane:latest" // ValidateNginxFieldExists accepts the nginx config and the configuration for the expected field, // and returns whether or not that field exists where it should. -func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) error { +func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField, opts ...Option) error { b, err := json.Marshal(conf) if err != nil { - return fmt.Errorf("error marshaling nginx config: %w", err) + marshalErr := fmt.Errorf("error marshaling nginx config: %w", err) + GinkgoWriter.Printf("%v\n", marshalErr) + + return marshalErr } for _, config := range conf.Config { @@ -55,7 +59,7 @@ func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) err for _, directive := range config.Parsed { if expFieldCfg.Server == "" && expFieldCfg.Upstream == "" { - if expFieldCfg.fieldFound(directive) { + if expFieldCfg.fieldFound(directive, opts...) { return nil } continue @@ -65,13 +69,15 @@ func ValidateNginxFieldExists(conf *Payload, expFieldCfg ExpectedNginxField) err return nil } - if expFieldCfg.Upstream != "" && fieldExistsInUpstream(expFieldCfg, *directive) { + if expFieldCfg.Upstream != "" && fieldExistsInUpstream(expFieldCfg, *directive, opts...) { return nil } } } + directiveErr := fmt.Errorf("directive %s not found in: nginx config %s", expFieldCfg.Directive, string(b)) + GinkgoWriter.Printf("ERROR: %v\n", directiveErr) - return fmt.Errorf("directive %s not found in: nginx config %s", expFieldCfg.Directive, string(b)) + return directiveErr } func fieldExistsInServer( @@ -94,7 +100,19 @@ func fieldExistsInServer( func fieldExistsInUpstream( expFieldCfg ExpectedNginxField, directive Directive, + opts ...Option, ) bool { + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + if options.logEnabled { + GinkgoWriter.Printf( + "Checking upstream for directive %q with value %q\n", + expFieldCfg.Directive, + expFieldCfg.Value, + ) + } if directive.Directive == "upstream" && directive.Args[0] == expFieldCfg.Upstream { for _, directive := range directive.Block { if expFieldCfg.fieldFound(directive) { @@ -115,7 +133,11 @@ func getServerName(serverBlock Directives) string { return "" } -func (e ExpectedNginxField) fieldFound(directive *Directive) bool { +func (e ExpectedNginxField) fieldFound(directive *Directive, opts ...Option) bool { + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } arg := strings.Join(directive.Args, " ") valueMatch := arg == e.Value @@ -123,7 +145,20 @@ func (e ExpectedNginxField) fieldFound(directive *Directive) bool { valueMatch = strings.Contains(arg, e.Value) } - return directive.Directive == e.Directive && valueMatch + if directive.Directive == e.Directive && valueMatch { + if options.logEnabled { + GinkgoWriter.Printf( + "Found field %q with value %q in field %q with value %q\n", + e.Directive, + e.Value, + directive.Directive, + arg, + ) + } + return true + } + + return false } func fieldExistsInLocation(locationDirective *Directive, expFieldCfg ExpectedNginxField) bool { @@ -201,7 +236,10 @@ func injectCrossplaneContainer( podClient := k8sClient.CoreV1().Pods(namespace) if _, err := podClient.UpdateEphemeralContainers(ctx, ngfPodName, pod, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("error adding ephemeral container: %w", err) + containerErr := fmt.Errorf("error adding ephemeral container: %w", err) + GinkgoWriter.Printf("%v\n", containerErr) + + return containerErr } return nil @@ -231,7 +269,10 @@ func createCrossplaneExecutor( exec, err := remotecommand.NewSPDYExecutor(k8sConfig, http.MethodPost, req.URL()) if err != nil { - return nil, fmt.Errorf("error creating executor: %w", err) + executorErr := fmt.Errorf("error creating executor: %w", err) + GinkgoWriter.Printf("%v\n", executorErr) + + return nil, executorErr } return exec, nil diff --git a/tests/framework/generate_manifests.go b/tests/framework/generate_manifests.go index e0ad091585..ce169bb23d 100644 --- a/tests/framework/generate_manifests.go +++ b/tests/framework/generate_manifests.go @@ -7,6 +7,7 @@ import ( "io" "text/template" + . "github.com/onsi/ginkgo/v2" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" @@ -150,7 +151,11 @@ func decodeObjects(reader io.Reader) ([]client.Object, error) { } // GenerateScaleListenerObjects generates objects for a given number of listeners for the scale test. -func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, error) { +func GenerateScaleListenerObjects( + numListeners int, + tls bool, + opts ...Option, +) (ScaleObjects, error) { var result ScaleObjects listeners := make([]listener, 0) @@ -183,8 +188,17 @@ func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, err backends = append(backends, backendName) + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + if !options.logEnabled { + GinkgoWriter.Printf("Generating manifests for listeners %v and route %v\n", listeners, r) + } objects, err := generateManifests(listeners, []route{r}) if err != nil { + GinkgoWriter.Printf("Error generating manifests: %v\n", err) + return ScaleObjects{}, err } @@ -209,6 +223,7 @@ func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, err } func generateSecrets(secrets []string) ([]client.Object, error) { + GinkgoWriter.Printf("Generating secrets\n") objects := make([]client.Object, 0, len(secrets)) for _, secret := range secrets { @@ -225,6 +240,7 @@ func generateSecrets(secrets []string) ([]client.Object, error) { objects = append(objects, objs...) } + GinkgoWriter.Printf("Generated %d secrets\n", len(objects)) return objects, nil } diff --git a/tests/framework/info.go b/tests/framework/info.go index c485edc9aa..773af10490 100644 --- a/tests/framework/info.go +++ b/tests/framework/info.go @@ -11,6 +11,11 @@ import ( // GetLogs returns the logs for all containers in all pods for a release. func GetLogs(rm ResourceManager, namespace string, releaseName string) string { + GinkgoWriter.Printf( + "Getting logs for all containers in all pods for release %q in namespace %q\n", + releaseName, + namespace, + ) var returnLogs string pods, err := rm.GetPods(namespace, client.MatchingLabels{ "app.kubernetes.io/instance": releaseName, @@ -26,6 +31,13 @@ func GetLogs(rm ResourceManager, namespace string, releaseName string) string { Container: container.Name, }) if err != nil { + GinkgoWriter.Printf( + "ERROR occurred during getting logs for container %q in pod %q in namespace %q: %v\n", + container.Name, + pod.Name, + pod.Namespace, + err, + ) returnLogs += fmt.Sprintf(" failed to get logs: %v\n", err) continue } @@ -40,6 +52,8 @@ func GetEvents(rm ResourceManager, namespace string) string { var returnEvents string events, err := rm.GetEvents(namespace) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting events in namespace %q: %v\n", namespace, err) + return fmt.Sprintf("failed to get events: %v", err) } @@ -60,6 +74,7 @@ func GetEvents(rm ResourceManager, namespace string) string { // GetBuildInfo returns the build information. func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) { + GinkgoWriter.Printf("Getting build info\n") commitHash = "unknown" commitTime = "unknown" dirtyBuild = "unknown" @@ -84,11 +99,12 @@ func GetBuildInfo() (commitHash string, commitTime string, dirtyBuild string) { } // AddNginxLogsAndEventsToReport adds nginx logs and events from the namespace to the report if the spec failed. -func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string) { +func AddNginxLogsAndEventsToReport(rm ResourceManager, namespace string, opts ...Option) { if CurrentSpecReport().Failed() { + GinkgoWriter.Printf("Current spec failed. Adding Nginx logs and events to report for namespace %q\n", namespace) var returnLogs string - nginxPodNames, _ := GetReadyNginxPodNames(rm.K8sClient, namespace, rm.TimeoutConfig.GetStatusTimeout) + nginxPodNames, _ := GetReadyNginxPodNames(rm.K8sClient, namespace, rm.TimeoutConfig.GetStatusTimeout, opts...) for _, nginxPodName := range nginxPodNames { returnLogs += fmt.Sprintf("Logs for Nginx Pod %s:\n", nginxPodName) diff --git a/tests/framework/k8s_client.go b/tests/framework/k8s_client.go new file mode 100644 index 0000000000..a623d6312f --- /dev/null +++ b/tests/framework/k8s_client.go @@ -0,0 +1,193 @@ +package framework + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// K8sClient wraps controller-runtime Client to add logging and custom behavior. +type K8sClient struct { + inner client.Client +} + +func (c *K8sClient) InNamespace(namespace string) client.ListOption { + return client.InNamespace(namespace) +} + +// HasLabels returns a ListOption that filters for objects which have +// all of the given label keys (regardless of value). +func (c *K8sClient) HasLabels(keys ...string) client.ListOption { + reqs := make([]metav1.LabelSelectorRequirement, 0, len(keys)) + for _, k := range keys { + reqs = append(reqs, metav1.LabelSelectorRequirement{ + Key: k, + Operator: metav1.LabelSelectorOpExists, + }) + } + sel := &metav1.LabelSelector{MatchExpressions: reqs} + ls, err := metav1.LabelSelectorAsSelector(sel) + if err != nil { + GinkgoWriter.Printf("error constructing label selector: %v\n", err) + // fallback to a selector that matches nothing + return client.MatchingLabelsSelector{Selector: labels.Nothing()} + } + return client.MatchingLabelsSelector{Selector: ls} +} + +// MatchingLabels is just a passthrough to the real helper. +func (c *K8sClient) MatchingLabels(m map[string]string) client.ListOption { + return client.MatchingLabels(m) +} + +func logOptions(opts ...Option) *Options { + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + + return options +} + +// NewK8sClient returns a new wrapped Kubernetes client. +func NewK8sClient(config *rest.Config, options client.Options) (K8sClient, error) { + inner, err := client.New(config, options) + if err != nil { + clientErr := fmt.Errorf("error creating k8s client: %w", err) + GinkgoWriter.Printf("%v\n", clientErr) + + return K8sClient{}, err + } + + return K8sClient{inner: inner}, nil +} + +// Get retrieves a resource by key, logging errors if enabled. +func (c *K8sClient) Get( + ctx context.Context, + key client.ObjectKey, + obj client.Object, + opts ...Option, +) error { + options := logOptions(opts...) + err := c.inner.Get(ctx, key, obj) + if err != nil { + if apierrors.IsNotFound(err) { + if options.logEnabled { + GinkgoWriter.Printf("Not found k8s resource %q error: %v\n", obj.GetName(), err) + } + return err + } + getErr := fmt.Errorf("error getting k8s resource %q: %w", obj.GetName(), err) + if options.logEnabled { + GinkgoWriter.Printf("%v\n", getErr) + } + + return getErr + } + + return nil +} + +// Create adds a new resource, returning an error on failure. +func (c *K8sClient) Create( + ctx context.Context, + obj client.Object, +) error { + err := c.inner.Create(ctx, obj) + if err != nil { + createErr := fmt.Errorf("error creating k8s resource %q: %w", obj.GetName(), err) + GinkgoWriter.Printf("%v\n", createErr) + + return createErr + } + return nil +} + +// Delete removes a resource, returning an error on failure. +func (c *K8sClient) Delete( + ctx context.Context, + obj client.Object, + deleteOpts []client.DeleteOption, + opts ...Option, +) error { + options := logOptions(opts...) + var dOpts []client.DeleteOption + for _, do := range deleteOpts { + if do != nil { + dOpts = append(dOpts, do) + } + } + + err := c.inner.Delete(ctx, obj, dOpts...) + if err != nil { + deleteErr := fmt.Errorf("error deleting k8s resource %q: %w", obj.GetName(), err) + if options.logEnabled { + GinkgoWriter.Printf("%v\n", deleteErr) + } + + return deleteErr + } + return nil +} + +// Update modifies a resource. +func (c *K8sClient) Update( + ctx context.Context, + obj client.Object, + updateOpts []client.UpdateOption, + opts ...Option, +) error { + options := logOptions(opts...) + var uOpts []client.UpdateOption + for _, uo := range updateOpts { + if uo != nil { + uOpts = append(uOpts, uo) + } + } + + if err := c.inner.Update(ctx, obj, uOpts...); err != nil { + updateDeploymentErr := fmt.Errorf("error updating Deployment: %w", err) + if options.logEnabled { + GinkgoWriter.Printf( + "ERROR occurred during updating Deployment in namespace %q with name %q, error: %s\n", + obj.GetNamespace(), + obj.GetName(), + updateDeploymentErr, + ) + } + + return updateDeploymentErr + } + + return nil +} + +// List retrieves a list of resources, returning an error on failure. +func (c *K8sClient) List( + ctx context.Context, + list client.ObjectList, + listOpts ...client.ListOption, +) error { + var opts []client.ListOption + for _, o := range listOpts { + if o != nil { + opts = append(opts, o) + } + } + + err := c.inner.List(ctx, list, opts...) + if err != nil { + listErr := fmt.Errorf("error listing k8s resources: %w", err) + GinkgoWriter.Printf("%v\n", listErr) + + return listErr + } + return nil +} diff --git a/tests/framework/load.go b/tests/framework/load.go index c687f275b1..d6caf312b5 100644 --- a/tests/framework/load.go +++ b/tests/framework/load.go @@ -7,6 +7,7 @@ import ( "net/http" "time" + . "github.com/onsi/ginkgo/v2" vegeta "github.com/tsenart/vegeta/v12/lib" ) @@ -49,6 +50,7 @@ type Metrics struct { // RunLoadTest uses Vegeta to send traffic to the provided Targets at the given rate for the given duration and writes // the results to the provided file. func RunLoadTest(cfg LoadTestConfig) (vegeta.Results, Metrics) { + GinkgoWriter.Printf("Running load test: %s\n", cfg.Description) vegTargets := convertTargetToVegetaTarget(cfg.Targets) targeter := vegeta.NewStaticTargeter(vegTargets...) @@ -61,7 +63,12 @@ func RunLoadTest(cfg LoadTestConfig) (vegeta.Results, Metrics) { Timeout: vegeta.DefaultTimeout, Transport: &http.Transport{ DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { - return dialer.DialContext(ctx, network, cfg.Proxy) + conn, err := dialer.DialContext(ctx, network, cfg.Proxy) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during dialing %q in %q network, error: %s\n", cfg.Proxy, network, err) + } + + return conn, err }, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, //nolint:gosec // self-signed cert for testing diff --git a/tests/framework/logging.go b/tests/framework/logging.go new file mode 100644 index 0000000000..aaf9f54a48 --- /dev/null +++ b/tests/framework/logging.go @@ -0,0 +1,13 @@ +package framework + +type Option func(*Options) + +type Options struct { + logEnabled bool +} + +func WithLoggingDisabled() Option { + return func(opts *Options) { + opts.logEnabled = false + } +} diff --git a/tests/framework/ngf.go b/tests/framework/ngf.go index 8bff1a4194..0f2c14cf3a 100644 --- a/tests/framework/ngf.go +++ b/tests/framework/ngf.go @@ -14,7 +14,6 @@ import ( apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -41,6 +40,7 @@ type InstallationConfig struct { // InstallGatewayAPI installs the specified version of the Gateway API resources. func InstallGatewayAPI(apiVersion string) ([]byte, error) { apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion) + GinkgoWriter.Printf("Installing Gateway API version %q at API path %q\n", apiVersion, apiPath) cmd := exec.CommandContext( context.Background(), @@ -48,8 +48,11 @@ func InstallGatewayAPI(apiVersion string) ([]byte, error) { ) output, err := cmd.CombinedOutput() if err != nil { + GinkgoWriter.Printf("Error installing Gateway API version %q: %v\n", apiVersion, err) + return output, err } + GinkgoWriter.Printf("Successfully installed Gateway API version %q\n", apiVersion) return nil, nil } @@ -57,11 +60,15 @@ func InstallGatewayAPI(apiVersion string) ([]byte, error) { // UninstallGatewayAPI uninstalls the specified version of the Gateway API resources. func UninstallGatewayAPI(apiVersion string) ([]byte, error) { apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion) + GinkgoWriter.Printf("Uninstalling Gateway API version %q at API path %q\n", apiVersion, apiPath) output, err := exec.CommandContext(context.Background(), "kubectl", "delete", "-f", apiPath).CombinedOutput() if err != nil && !strings.Contains(string(output), "not found") { + GinkgoWriter.Printf("Error uninstalling Gateway API version %q: %v\n", apiVersion, err) + return output, err } + GinkgoWriter.Printf("Successfully uninstalled Gateway API version %q\n", apiVersion) return nil, nil } @@ -93,10 +100,15 @@ func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) { } // CreateLicenseSecret creates the NGINX Plus JWT secret. -func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) error { +func CreateLicenseSecret(k8sClient K8sClient, namespace, filename string) error { + GinkgoWriter.Printf("Creating NGINX Plus license secret in namespace %q from file %q\n", namespace, filename) + conf, err := os.ReadFile(filename) if err != nil { - return fmt.Errorf("error reading file %q: %w", filename, err) + readFileErr := fmt.Errorf("error reading file %q: %w", filename, err) + GinkgoWriter.Printf("%v\n", readFileErr) + + return readFileErr } ctx, cancel := context.WithTimeout(context.Background(), DefaultTimeoutConfig().CreateTimeout) @@ -123,7 +135,10 @@ func CreateLicenseSecret(k8sClient client.Client, namespace, filename string) er } if err := k8sClient.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("error creating secret: %w", err) + createSecretErr := fmt.Errorf("error creating secret: %w", err) + GinkgoWriter.Printf("%v\n", createSecretErr) + + return createSecretErr } return nil @@ -166,10 +181,11 @@ func UpgradeNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) { } // UninstallNGF uninstalls NGF. -func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, error) { +func UninstallNGF(cfg InstallationConfig, k8sClient K8sClient) ([]byte, error) { args := []string{ "uninstall", cfg.ReleaseName, "--namespace", cfg.Namespace, } + GinkgoWriter.Printf("Uninstalling NGF with command: helm %v\n", strings.Join(args, " ")) output, err := exec.CommandContext(context.Background(), "helm", args...).CombinedOutput() if err != nil && !strings.Contains(string(output), "release: not found") { @@ -179,20 +195,20 @@ func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, erro ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err = k8sClient.Delete(ctx, &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cfg.Namespace}}) + err = k8sClient.Delete(ctx, &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cfg.Namespace}}, nil) if err != nil && !apierrors.IsNotFound(err) { return nil, err } var crList apiext.CustomResourceDefinitionList - if err := k8sClient.List(ctx, &crList); err != nil { + if err := k8sClient.List(ctx, &crList, nil); err != nil { return nil, err } for _, cr := range crList.Items { if strings.Contains(cr.Spec.Group, "gateway.nginx.org") { cr := cr - if err := k8sClient.Delete(ctx, &cr); err != nil && !apierrors.IsNotFound(err) { + if err := k8sClient.Delete(ctx, &cr, nil); err != nil && !apierrors.IsNotFound(err) { return nil, err } } @@ -204,6 +220,7 @@ func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, erro func setTelemetryArgs(cfg InstallationConfig) []string { var args []string + GinkgoWriter.Printf("Setting telemetry to %v\n", cfg.Telemetry) if cfg.Telemetry { args = append(args, formatValueSet("nginxGateway.productTelemetry.enable", "true")...) } else { diff --git a/tests/framework/portforward.go b/tests/framework/portforward.go index 9465c330d6..7fdd2aa4f5 100644 --- a/tests/framework/portforward.go +++ b/tests/framework/portforward.go @@ -11,6 +11,7 @@ import ( "sync" "time" + . "github.com/onsi/ginkgo/v2" "k8s.io/client-go/rest" "k8s.io/client-go/tools/portforward" "k8s.io/client-go/transport/spdy" @@ -20,12 +21,18 @@ import ( func PortForward(config *rest.Config, namespace, podName string, ports []string, stopCh <-chan struct{}) error { roundTripper, upgrader, err := spdy.RoundTripperFor(config) if err != nil { - return fmt.Errorf("error creating roundtripper: %w", err) + roundTripperErr := fmt.Errorf("error creating roundtripper: %w", err) + GinkgoWriter.Printf("%v\n", roundTripperErr) + + return roundTripperErr } serverURL, err := url.Parse(config.Host) if err != nil { - return fmt.Errorf("error parsing rest config host: %w", err) + parseConfigErr := fmt.Errorf("error parsing rest config host: %w", err) + GinkgoWriter.Printf("%v\n", parseConfigErr) + + return parseConfigErr } serverURL.Path = path.Join( @@ -35,14 +42,24 @@ func PortForward(config *rest.Config, namespace, podName string, ports []string, "portforward", ) + GinkgoWriter.Printf("Creating new dialer for serverURL: %q\n", serverURL) dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, serverURL) forward := func() error { readyCh := make(chan struct{}, 1) + GinkgoWriter.Printf( + "Starting port-forward to pod %q in namespace %q for ports %v\n", + podName, + namespace, + ports, + ) forwarder, err := portforward.New(dialer, ports, stopCh, readyCh, newSafeBuffer(), newSafeBuffer()) if err != nil { - return fmt.Errorf("error creating port forwarder: %w", err) + createPortForwardErr := fmt.Errorf("error creating port forwarder: %w", err) + GinkgoWriter.Printf("%v\n", createPortForwardErr) + + return createPortForwardErr } return forwarder.ForwardPorts() diff --git a/tests/framework/prometheus.go b/tests/framework/prometheus.go index 24c594ddf2..6067ad4de3 100644 --- a/tests/framework/prometheus.go +++ b/tests/framework/prometheus.go @@ -10,6 +10,7 @@ import ( "os/exec" "time" + . "github.com/onsi/ginkgo/v2" "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" @@ -49,7 +50,10 @@ func InstallPrometheus( "https://prometheus-community.github.io/helm-charts", ).CombinedOutput() if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to add Prometheus helm repo: %w; output: %s", err, string(output)) + prometheusErr := fmt.Errorf("failed to add Prometheus helm repo: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", prometheusErr) + + return PrometheusInstance{}, prometheusErr } output, err = exec.CommandContext( @@ -59,7 +63,10 @@ func InstallPrometheus( "update", ).CombinedOutput() if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to update helm repos: %w; output: %s", err, string(output)) + helmReposErr := fmt.Errorf("failed to update helm repos: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", helmReposErr) + + return PrometheusInstance{}, helmReposErr } scrapeInterval := fmt.Sprintf("%ds", int(cfg.ScrapeInterval.Seconds())) @@ -77,24 +84,36 @@ func InstallPrometheus( "--wait", ).CombinedOutput() if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to install Prometheus: %w; output: %s", err, string(output)) + prometheusInstallationErr := fmt.Errorf("failed to install Prometheus: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", prometheusInstallationErr) + + return PrometheusInstance{}, prometheusInstallationErr } pods, err := rm.GetPods(prometheusNamespace, client.MatchingLabels{ "app.kubernetes.io/name": "prometheus", }) if err != nil { - return PrometheusInstance{}, fmt.Errorf("failed to get Prometheus pods: %w", err) + podsErr := fmt.Errorf("failed to get Prometheus pods: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", podsErr) + + return PrometheusInstance{}, podsErr } if len(pods) != 1 { - return PrometheusInstance{}, fmt.Errorf("expected one Prometheus pod, found %d", len(pods)) + manyPodsErr := fmt.Errorf("expected one Prometheus pod, found %d", len(pods)) + GinkgoWriter.Printf("ERROR: %v\n", manyPodsErr) + + return PrometheusInstance{}, manyPodsErr } pod := pods[0] if pod.Status.PodIP == "" { - return PrometheusInstance{}, errors.New("the Prometheus pod has no IP") + podIPErr := errors.New("the Prometheus pod has no IP") + GinkgoWriter.Printf("ERROR: %v\n", podIPErr) + + return PrometheusInstance{}, podIPErr } var queryTimeout time.Duration @@ -114,6 +133,7 @@ func InstallPrometheus( // UninstallPrometheus uninstalls Prometheus from the cluster. func UninstallPrometheus(rm ResourceManager) error { + GinkgoWriter.Printf("Uninstalling Prometheus from namespace %q\n", prometheusNamespace) output, err := exec.CommandContext( context.Background(), "helm", @@ -122,11 +142,17 @@ func UninstallPrometheus(rm ResourceManager) error { "-n", prometheusNamespace, ).CombinedOutput() if err != nil { - return fmt.Errorf("failed to uninstall Prometheus: %w; output: %s", err, string(output)) + uninstallErr := fmt.Errorf("failed to uninstall Prometheus: %w; output: %s", err, string(output)) + GinkgoWriter.Printf("ERROR: %v\n", uninstallErr) + + return uninstallErr } if err := rm.DeleteNamespace(prometheusNamespace); err != nil { - return fmt.Errorf("failed to delete Prometheus namespace: %w", err) + deleteNSErr := fmt.Errorf("failed to delete Prometheus namespace: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", deleteNSErr) + + return deleteNSErr } return nil @@ -150,8 +176,12 @@ type PrometheusInstance struct { // PortForward starts port forwarding to the Prometheus instance. func (ins *PrometheusInstance) PortForward(config *rest.Config, stopCh <-chan struct{}) error { + GinkgoWriter.Printf("Starting port forwarding to Prometheus pod %q in namespace %q\n", ins.podName, ins.podNamespace) if ins.portForward { - panic("port forwarding already started") + infoMsg := "port forwarding already started" + GinkgoWriter.Printf("INFO: %s\n", infoMsg) + + panic(infoMsg) } ins.portForward = true @@ -161,6 +191,7 @@ func (ins *PrometheusInstance) PortForward(config *rest.Config, stopCh <-chan st } func (ins *PrometheusInstance) getAPIClient() (v1.API, error) { + GinkgoWriter.Printf("Creating Prometheus API client for pod %q in namespace %q\n", ins.podName, ins.podNamespace) var endpoint string if ins.portForward { endpoint = fmt.Sprintf("http://localhost:%d", PrometheusPortForwardPort) @@ -175,6 +206,8 @@ func (ins *PrometheusInstance) getAPIClient() (v1.API, error) { c, err := api.NewClient(cfg) if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating Prometheus API client: %v\n", err) + return nil, err } @@ -185,7 +218,10 @@ func (ins *PrometheusInstance) ensureAPIClient() error { if ins.apiClient == nil { ac, err := ins.getAPIClient() if err != nil { - return fmt.Errorf("failed to get Prometheus API client: %w", err) + apiClientErr := fmt.Errorf("failed to get Prometheus API client: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", apiClientErr) + + return apiClientErr } ins.apiClient = ac } @@ -195,6 +231,7 @@ func (ins *PrometheusInstance) ensureAPIClient() error { // Query sends a query to Prometheus. func (ins *PrometheusInstance) Query(query string) (model.Value, error) { + GinkgoWriter.Printf("Querying Prometheus with query: %q\n", query) ctx, cancel := context.WithTimeout(context.Background(), ins.queryTimeout) defer cancel() @@ -209,10 +246,14 @@ func (ins *PrometheusInstance) QueryWithCtx(ctx context.Context, query string) ( result, warnings, err := ins.apiClient.Query(ctx, query, time.Time{}) if err != nil { - return nil, fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", queryErr) + + return nil, queryErr } if len(warnings) > 0 { + GinkgoWriter.Printf("WARNING: Prometheus query returned warnings: %v\n", warnings) slog.InfoContext(context.Background(), "Prometheus query returned warnings", "query", query, @@ -235,16 +276,23 @@ func (ins *PrometheusInstance) QueryRange(query string, promRange v1.Range) (mod func (ins *PrometheusInstance) QueryRangeWithCtx(ctx context.Context, query string, promRange v1.Range, ) (model.Value, error) { + GinkgoWriter.Printf("Querying Prometheus with range query: %q\n", query) if err := ins.ensureAPIClient(); err != nil { + GinkgoWriter.Printf("ERROR during ensureAPIClient for prometheus: %v\n", err) + return nil, err } result, warnings, err := ins.apiClient.QueryRange(ctx, query, promRange) if err != nil { - return nil, fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + GinkgoWriter.Printf("ERROR: %v\n", queryErr) + + return nil, queryErr } if len(warnings) > 0 { + GinkgoWriter.Printf("WARNING: Prometheus range query returned warnings: %v\n", warnings) slog.InfoContext(context.Background(), "Prometheus range query returned warnings", "query", query, @@ -260,11 +308,17 @@ func (ins *PrometheusInstance) QueryRangeWithCtx(ctx context.Context, func GetFirstValueOfPrometheusVector(val model.Value) (float64, error) { res, ok := val.(model.Vector) if !ok { - return 0, fmt.Errorf("expected a vector, got %T", val) + valueErr := fmt.Errorf("expected a vector, got %T", val) + GinkgoWriter.Printf("ERROR: %v\n", valueErr) + + return 0, valueErr } if len(res) == 0 { - return 0, errors.New("empty vector") + vectorErr := errors.New("empty vector") + GinkgoWriter.Printf("ERROR: %v\n", vectorErr) + + return 0, vectorErr } return float64(res[0].Value), nil @@ -272,8 +326,11 @@ func GetFirstValueOfPrometheusVector(val model.Value) (float64, error) { // WritePrometheusMatrixToCSVFile writes a Prometheus matrix to a CSV file. func WritePrometheusMatrixToCSVFile(fileName string, value model.Value) error { + GinkgoWriter.Printf("Writing Prometheus matrix to CSV file %q\n", fileName) file, err := os.Create(fileName) if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating file %q: %v\n", fileName, err) + return err } defer file.Close() @@ -282,13 +339,18 @@ func WritePrometheusMatrixToCSVFile(fileName string, value model.Value) error { matrix, ok := value.(model.Matrix) if !ok { - return fmt.Errorf("expected a matrix, got %T", value) + matrixErr := fmt.Errorf("expected a matrix, got %T", value) + GinkgoWriter.Printf("ERROR: %v\n", matrixErr) + + return matrixErr } for _, sample := range matrix { for _, pair := range sample.Values { record := []string{fmt.Sprint(pair.Timestamp.Unix()), pair.Value.String()} if err := csvWriter.Write(record); err != nil { + GinkgoWriter.Printf("ERROR: %v\n", err) + return err } } @@ -409,18 +471,33 @@ func CreateMetricExistChecker( query string, getTime func() time.Time, modifyTime func(), + opts ...Option, ) func() error { return func() error { queryWithTimestamp := fmt.Sprintf("%s @ %d", query, getTime().Unix()) + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } result, err := promInstance.Query(queryWithTimestamp) if err != nil { - return fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating metric existence checker: %v\n", queryErr) + } + + return queryErr } if result.String() == "" { modifyTime() - return errors.New("empty result") + emptyResultErr := errors.New("empty result") + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating metric existence checker: %v\n", emptyResultErr) + } + + return emptyResultErr } return nil @@ -436,6 +513,7 @@ func CreateEndTimeFinder( endTime *time.Time, queryRangeStep time.Duration, ) func() error { + GinkgoWriter.Printf("Creating end time finder with start time %v and initial end time %v\n", startTime, endTime) return func() error { result, err := promInstance.QueryRange(query, v1.Range{ Start: startTime, @@ -443,12 +521,18 @@ func CreateEndTimeFinder( Step: queryRangeStep, }) if err != nil { - return fmt.Errorf("failed to query Prometheus: %w", err) + queryErr := fmt.Errorf("failed to query Prometheus: %w", err) + GinkgoWriter.Printf("ERROR during creating end time finder: %v\n", queryErr) + + return queryErr } if result.String() == "" { *endTime = time.Now() - return errors.New("empty result") + emptyResultsErr := errors.New("empty result") + GinkgoWriter.Printf("ERROR during creating end time finder: %v\n", emptyResultsErr) + + return emptyResultsErr } return nil @@ -456,15 +540,33 @@ func CreateEndTimeFinder( } // CreateResponseChecker returns a function that checks if there is a successful response from a url. -func CreateResponseChecker(url, address string, requestTimeout time.Duration) func() error { +func CreateResponseChecker(url, address string, requestTimeout time.Duration, opts ...Option) func() error { + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + if options.logEnabled { + GinkgoWriter.Printf("Starting checking response for url %q and address %q\n", url, address) + } + return func() error { - status, _, err := Get(url, address, requestTimeout, nil, nil) + status, _, err := Get(url, address, requestTimeout, nil, nil, opts...) if err != nil { - return fmt.Errorf("bad response: %w", err) + badReqErr := fmt.Errorf("bad response: %w", err) + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating response checker: %v\n", badReqErr) + } + + return badReqErr } if status != 200 { - return fmt.Errorf("unexpected status code: %d", status) + statusErr := fmt.Errorf("unexpected status code: %d", status) + if options.logEnabled { + GinkgoWriter.Printf("ERROR during creating response checker: %v\n", statusErr) + } + + return statusErr } return nil @@ -474,11 +576,15 @@ func CreateResponseChecker(url, address string, requestTimeout time.Duration) fu func getFirstValueOfVector(query string, promInstance PrometheusInstance) (float64, error) { result, err := promInstance.Query(query) if err != nil { + GinkgoWriter.Printf("ERROR querying Prometheus during getting first value of vector: %v\n", err) + return 0, err } val, err := GetFirstValueOfPrometheusVector(result) if err != nil { + GinkgoWriter.Printf("ERROR getting first value of Prometheus vector: %v\n", err) + return 0, err } @@ -488,12 +594,17 @@ func getFirstValueOfVector(query string, promInstance PrometheusInstance) (float func getBuckets(query string, promInstance PrometheusInstance) ([]Bucket, error) { result, err := promInstance.Query(query) if err != nil { + GinkgoWriter.Printf("ERROR querying Prometheus during getting buckets: %v\n", err) + return nil, err } res, ok := result.(model.Vector) if !ok { - return nil, errors.New("could not convert result to vector") + convertationErr := errors.New("could not convert result to vector") + GinkgoWriter.Printf("ERROR during getting buckets: %v\n", convertationErr) + + return nil, convertationErr } buckets := make([]Bucket, 0, len(res)) diff --git a/tests/framework/request.go b/tests/framework/request.go index 7cc1b0384c..a04d8ccc66 100644 --- a/tests/framework/request.go +++ b/tests/framework/request.go @@ -11,6 +11,8 @@ import ( "net/http" "strings" "time" + + . "github.com/onsi/ginkgo/v2" ) // Get sends a GET request to the specified url. @@ -20,9 +22,22 @@ func Get( url, address string, timeout time.Duration, headers, queryParams map[string]string, + opts ...Option, ) (int, string, error) { - resp, err := makeRequest(http.MethodGet, url, address, nil, timeout, headers, queryParams) + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + + resp, err := makeRequest(http.MethodGet, url, address, nil, timeout, headers, queryParams, opts...) if err != nil { + if options.logEnabled { + GinkgoWriter.Printf( + "ERROR occurred during getting response, error: %s\nReturning status: 0, body: ''\n", + err, + ) + } + return 0, "", err } defer resp.Body.Close() @@ -30,8 +45,12 @@ func Get( body := new(bytes.Buffer) _, err = body.ReadFrom(resp.Body) if err != nil { + GinkgoWriter.Printf("ERROR in Body content: %v returning body: ''\n", err) return resp.StatusCode, "", err } + if options.logEnabled { + GinkgoWriter.Printf("Successfully received response and parsed body: %s\n", body.String()) + } return resp.StatusCode, body.String(), nil } @@ -44,7 +63,12 @@ func Post( timeout time.Duration, headers, queryParams map[string]string, ) (*http.Response, error) { - return makeRequest(http.MethodPost, url, address, body, timeout, headers, queryParams) + response, err := makeRequest(http.MethodPost, url, address, body, timeout, headers, queryParams) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting response, error: %s\n", err) + } + + return response, err } func makeRequest( @@ -52,6 +76,7 @@ func makeRequest( body io.Reader, timeout time.Duration, headers, queryParams map[string]string, + opts ...Option, ) (*http.Response, error) { dialer := &net.Dialer{} @@ -74,6 +99,23 @@ func makeRequest( ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() + options := &Options{logEnabled: true} + + for _, opt := range opts { + opt(options) + } + if options.logEnabled { + requestDetails := fmt.Sprintf( + "Method: %s, URL: %s, Address: %s, Headers: %v, QueryParams: %v\n", + strings.ToUpper(method), + url, + address, + headers, + queryParams, + ) + GinkgoWriter.Printf("Sending request: %s", requestDetails) + } + req, err := http.NewRequestWithContext(ctx, method, url, body) if err != nil { return nil, err diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go index 8b71b644f6..f8dd1e5b00 100644 --- a/tests/framework/resourcemanager.go +++ b/tests/framework/resourcemanager.go @@ -33,6 +33,7 @@ import ( "strings" "time" + . "github.com/onsi/ginkgo/v2" apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -53,7 +54,7 @@ import ( // ResourceManager handles creating/updating/deleting Kubernetes resources. type ResourceManager struct { - K8sClient client.Client + K8sClient K8sClient ClientGoClient kubernetes.Interface // used when k8sClient is not enough K8sConfig *rest.Config FS embed.FS @@ -75,7 +76,14 @@ type ClusterInfo struct { } // Apply creates or updates Kubernetes resources defined as Go objects. -func (rm *ResourceManager) Apply(resources []client.Object) error { +func (rm *ResourceManager) Apply(resources []client.Object, opts ...Option) error { + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + if options.logEnabled { + GinkgoWriter.Printf("Applying resources defined as Go objects\n") + } ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) defer cancel() @@ -89,13 +97,19 @@ func (rm *ResourceManager) Apply(resources []client.Object) error { t := reflect.TypeOf(resource).Elem() obj, ok = reflect.New(t).Interface().(client.Object) if !ok { - panic("failed to cast object to client.Object") + panicMsg := "failed to cast object to client.Object" + GinkgoWriter.Printf( + "PANIC occurred during applying creates or updates Kubernetes resources defined as Go objects: %s\n", + panicMsg, + ) + + panic(panicMsg) } } - if err := rm.K8sClient.Get(ctx, client.ObjectKeyFromObject(resource), obj); err != nil { + if err := rm.K8sClient.Get(ctx, client.ObjectKeyFromObject(resource), obj, opts...); err != nil { if !apierrors.IsNotFound(err) { - return fmt.Errorf("error getting resource: %w", err) + return err } if err := rm.K8sClient.Create(ctx, resource); err != nil { @@ -112,19 +126,32 @@ func (rm *ResourceManager) Apply(resources []client.Object) error { return err } resource.SetResourceVersion(obj.GetResourceVersion()) - return rm.K8sClient.Update(ctx, resource) + + return rm.K8sClient.Update(ctx, resource, nil) }) if err != nil { - return fmt.Errorf("error updating resource: %w", err) + retryErr := fmt.Errorf("error updating resource: %w", err) + GinkgoWriter.Printf("%s\n", retryErr) + + return retryErr } } - + if options.logEnabled { + GinkgoWriter.Printf("Resources defined as Go objects applied successfully\n") + } return nil } // ApplyFromFiles creates or updates Kubernetes resources defined within the provided YAML files. -func (rm *ResourceManager) ApplyFromFiles(files []string, namespace string) error { +func (rm *ResourceManager) ApplyFromFiles(files []string, namespace string, opts ...Option) error { + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } for _, file := range files { + if options.logEnabled { + GinkgoWriter.Printf("\nApplying resources from file: %q to namespace %q\n", file, namespace) + } data, err := rm.GetFileContents(file) if err != nil { return err @@ -134,20 +161,31 @@ func (rm *ResourceManager) ApplyFromFiles(files []string, namespace string) erro return err } } + if options.logEnabled { + GinkgoWriter.Printf("Resources from files applied successfully to namespace %q,\n", namespace) + } + return nil } -func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace string) error { +func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace string, opts ...Option) error { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) defer cancel() + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + if options.logEnabled { + GinkgoWriter.Printf("Applying resources from buffer to namespace %q\n", namespace) + } handlerFunc := func(obj unstructured.Unstructured) error { obj.SetNamespace(namespace) nsName := types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()} fetchedObj := obj.DeepCopy() - if err := rm.K8sClient.Get(ctx, nsName, fetchedObj); err != nil { + if err := rm.K8sClient.Get(ctx, nsName, fetchedObj, opts...); err != nil { if !apierrors.IsNotFound(err) { - return fmt.Errorf("error getting resource: %w", err) + return err } if err := rm.K8sClient.Create(ctx, &obj); err != nil { @@ -164,10 +202,16 @@ func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace strin return err } obj.SetResourceVersion(fetchedObj.GetResourceVersion()) - return rm.K8sClient.Update(ctx, &obj) + + return rm.K8sClient.Update(ctx, &obj, nil) }) if err != nil { - return fmt.Errorf("error updating resource: %w", err) + retryErr := fmt.Errorf("error updating resource: %w", err) + GinkgoWriter.Printf("%s\n", + retryErr, + ) + + return retryErr } return nil @@ -178,51 +222,59 @@ func (rm *ResourceManager) ApplyFromBuffer(buffer *bytes.Buffer, namespace strin // Delete deletes Kubernetes resources defined as Go objects. func (rm *ResourceManager) Delete(resources []client.Object, opts ...client.DeleteOption) error { + GinkgoWriter.Printf("Deleting resources\n") ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteTimeout) defer cancel() for _, resource := range resources { - if err := rm.K8sClient.Delete(ctx, resource, opts...); err != nil && !apierrors.IsNotFound(err) { + if err := rm.K8sClient.Delete(ctx, resource, opts); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("error deleting resource: %w", err) } } + GinkgoWriter.Printf("Resources deleted successfully\n") return nil } -func (rm *ResourceManager) DeleteNamespace(name string) error { +func (rm *ResourceManager) DeleteNamespace(name string, opts ...Option) error { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteNamespaceTimeout) + GinkgoWriter.Printf("Deleting namespace %q\n", name) defer cancel() ns := &core.Namespace{} - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Name: name}, ns); err != nil { + if err := rm.K8sClient.Get(ctx, types.NamespacedName{Name: name}, ns, opts...); err != nil { if apierrors.IsNotFound(err) { return nil } + return fmt.Errorf("error getting namespace: %w", err) } - if err := rm.K8sClient.Delete(ctx, ns); err != nil { + if err := rm.K8sClient.Delete(ctx, ns, nil, opts...); err != nil { return fmt.Errorf("error deleting namespace: %w", err) } + GinkgoWriter.Printf("Waiting for namespace %q to be deleted\n", name) // Because the namespace deletion is asynchronous, we need to wait for the namespace to be deleted. return wait.PollUntilContextCancel( ctx, 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { - if err := rm.K8sClient.Get(ctx, types.NamespacedName{Name: name}, ns); err != nil { + if err := rm.K8sClient.Get(ctx, types.NamespacedName{Name: name}, ns, opts...); err != nil { if apierrors.IsNotFound(err) { return true, nil } + return false, fmt.Errorf("error getting namespace: %w", err) } + return false, nil }) } -func (rm *ResourceManager) DeleteNamespaces(names []string) error { +func (rm *ResourceManager) DeleteNamespaces(names []string, opts ...Option) error { + GinkgoWriter.Printf("Deleting namespaces: %v\n", names) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteNamespaceTimeout*2) defer cancel() @@ -230,7 +282,7 @@ func (rm *ResourceManager) DeleteNamespaces(names []string) error { for _, name := range names { ns := &core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}} - if err := rm.K8sClient.Delete(ctx, ns); err != nil { + if err := rm.K8sClient.Delete(ctx, ns, nil, opts...); err != nil { if apierrors.IsNotFound(err) { continue } @@ -245,7 +297,7 @@ func (rm *ResourceManager) DeleteNamespaces(names []string) error { true, /* poll immediately */ func(ctx context.Context) (bool, error) { nsList := &core.NamespaceList{} - if err := rm.K8sClient.List(ctx, nsList); err != nil { + if err := rm.K8sClient.List(ctx, nsList, nil); err != nil { return false, nil //nolint:nilerr // retry on error } @@ -263,12 +315,13 @@ func (rm *ResourceManager) DeleteNamespaces(names []string) error { // DeleteFromFiles deletes Kubernetes resources defined within the provided YAML files. func (rm *ResourceManager) DeleteFromFiles(files []string, namespace string) error { + GinkgoWriter.Printf("Deleting resources from files: %v in namespace %q\n", files, namespace) handlerFunc := func(obj unstructured.Unstructured) error { obj.SetNamespace(namespace) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.DeleteTimeout) defer cancel() - if err := rm.K8sClient.Delete(ctx, &obj); err != nil && !apierrors.IsNotFound(err) { + if err := rm.K8sClient.Delete(ctx, &obj, nil); err != nil && !apierrors.IsNotFound(err) { return err } @@ -320,34 +373,49 @@ func (rm *ResourceManager) readAndHandleObject( // path or an https:// URL to YAML manifests and provides the contents. func (rm *ResourceManager) GetFileContents(file string) (*bytes.Buffer, error) { if strings.HasPrefix(file, "http://") { - return nil, fmt.Errorf("data can't be retrieved from %s: http is not supported, use https", file) + err := fmt.Errorf("data can't be retrieved from %s: http is not supported, use https", file) + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + + return nil, err } else if strings.HasPrefix(file, "https://") { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.ManifestFetchTimeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, file, nil) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + return nil, err } resp, err := http.DefaultClient.Do(req) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%d response when getting %s file contents", resp.StatusCode, file) + err = fmt.Errorf("%d response when getting %s file contents", resp.StatusCode, file) + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + + return nil, err } manifests := new(bytes.Buffer) count, err := manifests.ReadFrom(resp.Body) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + return nil, err } if resp.ContentLength != -1 && count != resp.ContentLength { - return nil, fmt.Errorf("received %d bytes from %s, expected %d", count, file, resp.ContentLength) + err = fmt.Errorf("received %d bytes from %s, expected %d", count, file, resp.ContentLength) + GinkgoWriter.Printf("ERROR occurred during getting contents for file %q, error: %s\n", file, err) + + return nil, err } return manifests, nil } @@ -358,6 +426,8 @@ func (rm *ResourceManager) GetFileContents(file string) (*bytes.Buffer, error) { b, err := rm.FS.ReadFile(file) if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting file contents for file %q, error: %s\n", file, err) + return nil, err } @@ -366,41 +436,65 @@ func (rm *ResourceManager) GetFileContents(file string) (*bytes.Buffer, error) { // WaitForAppsToBeReady waits for all apps in the specified namespace to be ready, // or until the ctx timeout is reached. -func (rm *ResourceManager) WaitForAppsToBeReady(namespace string) error { +func (rm *ResourceManager) WaitForAppsToBeReady(namespace string, opts ...Option) error { + GinkgoWriter.Printf("Waiting for apps to be ready in namespace %q\n", namespace) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) defer cancel() - return rm.WaitForAppsToBeReadyWithCtx(ctx, namespace) + return rm.WaitForAppsToBeReadyWithCtx(ctx, namespace, opts...) } // WaitForAppsToBeReadyWithCtx waits for all apps in the specified namespace to be ready or // until the provided context is canceled. -func (rm *ResourceManager) WaitForAppsToBeReadyWithCtx(ctx context.Context, namespace string) error { - if err := rm.WaitForPodsToBeReady(ctx, namespace); err != nil { +func (rm *ResourceManager) WaitForAppsToBeReadyWithCtx(ctx context.Context, namespace string, opts ...Option) error { + if err := rm.WaitForPodsToBeReady(ctx, namespace, opts...); err != nil { + GinkgoWriter.Printf("ERROR occurred during waiting for pods to be ready, error: %s\n", err) + return err } if err := rm.waitForHTTPRoutesToBeReady(ctx, namespace); err != nil { + GinkgoWriter.Printf("ERROR occurred during waiting for HTTPRoutes to be ready, error: %s\n", err) + return err } if err := rm.waitForGRPCRoutesToBeReady(ctx, namespace); err != nil { + GinkgoWriter.Printf("ERROR occurred during waiting for GRPCRoutes to be ready, error: %s\n", err) + return err } - return rm.waitForGatewaysToBeReady(ctx, namespace) + gatewayReadiness := rm.waitForGatewaysToBeReady(ctx, namespace) + if gatewayReadiness != nil { + GinkgoWriter.Printf("ERROR occurred during waiting for Gateways to be ready, error: %s\n", gatewayReadiness) + } + + return gatewayReadiness } // WaitForPodsToBeReady waits for all Pods in the specified namespace to be ready or // until the provided context is canceled. -func (rm *ResourceManager) WaitForPodsToBeReady(ctx context.Context, namespace string) error { - return wait.PollUntilContextCancel( +func (rm *ResourceManager) WaitForPodsToBeReady( + ctx context.Context, + namespace string, + opts ...Option, +) error { + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + waitingErr := wait.PollUntilContextCancel( ctx, 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { var podList core.PodList - if err := rm.K8sClient.List(ctx, &podList, client.InNamespace(namespace)); err != nil { + if err := rm.K8sClient.List( + ctx, + &podList, + rm.K8sClient.InNamespace(namespace), + ); err != nil { return false, err } @@ -412,10 +506,22 @@ func (rm *ResourceManager) WaitForPodsToBeReady(ctx context.Context, namespace s } } } + if options.logEnabled { + GinkgoWriter.Printf("Pods ready: %d out of %d in namespace %q\n", podsReady, len(podList.Items), namespace) + } return podsReady == len(podList.Items), nil }, ) + if waitingErr != nil { + GinkgoWriter.Printf( + "ERROR occurred during waiting for Pods to be ready in namespace %q, error: %s\n", + namespace, + waitingErr, + ) + } + + return waitingErr } func (rm *ResourceManager) waitForGatewaysToBeReady(ctx context.Context, namespace string) error { @@ -425,7 +531,11 @@ func (rm *ResourceManager) waitForGatewaysToBeReady(ctx context.Context, namespa true, /* poll immediately */ func(ctx context.Context) (bool, error) { var gatewayList v1.GatewayList - if err := rm.K8sClient.List(ctx, &gatewayList, client.InNamespace(namespace)); err != nil { + if err := rm.K8sClient.List( + ctx, + &gatewayList, + rm.K8sClient.InNamespace(namespace), + ); err != nil { return false, err } @@ -443,13 +553,18 @@ func (rm *ResourceManager) waitForGatewaysToBeReady(ctx context.Context, namespa } func (rm *ResourceManager) waitForHTTPRoutesToBeReady(ctx context.Context, namespace string) error { + GinkgoWriter.Printf("Waiting for HTTPRoutes to be ready in namespace %q\n", namespace) return wait.PollUntilContextCancel( ctx, 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { var routeList v1.HTTPRouteList - if err := rm.K8sClient.List(ctx, &routeList, client.InNamespace(namespace)); err != nil { + if err := rm.K8sClient.List( + ctx, + &routeList, + rm.K8sClient.InNamespace(namespace), + ); err != nil { return false, err } @@ -465,10 +580,13 @@ func (rm *ResourceManager) waitForHTTPRoutesToBeReady(ctx context.Context, names } func (rm *ResourceManager) waitForGRPCRoutesToBeReady(ctx context.Context, namespace string) error { + GinkgoWriter.Printf("Waiting for GRPCRoutes to be ready in namespace %q\n", namespace) // First, check if grpcroute even exists for v1. If not, ignore. var routeList v1.GRPCRouteList - err := rm.K8sClient.List(ctx, &routeList, client.InNamespace(namespace)) + err := rm.K8sClient.List(ctx, &routeList, rm.K8sClient.InNamespace(namespace)) if err != nil && strings.Contains(err.Error(), "no matches for kind") { + GinkgoWriter.Printf("No GRPCRoute resources found in namespace %q, skipping wait\n", namespace) + return nil } @@ -478,7 +596,11 @@ func (rm *ResourceManager) waitForGRPCRoutesToBeReady(ctx context.Context, names true, /* poll immediately */ func(ctx context.Context) (bool, error) { var routeList v1.GRPCRouteList - if err := rm.K8sClient.List(ctx, &routeList, client.InNamespace(namespace)); err != nil { + if err := rm.K8sClient.List( + ctx, + &routeList, + rm.K8sClient.InNamespace(namespace), + ); err != nil { return false, err } @@ -495,12 +617,16 @@ func (rm *ResourceManager) waitForGRPCRoutesToBeReady(ctx context.Context, names // GetLBIPAddress gets the IP or Hostname from the Loadbalancer service. func (rm *ResourceManager) GetLBIPAddress(namespace string) (string, error) { + GinkgoWriter.Printf("Getting LoadBalancer IP/Hostname in namespace %q\n", namespace) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.CreateTimeout) defer cancel() var serviceList core.ServiceList var address string - if err := rm.K8sClient.List(ctx, &serviceList, client.InNamespace(namespace)); err != nil { + if err := rm.K8sClient.List( + ctx, &serviceList, + rm.K8sClient.InNamespace(namespace), + ); err != nil { return "", err } var nsName types.NamespacedName @@ -551,6 +677,7 @@ func (rm *ResourceManager) waitForLBStatusToBeReady(ctx context.Context, svcNsNa // GetClusterInfo retrieves node info and Kubernetes version from the cluster. func (rm *ResourceManager) GetClusterInfo() (ClusterInfo, error) { + GinkgoWriter.Printf("Getting cluster info|nodes\n") ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) defer cancel() @@ -589,6 +716,7 @@ func (rm *ResourceManager) GetClusterInfo() (ClusterInfo, error) { // GetPodNames returns the names of all Pods in the specified namespace that match the given labels. func (rm *ResourceManager) GetPodNames(namespace string, labels client.MatchingLabels) ([]string, error) { + GinkgoWriter.Printf("Getting pod names in namespace %q with labels %v\n", namespace, labels) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) defer cancel() @@ -596,7 +724,7 @@ func (rm *ResourceManager) GetPodNames(namespace string, labels client.MatchingL if err := rm.K8sClient.List( ctx, &podList, - client.InNamespace(namespace), + rm.K8sClient.InNamespace(namespace), labels, ); err != nil { return nil, fmt.Errorf("error getting list of Pods: %w", err) @@ -607,12 +735,14 @@ func (rm *ResourceManager) GetPodNames(namespace string, labels client.MatchingL for _, pod := range podList.Items { names = append(names, pod.Name) } + GinkgoWriter.Printf("Found pod name in namespace %q: %v\n", namespace, names) return names, nil } // GetPods returns all Pods in the specified namespace that match the given labels. func (rm *ResourceManager) GetPods(namespace string, labels client.MatchingLabels) ([]core.Pod, error) { + GinkgoWriter.Printf("Getting pods in namespace %q with labels %v\n", namespace, labels) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) defer cancel() @@ -620,17 +750,19 @@ func (rm *ResourceManager) GetPods(namespace string, labels client.MatchingLabel if err := rm.K8sClient.List( ctx, &podList, - client.InNamespace(namespace), + rm.K8sClient.InNamespace(namespace), labels, ); err != nil { return nil, fmt.Errorf("error getting list of Pods: %w", err) } + GinkgoWriter.Printf("Found %d pods in namespace %q\n", len(podList.Items), namespace) return podList.Items, nil } // GetPod returns the Pod in the specified namespace with the given name. func (rm *ResourceManager) GetPod(namespace, name string) (*core.Pod, error) { + GinkgoWriter.Printf("Getting pod %q in namespace %q\n", name, namespace) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) defer cancel() @@ -638,6 +770,7 @@ func (rm *ResourceManager) GetPod(namespace, name string) (*core.Pod, error) { if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &pod); err != nil { return nil, fmt.Errorf("error getting Pod: %w", err) } + GinkgoWriter.Printf("Found pod %q in namespace %q\n", name, namespace) return &pod, nil } @@ -651,13 +784,27 @@ func (rm *ResourceManager) GetPodLogs(namespace, name string, opts *core.PodLogO logs, err := req.Stream(ctx) if err != nil { - return "", fmt.Errorf("error getting logs from Pod: %w", err) + getLogsErr := fmt.Errorf("error getting logs from Pod: %w", err) + GinkgoWriter.Printf("ERROR occurred during getting logs from pod %q in namespace %q, error: %s\n", + name, + namespace, + getLogsErr, + ) + + return "", getLogsErr } defer logs.Close() buf := new(bytes.Buffer) if _, err := buf.ReadFrom(logs); err != nil { - return "", fmt.Errorf("error reading logs from Pod: %w", err) + readLogsErr := fmt.Errorf("error reading logs from Pod: %w", err) + GinkgoWriter.Printf("ERROR occurred during reading logs from pod %q in namespace %q, error: %s\n", + name, + namespace, + readLogsErr, + ) + + return "", readLogsErr } return buf.String(), nil @@ -665,6 +812,7 @@ func (rm *ResourceManager) GetPodLogs(namespace, name string, opts *core.PodLogO // GetNGFDeployment returns the NGF Deployment in the specified namespace with the given release name. func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*apps.Deployment, error) { + GinkgoWriter.Printf("Getting NGF Deployment in namespace %q with release name %q\n", namespace, releaseName) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) defer cancel() @@ -673,7 +821,7 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app if err := rm.K8sClient.List( ctx, &deployments, - client.InNamespace(namespace), + rm.K8sClient.InNamespace(namespace), client.MatchingLabels{ "app.kubernetes.io/instance": releaseName, }, @@ -682,9 +830,22 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app } if len(deployments.Items) != 1 { - return nil, fmt.Errorf("expected 1 NGF Deployment, got %d", len(deployments.Items)) + deploymentsAmountErr := fmt.Errorf("expected 1 NGF Deployment, got %d", len(deployments.Items)) + GinkgoWriter.Printf("ERROR occurred during getting NGF Deployment in namespace %q with release name %q, error: %s\n", + namespace, + releaseName, + deploymentsAmountErr, + ) + + return nil, deploymentsAmountErr } + GinkgoWriter.Printf( + "Found NGF Deployment %q in namespace %q with release name %q\n", + deployments.Items[0].Name, + namespace, + releaseName, + ) deployment := deployments.Items[0] return &deployment, nil } @@ -693,6 +854,7 @@ func (rm *ResourceManager) getGatewayClassNginxProxy( namespace, releaseName string, ) (*ngfAPIv1alpha2.NginxProxy, error) { + GinkgoWriter.Printf("Getting NginxProxy in namespace %q with release name %q\n", namespace, releaseName) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) defer cancel() @@ -702,12 +864,18 @@ func (rm *ResourceManager) getGatewayClassNginxProxy( if err := rm.K8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: proxyName}, &proxy); err != nil { return nil, err } + GinkgoWriter.Printf("Successfully found NginxProxy %q in namespace %q\n", proxyName, namespace) return &proxy, nil } // ScaleNginxDeployment scales the Nginx Deployment to the specified number of replicas. func (rm *ResourceManager) ScaleNginxDeployment(namespace, releaseName string, replicas int32) error { + GinkgoWriter.Printf("Scaling Nginx Deployment in namespace %q with release name %q to %d replicas\n", + namespace, + releaseName, + replicas, + ) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.UpdateTimeout) defer cancel() @@ -715,20 +883,34 @@ func (rm *ResourceManager) ScaleNginxDeployment(namespace, releaseName string, r // may need refactoring. proxy, err := rm.getGatewayClassNginxProxy(namespace, releaseName) if err != nil { - return fmt.Errorf("error getting NginxProxy: %w", err) + getNginxProxyErr := fmt.Errorf("error getting NginxProxy: %w", err) + GinkgoWriter.Printf("ERROR occurred during getting NginxProxy in namespace %q with release name %q, error: %s\n", + namespace, + releaseName, + getNginxProxyErr, + ) + + return getNginxProxyErr } proxy.Spec.Kubernetes.Deployment.Replicas = &replicas - if err = rm.K8sClient.Update(ctx, proxy); err != nil { + if err = rm.K8sClient.Update(ctx, proxy, nil); err != nil { return fmt.Errorf("error updating NginxProxy: %w", err) } + GinkgoWriter.Printf("Successfully scaled Nginx Deployment in namespace %q with release name %q to %d replicas\n", + namespace, + releaseName, + replicas, + ) + return nil } // GetEvents returns all Events in the specified namespace. func (rm *ResourceManager) GetEvents(namespace string) (*core.EventList, error) { + GinkgoWriter.Printf("Getting Events in namespace %q\n", namespace) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) defer cancel() @@ -736,16 +918,18 @@ func (rm *ResourceManager) GetEvents(namespace string) (*core.EventList, error) if err := rm.K8sClient.List( ctx, &eventList, - client.InNamespace(namespace), + rm.K8sClient.InNamespace(namespace), ); err != nil { return &core.EventList{}, fmt.Errorf("error getting list of Events: %w", err) } + GinkgoWriter.Printf("Successfully found %d Events in namespace %q\n", len(eventList.Items), namespace) return &eventList, nil } // ScaleDeployment scales the Deployment to the specified number of replicas. func (rm *ResourceManager) ScaleDeployment(namespace, name string, replicas int32) error { + GinkgoWriter.Printf("Scaling Deployment %q in namespace %q to %d replicas\n", name, namespace, replicas) ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.UpdateTimeout) defer cancel() @@ -755,20 +939,23 @@ func (rm *ResourceManager) ScaleDeployment(namespace, name string, replicas int3 } deployment.Spec.Replicas = &replicas - if err := rm.K8sClient.Update(ctx, &deployment); err != nil { + if err := rm.K8sClient.Update(ctx, &deployment, nil); err != nil { return fmt.Errorf("error updating Deployment: %w", err) } + GinkgoWriter.Printf("Successfully scaled Deployment %q in namespace %q to %d replicas\n", name, namespace, replicas) return nil } // GetReadyNGFPodNames returns the name(s) of the NGF Pod(s). func GetReadyNGFPodNames( - k8sClient client.Client, + k8sClient K8sClient, namespace, releaseName string, timeout time.Duration, + opts ...Option, ) ([]string, error) { + GinkgoWriter.Printf("Getting ready NGF Pod names in namespace %q with release name %q\n", namespace, releaseName) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -783,7 +970,7 @@ func GetReadyNGFPodNames( if err := k8sClient.List( ctx, &podList, - client.InNamespace(namespace), + k8sClient.InNamespace(namespace), client.MatchingLabels{ "app.kubernetes.io/instance": releaseName, }, @@ -791,23 +978,39 @@ func GetReadyNGFPodNames( return false, fmt.Errorf("error getting list of NGF Pods: %w", err) } - ngfPodNames = getReadyPodNames(podList) + ngfPodNames = getReadyPodNames(podList, opts...) return len(ngfPodNames) > 0, nil }, ) if err != nil { - return nil, fmt.Errorf("timed out waiting for NGF Pods to be ready: %w", err) + waitingPodsErr := fmt.Errorf("timed out waiting for NGF Pods to be ready: %w", err) + GinkgoWriter.Printf( + "ERROR occurred during waiting for NGF Pods to be ready in namespace %q with release name %q, error: %s\n", + namespace, + releaseName, + waitingPodsErr, + ) + + return nil, waitingPodsErr } + GinkgoWriter.Printf( + "Successfully found ready NGF Pod names in namespace %q with release name %q: %v\n", + namespace, + releaseName, + ngfPodNames, + ) return ngfPodNames, nil } // GetReadyNginxPodNames returns the name(s) of the NGINX Pod(s). func GetReadyNginxPodNames( - k8sClient client.Client, + k8sClient K8sClient, namespace string, timeout time.Duration, + opts ...Option, ) ([]string, error) { + GinkgoWriter.Printf("Getting ready NGINX Pod names in namespace %q\n", namespace) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -822,24 +1025,42 @@ func GetReadyNginxPodNames( if err := k8sClient.List( ctx, &podList, - client.InNamespace(namespace), - client.HasLabels{"gateway.networking.k8s.io/gateway-name"}, + k8sClient.InNamespace(namespace), + k8sClient.HasLabels("gateway.networking.k8s.io/gateway-name"), ); err != nil { return false, fmt.Errorf("error getting list of NGINX Pods: %w", err) } - nginxPodNames = getReadyPodNames(podList) + nginxPodNames = getReadyPodNames(podList, opts...) return len(nginxPodNames) > 0, nil }, ) + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + if err != nil { - return nil, fmt.Errorf("timed out waiting for NGINX Pods to be ready: %w", err) + waitingPodsErr := fmt.Errorf("timed out waiting for NGINX Pods to be ready: %w", err) + if options.logEnabled { + GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n", + namespace, + waitingPodsErr, + ) + } + + return nil, waitingPodsErr } + GinkgoWriter.Printf( + "Successfully found ready NGINX Pod name(s) in namespace %q: %v\n", + namespace, + nginxPodNames, + ) return nginxPodNames, nil } -func getReadyPodNames(podList core.PodList) []string { +func getReadyPodNames(podList core.PodList, opts ...Option) []string { var names []string for _, pod := range podList.Items { for _, cond := range pod.Status.Conditions { @@ -848,6 +1069,13 @@ func getReadyPodNames(podList core.PodList) []string { } } } + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + if options.logEnabled { + GinkgoWriter.Printf("Found %d ready pod names: %v\n", len(names), names) + } return names } @@ -862,20 +1090,36 @@ func countNumberOfReadyParents(parents []v1.RouteParentStatus) int { } } } + GinkgoWriter.Printf("Found %d ready parent(s)\n", readyCount) return readyCount } // WaitForPodsToBeReadyWithCount waits for all Pods in the specified namespace to be ready or // until the provided context is canceled. -func (rm *ResourceManager) WaitForPodsToBeReadyWithCount(ctx context.Context, namespace string, count int) error { +func (rm *ResourceManager) WaitForPodsToBeReadyWithCount( + ctx context.Context, + namespace string, + count int, + opts ...Option, +) error { + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + GinkgoWriter.Printf("Waiting for %d pods to be ready in namespace %q\n", count, namespace) + return wait.PollUntilContextCancel( ctx, 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { var podList core.PodList - if err := rm.K8sClient.List(ctx, &podList, client.InNamespace(namespace)); err != nil { + if err := rm.K8sClient.List( + ctx, + &podList, + rm.K8sClient.InNamespace(namespace), + ); err != nil { return false, err } @@ -887,6 +1131,9 @@ func (rm *ResourceManager) WaitForPodsToBeReadyWithCount(ctx context.Context, na } } } + if options.logEnabled { + GinkgoWriter.Printf("Found %d/%d ready pods in namespace %q\n", podsReady, count, namespace) + } return podsReady == count, nil }, @@ -900,6 +1147,11 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration( name string, generation int, ) error { + GinkgoWriter.Printf("Waiting for Gateway %q in namespace %q to have ObservedGeneration %d\n", + name, + namespace, + generation, + ) return wait.PollUntilContextCancel( ctx, 500*time.Millisecond, @@ -924,7 +1176,18 @@ func (rm *ResourceManager) WaitForGatewayObservedGeneration( // GetNginxConfig uses crossplane to get the nginx configuration and convert it to JSON. // If the crossplane image is loaded locally on the node, crossplaneImageRepo can be empty. -func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneImageRepo string) (*Payload, error) { +func (rm *ResourceManager) GetNginxConfig( + nginxPodName, + namespace, + crossplaneImageRepo string, + opts ...Option, +) (*Payload, error) { + GinkgoWriter.Printf("Getting NGINX config from pod %q in namespace %q\n", nginxPodName, namespace) + options := &Options{logEnabled: true} + for _, opt := range opts { + opt(options) + } + if err := injectCrossplaneContainer( rm.ClientGoClient, rm.TimeoutConfig.UpdateTimeout, @@ -932,11 +1195,15 @@ func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneIma namespace, crossplaneImageRepo, ); err != nil { + GinkgoWriter.Printf("ERROR occurred during injecting crossplane container, error: %s\n", err) + return nil, err } exec, err := createCrossplaneExecutor(rm.ClientGoClient, rm.K8sConfig, nginxPodName, namespace) if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating crossplane executor, error: %s\n", err) + return nil, err } @@ -965,13 +1232,29 @@ func (rm *ResourceManager) GetNginxConfig(nginxPodName, namespace, crossplaneIma return true, nil }, ); err != nil { - return nil, fmt.Errorf("could not connect to ephemeral container: %w", err) + containerErr := fmt.Errorf("could not connect to ephemeral container: %w", err) + if options.logEnabled { + GinkgoWriter.Printf("ERROR occurred during waiting for NGINX Pods to be ready in namespace %q, error: %s\n", + namespace, + containerErr, + ) + } + + return nil, containerErr } conf := &Payload{} if err := json.Unmarshal(buf.Bytes(), conf); err != nil { - return nil, fmt.Errorf("error unmarshaling nginx config: %w", err) + unmarshalErr := fmt.Errorf("error unmarshaling nginx config: %w", err) + GinkgoWriter.Printf("ERROR occurred during unmarshaling nginx config from pod %q in namespace %q, error: %s\n", + nginxPodName, + namespace, + unmarshalErr, + ) + + return nil, unmarshalErr } + GinkgoWriter.Printf("Successfully got NGINX config from pod %q in namespace %q\n", nginxPodName, namespace) return conf, nil } diff --git a/tests/framework/results.go b/tests/framework/results.go index 97a4c5d57a..69ac6e32b6 100644 --- a/tests/framework/results.go +++ b/tests/framework/results.go @@ -9,25 +9,29 @@ import ( "os/exec" "path/filepath" + . "github.com/onsi/ginkgo/v2" vegeta "github.com/tsenart/vegeta/v12/lib" ) // CreateResultsDir creates and returns the name of the results directory for a test. func CreateResultsDir(testName, version string) (string, error) { - pwd, err := os.Getwd() + pwd, err := GetWorkingDir() if err != nil { return "", err } dirName := filepath.Join(filepath.Dir(pwd), "results", testName, version) - return dirName, os.MkdirAll(dirName, 0o777) + return dirName, MkdirAll(dirName, 0o777) } // CreateResultsFile creates and returns the results file for a test. func CreateResultsFile(filename string) (*os.File, error) { - outFile, err := os.OpenFile(filename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0o644) + GinkgoWriter.Printf("Creating results file %q\n", filename) + outFile, err := OpenFile(filename, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0o644) if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating results file %q, error: %s\n", filename, err) + return nil, err } @@ -59,18 +63,24 @@ func WriteSystemInfoToFile(file *os.File, ci ClusterInfo, plus bool) error { plus, commit, date, dirty, clusterType, ci.NodeCount, ci.K8sVersion, ci.CPUCountPerNode, ci.MemoryPerNode, ci.MaxPodsPerNode, ) if _, err := fmt.Fprint(file, text); err != nil { + GinkgoWriter.Printf("ERROR occurred during writing system info to results file, error: %s\n", err) + return err } if ci.IsGKE { if _, err := fmt.Fprintf(file, "- Zone: %s\n- Instance Type: %s\n", ci.GkeZone, ci.GkeInstanceType); err != nil { + GinkgoWriter.Printf("ERROR occurred during writing GKE info to results file, error: %s\n", err) + return err } } + GinkgoWriter.Printf("Wrote system info to results file\n") + return nil } func generatePNG(resultsDir, inputFilename, outputFilename, configFilename string) error { - pwd, err := os.Getwd() + pwd, err := GetWorkingDir() if err != nil { return err } @@ -83,6 +93,13 @@ func generatePNG(resultsDir, inputFilename, outputFilename, configFilename strin output, err := cmd.CombinedOutput() if err != nil { + GinkgoWriter.Printf( + "ERROR occurred during generating PNG %q using gnuplot, error: %s, output: %s\n", + outputFilename, + err, + string(output), + ) + return fmt.Errorf("failed to generate PNG: %w; output: %s", err, string(output)) } @@ -112,19 +129,116 @@ func GenerateMemoryPNG(resultsDir, inputFilename, outputFilename string) error { // WriteMetricsResults writes the metrics results to the results file in text format. func WriteMetricsResults(resultsFile *os.File, metrics *Metrics) error { reporter := vegeta.NewTextReporter(&metrics.Metrics) + reporterErr := reporter.Report(resultsFile) + if reporterErr != nil { + GinkgoWriter.Printf("ERROR occurred during writing metrics results to results file, error: %s\n", reporterErr) + } + GinkgoWriter.Printf("Wrote metrics results to results file %q\n", resultsFile.Name()) - return reporter.Report(resultsFile) + return reporterErr } // WriteContent writes basic content to the results file. func WriteContent(resultsFile *os.File, content string) error { if _, err := fmt.Fprintln(resultsFile, content); err != nil { + GinkgoWriter.Printf("ERROR occurred during writing content to results file, error: %s\n", err) + return err } return nil } +// GetWorkingDir returns the current working directory. +func GetWorkingDir() (string, error) { + pwd, err := os.Getwd() + if err != nil { + GinkgoWriter.Printf("ERROR occurred during getting current working directory %q, error: %s\n", pwd, err) + } + + return pwd, err +} + +// CreateFile creates a new file with the given name. +func CreateFile(fileName string) (*os.File, error) { + file, err := os.Create(fileName) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating file %q, error: %s\n", fileName, err) + } + + return file, err +} + +// OpenFile opens an existing file with the given name. +func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { + file, err := os.OpenFile(name, flag, perm) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during openning results file %q, error: %s\n", name, err) + } + + return file, err +} + +// MkdirAll creates a directory with the specified permissions. +func MkdirAll(path string, perm os.FileMode) error { + err := os.MkdirAll(path, perm) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during creating directory %q, error: %s\n", path, err) + } + + return err +} + +// ReadFile reads the contents of a file. +func ReadFile(file string) ([]byte, error) { + result, err := os.ReadFile(file) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during reading file %q, error: %s\n", file, err) + } + + return result, err +} + +// WriteString writes a string to the given file. +func WriteString(file *os.File, content string) (int, error) { + result, err := io.WriteString(file, content) + if err != nil { + GinkgoWriter.Printf("ERROR writing error log file: %v\n", err) + } + return result, err +} + +// WriteCSVRecord writes a CSV record using the given writer. +func WriteCSVRecord(writer *csv.Writer, record []string) error { + err := writer.Write(record) + if err != nil { + GinkgoWriter.Printf("ERROR writing CSV record: %v\n", err) + } + + return err +} + +// UserHomeDir returns the user's home directory. +func UserHomeDir() (string, error) { + dir, err := os.UserHomeDir() + if err != nil { + GinkgoWriter.Printf("ERROR getting user home directory, error: %s\n", err) + } + GinkgoWriter.Printf("User home directory is %q\n", dir) + + return dir, err +} + +// Remove removes the specified file or empty directory. +func Remove(name string) error { + err := os.Remove(name) + if err != nil { + GinkgoWriter.Printf("ERROR occurred during removing %q, error: %s\n", name, err) + } + + return err +} + // NewVegetaCSVEncoder returns a vegeta CSV encoder. func NewVegetaCSVEncoder(w io.Writer) vegeta.Encoder { return vegeta.NewCSVEncoder(w) @@ -132,18 +246,18 @@ func NewVegetaCSVEncoder(w io.Writer) vegeta.Encoder { // NewCSVResultsWriter creates and returns a CSV results file and writer. func NewCSVResultsWriter(resultsDir, fileName string, resultHeaders ...string) (*os.File, *csv.Writer, error) { - if err := os.MkdirAll(resultsDir, 0o750); err != nil { + if err := MkdirAll(resultsDir, 0o750); err != nil { return nil, nil, err } - file, err := os.OpenFile(filepath.Join(resultsDir, fileName), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) + file, err := OpenFile(filepath.Join(resultsDir, fileName), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) if err != nil { return nil, nil, err } writer := csv.NewWriter(file) - if err = writer.Write(resultHeaders); err != nil { + if err = WriteCSVRecord(writer, resultHeaders); err != nil { return nil, nil, err } diff --git a/tests/results/reconfig/edge/edge-oss.md b/tests/results/reconfig/edge/edge-oss.md index a9a5e8f46b..a904a68797 100644 --- a/tests/results/reconfig/edge/edge-oss.md +++ b/tests/results/reconfig/edge/edge-oss.md @@ -6,201 +6,100 @@ NGINX Plus: false NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: e03b181692b24863f029d591a9cac59e1d44a8b7 +- Date: 2025-09-09T14:26:43Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.33.3-gke.1136000 - vCPUs per node: 16 -- RAM per node: 65851340Ki +- RAM per node: 65851524Ki - Max pods per node: 110 - Zone: us-west1-b - Instance Type: n2d-standard-16 ## Test 1: Resources exist before startup - NumResources 30 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 3s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 2 -- NGINX Reload Average Time: 101ms -- Reload distribution: - - 500.0ms: 2 - - 1000.0ms: 2 - - 5000.0ms: 2 - - 10000.0ms: 2 - - 30000.0ms: 2 - - +Infms: 2 +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 23s ### Event Batch Processing -- Event Batch Total: 5 -- Event Batch Processing Average Time: 53ms +- Event Batch Total: 10 +- Event Batch Processing Average Time: 2ms - Event Batch Processing distribution: - - 500.0ms: 5 - - 1000.0ms: 5 - - 5000.0ms: 5 - - 10000.0ms: 5 - - 30000.0ms: 5 - - +Infms: 5 + - 500.0ms: 10 + - 1000.0ms: 10 + - 5000.0ms: 10 + - 10000.0ms: 10 + - 30000.0ms: 10 + - +Infms: 10 ### NGINX Error Logs - ## Test 1: Resources exist before startup - NumResources 150 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 3s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 2 -- NGINX Reload Average Time: 88ms -- Reload distribution: - - 500.0ms: 2 - - 1000.0ms: 2 - - 5000.0ms: 2 - - 10000.0ms: 2 - - 30000.0ms: 2 - - +Infms: 2 +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 21s ### Event Batch Processing -- Event Batch Total: 6 -- Event Batch Processing Average Time: 45ms +- Event Batch Total: 8 +- Event Batch Processing Average Time: 8ms - Event Batch Processing distribution: - - 500.0ms: 6 - - 1000.0ms: 6 - - 5000.0ms: 6 - - 10000.0ms: 6 - - 30000.0ms: 6 - - +Infms: 6 + - 500.0ms: 8 + - 1000.0ms: 8 + - 5000.0ms: 8 + - 10000.0ms: 8 + - 30000.0ms: 8 + - +Infms: 8 ### NGINX Error Logs +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 30 -## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 30 - -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 8s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 63 -- NGINX Reload Average Time: 125ms -- Reload distribution: - - 500.0ms: 63 - - 1000.0ms: 63 - - 5000.0ms: 63 - - 10000.0ms: 63 - - 30000.0ms: 63 - - +Infms: 63 +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 26s ### Event Batch Processing -- Event Batch Total: 337 -- Event Batch Processing Average Time: 23ms +- Event Batch Total: 263 +- Event Batch Processing Average Time: 31ms - Event Batch Processing distribution: - - 500.0ms: 337 - - 1000.0ms: 337 - - 5000.0ms: 337 - - 10000.0ms: 337 - - 30000.0ms: 337 - - +Infms: 337 + - 500.0ms: 256 + - 1000.0ms: 263 + - 5000.0ms: 263 + - 10000.0ms: 263 + - 30000.0ms: 263 + - +Infms: 263 ### NGINX Error Logs +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 150 -## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 150 - -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 44s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 343 -- NGINX Reload Average Time: 125ms -- Reload distribution: - - 500.0ms: 343 - - 1000.0ms: 343 - - 5000.0ms: 343 - - 10000.0ms: 343 - - 30000.0ms: 343 - - +Infms: 343 +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 102s ### Event Batch Processing -- Event Batch Total: 1689 -- Event Batch Processing Average Time: 25ms -- Event Batch Processing distribution: - - 500.0ms: 1689 - - 1000.0ms: 1689 - - 5000.0ms: 1689 - - 10000.0ms: 1689 - - 30000.0ms: 1689 - - +Infms: 1689 - -### NGINX Error Logs - - -## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 30 - -### Reloads and Time to Ready - -- TimeToReadyTotal: < 1s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 64 -- NGINX Reload Average Time: 125ms -- Reload distribution: - - 500.0ms: 64 - - 1000.0ms: 64 - - 5000.0ms: 64 - - 10000.0ms: 64 - - 30000.0ms: 64 - - +Infms: 64 - -### Event Batch Processing - -- Event Batch Total: 321 -- Event Batch Processing Average Time: 25ms -- Event Batch Processing distribution: - - 500.0ms: 321 - - 1000.0ms: 321 - - 5000.0ms: 321 - - 10000.0ms: 321 - - 30000.0ms: 321 - - +Infms: 321 - -### NGINX Error Logs - - -## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 150 - -### Reloads and Time to Ready - -- TimeToReadyTotal: < 1s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 342 -- NGINX Reload Average Time: 125ms -- Reload distribution: - - 500.0ms: 342 - - 1000.0ms: 342 - - 5000.0ms: 342 - - 10000.0ms: 342 - - 30000.0ms: 342 - - +Infms: 342 - -### Event Batch Processing - -- Event Batch Total: 1639 -- Event Batch Processing Average Time: 26ms +- Event Batch Total: 1240 +- Event Batch Processing Average Time: 23ms - Event Batch Processing distribution: - - 500.0ms: 1639 - - 1000.0ms: 1639 - - 5000.0ms: 1639 - - 10000.0ms: 1639 - - 30000.0ms: 1639 - - +Infms: 1639 + - 500.0ms: 1209 + - 1000.0ms: 1240 + - 5000.0ms: 1240 + - 10000.0ms: 1240 + - 30000.0ms: 1240 + - +Infms: 1240 ### NGINX Error Logs diff --git a/tests/results/reconfig/edge/edge-plus.md b/tests/results/reconfig/edge/edge-plus.md index b339fb3c0d..d8e6dd846e 100644 --- a/tests/results/reconfig/edge/edge-plus.md +++ b/tests/results/reconfig/edge/edge-plus.md @@ -6,202 +6,100 @@ NGINX Plus: true NGINX Gateway Fabric: -- Commit: 9155a2b6a8d3179165797ef3e789e97283f7a695 -- Date: 2025-03-15T07:17:11Z +- Commit: e03b181692b24863f029d591a9cac59e1d44a8b7 +- Date: 2025-09-09T14:26:43Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.31.6-gke.1020000 +- k8s version: v1.33.3-gke.1136000 - vCPUs per node: 16 -- RAM per node: 65851340Ki +- RAM per node: 65851524Ki - Max pods per node: 110 - Zone: us-west1-b - Instance Type: n2d-standard-16 ## Test 1: Resources exist before startup - NumResources 30 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 4s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 2 -- NGINX Reload Average Time: 100ms -- Reload distribution: - - 500.0ms: 2 - - 1000.0ms: 2 - - 5000.0ms: 2 - - 10000.0ms: 2 - - 30000.0ms: 2 - - +Infms: 2 +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 16s ### Event Batch Processing -- Event Batch Total: 6 -- Event Batch Processing Average Time: 52ms +- Event Batch Total: 8 +- Event Batch Processing Average Time: 18ms - Event Batch Processing distribution: - - 500.0ms: 6 - - 1000.0ms: 6 - - 5000.0ms: 6 - - 10000.0ms: 6 - - 30000.0ms: 6 - - +Infms: 6 + - 500.0ms: 8 + - 1000.0ms: 8 + - 5000.0ms: 8 + - 10000.0ms: 8 + - 30000.0ms: 8 + - +Infms: 8 ### NGINX Error Logs - ## Test 1: Resources exist before startup - NumResources 150 -### Reloads and Time to Ready - -- TimeToReadyTotal: 4s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 2 -- NGINX Reload Average Time: 100ms -- Reload distribution: - - 500.0ms: 2 - - 1000.0ms: 2 - - 5000.0ms: 2 - - 10000.0ms: 2 - - 30000.0ms: 2 - - +Infms: 2 - -### Event Batch Processing - -- Event Batch Total: 6 -- Event Batch Processing Average Time: 53ms -- Event Batch Processing distribution: - - 500.0ms: 6 - - 1000.0ms: 6 - - 5000.0ms: 6 - - 10000.0ms: 6 - - 30000.0ms: 6 - - +Infms: 6 - -### NGINX Error Logs - +### Time to Ready -## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 30 - -### Reloads and Time to Ready - -- TimeToReadyTotal: 8s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 47 -- NGINX Reload Average Time: 148ms -- Reload distribution: - - 500.0ms: 47 - - 1000.0ms: 47 - - 5000.0ms: 47 - - 10000.0ms: 47 - - 30000.0ms: 47 - - +Infms: 47 +Time To Ready Description: From when NGF starts to when the NGINX configuration is fully configured +- TimeToReadyTotal: 22s ### Event Batch Processing -- Event Batch Total: 322 -- Event Batch Processing Average Time: 25ms +- Event Batch Total: 8 +- Event Batch Processing Average Time: 21ms - Event Batch Processing distribution: - - 500.0ms: 322 - - 1000.0ms: 322 - - 5000.0ms: 322 - - 10000.0ms: 322 - - 30000.0ms: 322 - - +Infms: 322 + - 500.0ms: 8 + - 1000.0ms: 8 + - 5000.0ms: 8 + - 10000.0ms: 8 + - 30000.0ms: 8 + - +Infms: 8 ### NGINX Error Logs +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 30 -## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 150 - -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: 20s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 117 -- NGINX Reload Average Time: 150ms -- Reload distribution: - - 500.0ms: 117 - - 1000.0ms: 117 - - 5000.0ms: 117 - - 10000.0ms: 117 - - 30000.0ms: 117 - - +Infms: 117 +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 24s ### Event Batch Processing -- Event Batch Total: 1460 -- Event Batch Processing Average Time: 14ms +- Event Batch Total: 226 +- Event Batch Processing Average Time: 57ms - Event Batch Processing distribution: - - 500.0ms: 1460 - - 1000.0ms: 1460 - - 5000.0ms: 1460 - - 10000.0ms: 1460 - - 30000.0ms: 1460 - - +Infms: 1460 + - 500.0ms: 220 + - 1000.0ms: 221 + - 5000.0ms: 226 + - 10000.0ms: 226 + - 30000.0ms: 226 + - +Infms: 226 ### NGINX Error Logs -2025/03/15 17:00:26 [emerg] 48#48: invalid instance state file "/var/lib/nginx/state/nginx-mgmt-state" - - -## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 30 - -### Reloads and Time to Ready - -- TimeToReadyTotal: < 1s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 46 -- NGINX Reload Average Time: 133ms -- Reload distribution: - - 500.0ms: 46 - - 1000.0ms: 46 - - 5000.0ms: 46 - - 10000.0ms: 46 - - 30000.0ms: 46 - - +Infms: 46 - -### Event Batch Processing - -- Event Batch Total: 291 -- Event Batch Processing Average Time: 28ms -- Event Batch Processing distribution: - - 500.0ms: 291 - - 1000.0ms: 291 - - 5000.0ms: 291 - - 10000.0ms: 291 - - 30000.0ms: 291 - - +Infms: 291 - -### NGINX Error Logs - -## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 150 +## Test 2: Start NGF, deploy Gateway, wait until NGINX agent instance connects to NGF, create many resources attached to GW - NumResources 150 -### Reloads and Time to Ready +### Time to Ready -- TimeToReadyTotal: < 1s -- TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 258 -- NGINX Reload Average Time: 132ms -- Reload distribution: - - 500.0ms: 258 - - 1000.0ms: 258 - - 5000.0ms: 258 - - 10000.0ms: 258 - - 30000.0ms: 258 - - +Infms: 258 +Time To Ready Description: From when NGINX receives the first configuration created by NGF to when the NGINX configuration is fully configured +- TimeToReadyTotal: 128s ### Event Batch Processing -- Event Batch Total: 1501 -- Event Batch Processing Average Time: 29ms +- Event Batch Total: 1285 +- Event Batch Processing Average Time: 30ms - Event Batch Processing distribution: - - 500.0ms: 1501 - - 1000.0ms: 1501 - - 5000.0ms: 1501 - - 10000.0ms: 1501 - - 30000.0ms: 1501 - - +Infms: 1501 + - 500.0ms: 1260 + - 1000.0ms: 1266 + - 5000.0ms: 1285 + - 10000.0ms: 1285 + - 30000.0ms: 1285 + - +Infms: 1285 ### NGINX Error Logs diff --git a/tests/suite/advanced_routing_test.go b/tests/suite/advanced_routing_test.go index 465654246f..f53a1615dc 100644 --- a/tests/suite/advanced_routing_test.go +++ b/tests/suite/advanced_routing_test.go @@ -38,9 +38,13 @@ var _ = Describe("AdvancedRouting", Ordered, Label("functional", "routing"), fun Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := framework.GetReadyNginxPodNames( + resourceManager.K8sClient, + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -116,17 +120,23 @@ func expectRequestToRespondFromExpectedServer( appURL, address, expServerName string, headers, queryParams map[string]string, ) error { + GinkgoWriter.Printf("Expecting request to respond from the server %q\n", expServerName) status, body, err := framework.Get(appURL, address, timeoutConfig.RequestTimeout, headers, queryParams) if err != nil { return err } if status != http.StatusOK { - return errors.New("http status was not 200") + statusErr := errors.New("http status was not 200") + GinkgoWriter.Printf("ERROR: %v\n", statusErr) + + return statusErr } actualServerName, err := extractServerName(body) if err != nil { + GinkgoWriter.Printf("ERROR extracting server name from response body: %v\n", err) + return err } diff --git a/tests/suite/client_settings_test.go b/tests/suite/client_settings_test.go index 557146b789..4963484892 100644 --- a/tests/suite/client_settings_test.go +++ b/tests/suite/client_settings_test.go @@ -47,7 +47,11 @@ var _ = Describe("ClientSettingsPolicy", Ordered, Label("functional", "cspolicy" Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := framework.GetReadyNginxPodNames( + resourceManager.K8sClient, + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -393,7 +397,7 @@ func waitForClientSettingsAncestorStatus( func(ctx context.Context) (bool, error) { var pol ngfAPI.ClientSettingsPolicy - if err := k8sClient.Get(ctx, policyNsname, &pol); err != nil { + if err := resourceManager.K8sClient.Get(ctx, policyNsname, &pol); err != nil { return false, err } @@ -404,7 +408,10 @@ func waitForClientSettingsAncestorStatus( } if len(pol.Status.Ancestors) != 1 { - return false, fmt.Errorf("policy has %d ancestors, expected 1", len(pol.Status.Ancestors)) + ancestorsAmountErr := fmt.Errorf("policy has %d ancestors, expected 1", len(pol.Status.Ancestors)) + GinkgoWriter.Printf("ERROR: %v\n", ancestorsAmountErr) + + return false, ancestorsAmountErr } ancestor := pol.Status.Ancestors[0] @@ -425,20 +432,33 @@ func ancestorStatusMustHaveAcceptedCondition( condStatus metav1.ConditionStatus, condReason v1alpha2.PolicyConditionReason, ) error { + GinkgoWriter.Printf("Checking if ancestor status has accepted condition\n") if len(status.Conditions) != 1 { - return fmt.Errorf("expected 1 condition in status, got %d", len(status.Conditions)) + tooManyConditionsErr := fmt.Errorf("expected 1 condition in status, got %d", len(status.Conditions)) + GinkgoWriter.Printf("ERROR: %v\n", tooManyConditionsErr) + + return tooManyConditionsErr } if status.Conditions[0].Type != string(v1alpha2.RouteConditionAccepted) { - return fmt.Errorf("expected condition type to be Accepted, got %s", status.Conditions[0].Type) + wrongTypeErr := fmt.Errorf("expected condition type to be Accepted, got %s", status.Conditions[0].Type) + GinkgoWriter.Printf("ERROR: %v\n", wrongTypeErr) + + return wrongTypeErr } if status.Conditions[0].Status != condStatus { - return fmt.Errorf("expected condition status to be %s, got %s", condStatus, status.Conditions[0].Status) + wrongStatusErr := fmt.Errorf("expected condition status to be %s, got %s", condStatus, status.Conditions[0].Status) + GinkgoWriter.Printf("ERROR: %v\n", wrongStatusErr) + + return wrongStatusErr } if status.Conditions[0].Reason != string(condReason) { - return fmt.Errorf("expected condition reason to be %s, got %s", condReason, status.Conditions[0].Reason) + wrongReasonErr := fmt.Errorf("expected condition reason to be %s, got %s", condReason, status.Conditions[0].Reason) + GinkgoWriter.Printf("ERROR: %v\n", wrongReasonErr) + + return wrongReasonErr } return nil @@ -450,46 +470,79 @@ func ancestorMustEqualTargetRef( namespace string, ) error { if ancestor.ControllerName != ngfControllerName { - return fmt.Errorf( + controllerNameErr := fmt.Errorf( "expected ancestor controller name to be %s, got %s", ngfControllerName, ancestor.ControllerName, ) + GinkgoWriter.Printf("ERROR: %v\n", controllerNameErr) + + return controllerNameErr } if ancestor.AncestorRef.Namespace == nil { - return fmt.Errorf("expected ancestor namespace to be %s, got nil", namespace) + nsErr := fmt.Errorf("expected ancestor namespace to be %s, got nil", namespace) + GinkgoWriter.Printf("ERROR: %v\n", nsErr) + + return nsErr } if string(*ancestor.AncestorRef.Namespace) != namespace { - return fmt.Errorf( + wrongNSErr := fmt.Errorf( "expected ancestor namespace to be %s, got %s", namespace, string(*ancestor.AncestorRef.Namespace), ) + GinkgoWriter.Printf("ERROR: %v\n", wrongNSErr) + + return wrongNSErr } ancestorRef := ancestor.AncestorRef if ancestorRef.Name != targetRef.Name { - return fmt.Errorf("expected ancestorRef to have name %s, got %s", targetRef.Name, ancestorRef.Name) + wrongNameErr := fmt.Errorf("expected ancestorRef to have name %s, got %s", targetRef.Name, ancestorRef.Name) + GinkgoWriter.Printf("ERROR: %v\n", wrongNameErr) + + return wrongNameErr } if ancestorRef.Group == nil { - return fmt.Errorf("expected ancestorRef to have group %s, got nil", targetRef.Group) + emptyGroupErr := fmt.Errorf("expected ancestorRef to have group %s, got nil", targetRef.Group) + GinkgoWriter.Printf("ERROR: %v\n", emptyGroupErr) + + return emptyGroupErr } if *ancestorRef.Group != targetRef.Group { - return fmt.Errorf("expected ancestorRef to have group %s, got %s", targetRef.Group, string(*ancestorRef.Group)) + wrongGroupErr := fmt.Errorf( + "expected ancestorRef to have group %s, got %s", + targetRef.Group, + string(*ancestorRef.Group), + ) + GinkgoWriter.Printf("ERROR: %v\n", wrongGroupErr) + + return wrongGroupErr } if ancestorRef.Kind == nil { - return fmt.Errorf("expected ancestorRef to have kind %s, got nil", targetRef.Kind) + noKindErr := fmt.Errorf("expected ancestorRef to have kind %s, got nil", targetRef.Kind) + GinkgoWriter.Printf("ERROR: %v\n", noKindErr) + + return noKindErr } if *ancestorRef.Kind != targetRef.Kind { - return fmt.Errorf("expected ancestorRef to have kind %s, got %s", targetRef.Kind, string(*ancestorRef.Kind)) + wrongKindErr := fmt.Errorf( + "expected ancestorRef to have kind %s, got %s", + targetRef.Kind, + string(*ancestorRef.Kind), + ) + GinkgoWriter.Printf("ERROR: %v\n", wrongKindErr) + + return wrongKindErr } + GinkgoWriter.Printf("Success: AncestorRef %q matches TargetRef\n", ancestorRef.Name) return nil } diff --git a/tests/suite/dataplane_perf_test.go b/tests/suite/dataplane_perf_test.go index cce009ab03..c4244ded9c 100644 --- a/tests/suite/dataplane_perf_test.go +++ b/tests/suite/dataplane_perf_test.go @@ -64,9 +64,14 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"), Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) - - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetTimeout) + Expect(resourceManager.WaitForAppsToBeReady(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) + + nginxPodNames, err := framework.GetReadyNginxPodNames( + k8sClient, + namespace, + timeoutConfig.GetTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -88,7 +93,7 @@ var _ = Describe("Dataplane performance", Ordered, Label("nfr", "performance"), }) AfterAll(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) + framework.AddNginxLogsAndEventsToReport(resourceManager, namespace, framework.WithLoggingDisabled()) cleanUpPortForward() Expect(resourceManager.DeleteFromFiles(files, namespace)).To(Succeed()) diff --git a/tests/suite/graceful_recovery_test.go b/tests/suite/graceful_recovery_test.go index ca50c2f6c3..fca1c33720 100644 --- a/tests/suite/graceful_recovery_test.go +++ b/tests/suite/graceful_recovery_test.go @@ -74,7 +74,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra defer cancel() var pod core.Pod - if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: podName}, &pod); err != nil { + if err := resourceManager.K8sClient.Get( + ctx, + types.NamespacedName{Namespace: namespace, Name: podName}, + &pod, + ); err != nil { return 0, fmt.Errorf("error retrieving Pod: %w", err) } @@ -95,8 +99,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra } if restartCount != currentRestartCount+1 { - return fmt.Errorf("expected current restart count: %d to match incremented restart count: %d", + restartErr := fmt.Errorf("expected current restart count: %d to match incremented restart count: %d", restartCount, currentRestartCount+1) + GinkgoWriter.Printf("%s\n", restartErr) + + return restartErr } return nil @@ -107,7 +114,7 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra defer cancel() var nodes core.NodeList - if err := k8sClient.List(ctx, &nodes); err != nil { + if err := resourceManager.K8sClient.List(ctx, &nodes, nil); err != nil { return nil, fmt.Errorf("error listing nodes: %w", err) } @@ -125,31 +132,47 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra defer cancel() var nginxPod core.Pod - if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns.Name, Name: nginxPodName}, &nginxPod); err != nil { + if err := resourceManager.K8sClient.Get( + ctx, + types.NamespacedName{Namespace: ns.Name, Name: nginxPodName}, + &nginxPod, + ); err != nil { return nil, fmt.Errorf("error retrieving nginx Pod: %w", err) } b, err := resourceManager.GetFileContents("graceful-recovery/node-debugger-job.yaml") if err != nil { - return nil, fmt.Errorf("error processing node debugger job file: %w", err) + debugErr := fmt.Errorf("error processing node debugger job file: %w", err) + GinkgoWriter.Printf("%s\n", debugErr) + + return nil, debugErr } job := &v1.Job{} if err = yaml.Unmarshal(b.Bytes(), job); err != nil { - return nil, fmt.Errorf("error with yaml unmarshal: %w", err) + yamlErr := fmt.Errorf("error with yaml unmarshal: %w", err) + GinkgoWriter.Printf("%s\n", yamlErr) + + return nil, yamlErr } job.Spec.Template.Spec.NodeSelector["kubernetes.io/hostname"] = nginxPod.Spec.NodeName if len(job.Spec.Template.Spec.Containers) != 1 { - return nil, fmt.Errorf( + containerErr := fmt.Errorf( "expected node debugger job to contain one container, actual number: %d", len(job.Spec.Template.Spec.Containers), ) + GinkgoWriter.Printf("ERROR: %s\n", containerErr) + + return nil, containerErr } job.Namespace = ns.Name if err = resourceManager.Apply([]client.Object{job}); err != nil { - return nil, fmt.Errorf("error in applying job: %w", err) + jobErr := fmt.Errorf("error in applying job: %w", err) + GinkgoWriter.Printf("%s\n", jobErr) + + return nil, jobErr } return job, nil @@ -207,7 +230,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = framework.GetReadyNginxPodNames( + k8sClient, + ns.Name, + timeoutConfig.GetStatusTimeout, + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -286,6 +313,7 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), ) return len(podNames) == 1 && err == nil }). @@ -307,7 +335,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra var nginxPodNames []string Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = framework.GetReadyNginxPodNames( + resourceManager.K8sClient, + ns.Name, + timeoutConfig.GetStatusTimeout, + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout * 2). @@ -352,12 +384,15 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra var lease coordination.Lease key := types.NamespacedName{Name: "ngf-test-nginx-gateway-fabric-leader-election", Namespace: ngfNamespace} - if err := k8sClient.Get(ctx, key, &lease); err != nil { + if err := resourceManager.K8sClient.Get(ctx, key, &lease); err != nil { return "", errors.New("could not retrieve leader election lease") } if *lease.Spec.HolderIdentity == "" { - return "", errors.New("leader election lease holder identity is empty") + leaderErr := errors.New("leader election lease holder identity is empty") + GinkgoWriter.Printf("ERROR: %s\n", leaderErr) + + return "", leaderErr } return *lease.Spec.HolderIdentity, nil @@ -381,7 +416,13 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra } BeforeAll(func() { - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout) + podNames, err := framework.GetReadyNGFPodNames( + resourceManager.K8sClient, + ngfNamespace, + releaseName, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) @@ -397,7 +438,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := framework.GetReadyNginxPodNames( + resourceManager.K8sClient, + ns.Name, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -431,7 +476,11 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra It("recovers when nginx container is restarted", func() { restartNginxContainer(activeNginxPodName, ns.Name, nginxContainerName) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := framework.GetReadyNginxPodNames( + resourceManager.K8sClient, + ns.Name, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) activeNginxPodName = nginxPodNames[0] @@ -460,16 +509,17 @@ var _ = Describe("Graceful Recovery test", Ordered, FlakeAttempts(2), Label("gra ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout) defer cancel() - Expect(k8sClient.Delete(ctx, ngfPod)).To(Succeed()) + Expect(resourceManager.K8sClient.Delete(ctx, ngfPod, nil)).To(Succeed()) var newNGFPodNames []string Eventually( func() bool { newNGFPodNames, err = framework.GetReadyNGFPodNames( - k8sClient, + resourceManager.K8sClient, ngfNamespace, releaseName, timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), ) return len(newNGFPodNames) == 1 && err == nil }). @@ -531,6 +581,14 @@ func expectRequestToSucceed(appURL, address string, responseBodyMessage string) return err } +// The function is expecting the request to fail (hence the name) because NGINX is not there to route the request. +// The purpose of the graceful recovery test is to simulate various failure scenarios including NGINX +// container restarts, NGF pod restarts, and Kubernetes node restarts to show the system can recover +// after these real world scenarios and resume serving application traffic after recovery. +// In this case, we verify that our requests fail and then that eventually are successful again - verifying that +// NGINX went down and came back up again. +// We only want an error returned from this particular function if it does not appear that NGINX has +// stopped serving traffic. func expectRequestToFail(appURL, address string) error { status, body, err := framework.Get(appURL, address, timeoutConfig.RequestTimeout, nil, nil) if status != 0 { diff --git a/tests/suite/longevity_test.go b/tests/suite/longevity_test.go index 8734ffa5bc..6b0461bcdd 100644 --- a/tests/suite/longevity_test.go +++ b/tests/suite/longevity_test.go @@ -56,7 +56,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.ApplyFromFiles(promFile, ngfNamespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) }) It("collects results", Label("longevity-teardown"), func() { @@ -75,14 +75,14 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu Expect(framework.WriteSystemInfoToFile(resultsFile, clusterInfo, *plusEnabled)).To(Succeed()) // gather wrk output - homeDir, err := os.UserHomeDir() + homeDir, err := framework.UserHomeDir() Expect(err).ToNot(HaveOccurred()) Expect(framework.WriteContent(resultsFile, "\n## Traffic\n")).To(Succeed()) Expect(writeTrafficResults(resultsFile, homeDir, "coffee.txt", "HTTP")).To(Succeed()) Expect(writeTrafficResults(resultsFile, homeDir, "tea.txt", "HTTPS")).To(Succeed()) - framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) + framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name, framework.WithLoggingDisabled()) Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.DeleteNamespace(ns.Name)).To(Succeed()) }) @@ -90,7 +90,7 @@ var _ = Describe("Longevity", Label("longevity-setup", "longevity-teardown"), fu func writeTrafficResults(resultsFile *os.File, homeDir, filename, testname string) error { file := fmt.Sprintf("%s/%s", homeDir, filename) - content, err := os.ReadFile(file) + content, err := framework.ReadFile(file) if err != nil { return err } diff --git a/tests/suite/nginxgateway_test.go b/tests/suite/nginxgateway_test.go index 9c3497d891..ab7ce27e03 100644 --- a/tests/suite/nginxgateway_test.go +++ b/tests/suite/nginxgateway_test.go @@ -34,7 +34,7 @@ var _ = Describe("NginxGateway", Ordered, Label("functional", "nginxGateway"), f var nginxGateway ngfAPI.NginxGateway - if err := k8sClient.Get(ctx, nsname, &nginxGateway); err != nil { + if err := resourceManager.K8sClient.Get(ctx, nsname, &nginxGateway); err != nil { return nginxGateway, fmt.Errorf("failed to get nginxGateway: %w", err) } @@ -43,14 +43,20 @@ var _ = Describe("NginxGateway", Ordered, Label("functional", "nginxGateway"), f verifyNginxGatewayConditions := func(ng ngfAPI.NginxGateway) error { if ng.Status.Conditions == nil { - return errors.New("nginxGateway has no conditions") + noConditionsErr := errors.New("nginxGateway has no conditions") + GinkgoWriter.Printf("ERROR: %v\n", noConditionsErr) + + return noConditionsErr } if len(ng.Status.Conditions) != 1 { - return fmt.Errorf( + tooManyConditionsErr := fmt.Errorf( "expected nginxGateway to have only one condition, instead has %d conditions", len(ng.Status.Conditions), ) + GinkgoWriter.Printf("ERROR: %v\n", tooManyConditionsErr) + + return tooManyConditionsErr } return nil @@ -72,22 +78,30 @@ var _ = Describe("NginxGateway", Ordered, Label("functional", "nginxGateway"), f condition := ng.Status.Conditions[0] if condition.Type != "Valid" { - return fmt.Errorf( + invalidConditionTypeErr := fmt.Errorf( "expected nginxGateway condition type to be Valid, instead has type %s", condition.Type, ) + GinkgoWriter.Printf("ERROR: %v\n", invalidConditionTypeErr) + + return invalidConditionTypeErr } if condition.Reason != "Valid" { - return fmt.Errorf("expected nginxGateway reason to be Valid, instead is %s", condition.Reason) + invalidReasonErr := fmt.Errorf("expected nginxGateway reason to be Valid, instead is %s", condition.Reason) + GinkgoWriter.Printf("ERROR: %v\n", invalidReasonErr) + + return invalidReasonErr } if condition.ObservedGeneration != expObservedGen { - return fmt.Errorf( + observedGenerationErr := fmt.Errorf( "expected nginxGateway observed generation to be %d, instead is %d", expObservedGen, condition.ObservedGeneration, ) + GinkgoWriter.Printf("ERROR: %v\n", observedGenerationErr) + return observedGenerationErr } return nil @@ -105,7 +119,10 @@ var _ = Describe("NginxGateway", Ordered, Label("functional", "nginxGateway"), f } if len(podNames) != 1 { - return "", fmt.Errorf("expected 1 pod name, got %d", len(podNames)) + tooManyPodsErr := fmt.Errorf("expected 1 pod name, got %d", len(podNames)) + GinkgoWriter.Printf("ERROR: %v\n", tooManyPodsErr) + + return "", tooManyPodsErr } return podNames[0], nil diff --git a/tests/suite/reconfig_test.go b/tests/suite/reconfig_test.go index ab990e1808..46e24b635e 100644 --- a/tests/suite/reconfig_test.go +++ b/tests/suite/reconfig_test.go @@ -66,7 +66,11 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r k8sConfig := ctlr.GetConfigOrDie() if !clusterInfo.IsGKE { - Expect(promInstance.PortForward(k8sConfig, promPortForwardStopCh)).To(Succeed()) + pfErr := promInstance.PortForward(k8sConfig, promPortForwardStopCh) + if pfErr != nil { + GinkgoWriter.Printf("ERROR occurred during port-forwarding Prometheus, error: %s\n", pfErr) + } + Expect(pfErr).NotTo(HaveOccurred()) } }) @@ -85,6 +89,8 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r }) createUniqueResources := func(resourceCount int, fileName string) error { + GinkgoWriter.Printf("Creating %d unique resources from %s\n", resourceCount, fileName) + var appliedResources []string for i := 1; i <= resourceCount; i++ { namespace := "namespace" + strconv.Itoa(i) @@ -98,11 +104,24 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r fileString = strings.ReplaceAll(fileString, "tea", "tea"+namespace) data := bytes.NewBufferString(fileString) - - if err := resourceManager.ApplyFromBuffer(data, namespace); err != nil { + appliedResources = append(appliedResources, namespace) + + if err := resourceManager.ApplyFromBuffer(data, namespace, framework.WithLoggingDisabled()); err != nil { + GinkgoWriter.Printf( + "ERROR on creating and applying unique resources, could proceed %v\n the error happened on %q: %v\n", + appliedResources, + namespace, + err, + ) return fmt.Errorf("error processing manifest file: %w", err) } } + GinkgoWriter.Printf( + "Successfully created %d unique resources from %s: %v\n", + resourceCount, + fileName, + appliedResources, + ) return nil } @@ -117,7 +136,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r Name: "namespace" + strconv.Itoa(i), }, } - Expect(k8sClient.Create(ctx, &ns)).To(Succeed()) + Expect(resourceManager.K8sClient.Create(ctx, &ns)).To(Succeed()) } Expect(resourceManager.Apply([]client.Object{&reconfigNamespace})).To(Succeed()) @@ -126,7 +145,9 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r "reconfig/cafe-secret.yaml", "reconfig/reference-grant.yaml", }, - reconfigNamespace.Name)).To(Succeed()) + reconfigNamespace.Name, + framework.WithLoggingDisabled(), + )).To(Succeed()) Expect(createUniqueResources(resourceCount, "manifests/reconfig/cafe.yaml")).To(Succeed()) @@ -138,7 +159,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r Name: "namespace" + strconv.Itoa(i), }, } - Expect(resourceManager.WaitForPodsToBeReady(ctx, ns.Name)).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReady(ctx, ns.Name, framework.WithLoggingDisabled())).To(Succeed()) } } @@ -147,21 +168,24 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r defer cancel() var namespaces core.NamespaceList - if err := k8sClient.List(ctx, &namespaces); err != nil { + if err := resourceManager.K8sClient.List(ctx, &namespaces, nil); err != nil { return fmt.Errorf("error getting namespaces: %w", err) } + GinkgoWriter.Printf("Found %d namespaces, expected at least%d\n", len(namespaces.Items), resourceCount) Expect(len(namespaces.Items)).To(BeNumerically(">=", resourceCount)) var routes v1.HTTPRouteList - if err := k8sClient.List(ctx, &routes); err != nil { + if err := resourceManager.K8sClient.List(ctx, &routes, nil); err != nil { return fmt.Errorf("error getting HTTPRoutes: %w", err) } + GinkgoWriter.Printf("Found %d HTTPRoutes, expected %d\n", len(routes.Items), resourceCount*3) Expect(routes.Items).To(HaveLen(resourceCount * 3)) var pods core.PodList - if err := k8sClient.List(ctx, &pods); err != nil { + if err := resourceManager.K8sClient.List(ctx, &pods, nil); err != nil { return fmt.Errorf("error getting Pods: %w", err) } + GinkgoWriter.Printf("Found %d Pods, expected at least %d\n", len(pods.Items), resourceCount*2) Expect(len(pods.Items)).To(BeNumerically(">=", resourceCount*2)) return nil @@ -175,7 +199,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r namespaces[i] = "namespace" + strconv.Itoa(i+1) } - err = resourceManager.DeleteNamespaces(namespaces) + err = resourceManager.DeleteNamespaces(namespaces, framework.WithLoggingDisabled()) Expect(resourceManager.DeleteNamespace(reconfigNamespace.Name)).To(Succeed()) return err @@ -196,7 +220,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r } // each call to ValidateNginxFieldExists takes about 1ms - if err := framework.ValidateNginxFieldExists(conf, expUpstream); err != nil { + if err := framework.ValidateNginxFieldExists(conf, expUpstream, framework.WithLoggingDisabled()); err != nil { select { case <-ctx.Done(): return fmt.Errorf("error validating nginx conf was generated in "+namespace+": %w", err.Error()) @@ -222,6 +246,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r if stringTimeToReadyTotal == "0" { stringTimeToReadyTotal = "< 1" } + GinkgoWriter.Printf("Calculated time to ready total for %q: %s\n", nginxPodName, stringTimeToReadyTotal) return stringTimeToReadyTotal } @@ -327,6 +352,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r k8sClient, reconfigNamespace.Name, timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -371,7 +397,13 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r cfg.nfr = true setup(cfg) - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + podNames, err := framework.GetReadyNGFPodNames( + k8sClient, + ngfNamespace, + releaseName, + timeoutConfig.GetTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) ngfPodName := podNames[0] @@ -386,6 +418,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r k8sClient, reconfigNamespace.Name, timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), ) return len(nginxPodNames) == 1 && err == nil }). @@ -406,7 +439,7 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r File: "http.conf", } - return framework.ValidateNginxFieldExists(conf, defaultUpstream) == nil + return framework.ValidateNginxFieldExists(conf, defaultUpstream, framework.WithLoggingDisabled()) == nil }). WithTimeout(timeoutConfig.CreateTimeout). Should(BeTrue()) @@ -439,7 +472,11 @@ var _ = Describe("Reconfiguration Performance Testing", Ordered, Label("nfr", "r }) AfterEach(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, reconfigNamespace.Name) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + reconfigNamespace.Name, + framework.WithLoggingDisabled(), + ) Expect(cleanupResources()).Should(Succeed()) teardown(releaseName) diff --git a/tests/suite/sample_test.go b/tests/suite/sample_test.go index a79dda5a7d..976bb9f09a 100644 --- a/tests/suite/sample_test.go +++ b/tests/suite/sample_test.go @@ -38,7 +38,11 @@ var _ = Describe("Basic test example", Label("functional"), func() { Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := framework.GetReadyNginxPodNames( + resourceManager.K8sClient, + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 482740b726..3244e0a2ae 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -108,9 +108,15 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { Name: namespace, }, } - Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) + Expect(resourceManager.Apply([]client.Object{ns}, framework.WithLoggingDisabled())).To(Succeed()) - podNames, err := framework.GetReadyNGFPodNames(k8sClient, ngfNamespace, releaseName, timeoutConfig.GetTimeout) + podNames, err := framework.GetReadyNGFPodNames( + k8sClient, + ngfNamespace, + releaseName, + timeoutConfig.GetTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(podNames).To(HaveLen(1)) ngfPodName = podNames[0] @@ -155,6 +161,8 @@ The logs are attached only if there are errors. writeScaleResults := func(dest io.Writer, results scaleTestResults) error { tmpl, err := template.New("results").Parse(scaleResultTemplate) if err != nil { + GinkgoWriter.Printf("ERROR creating results template: %v\n", err) + return err } @@ -194,12 +202,12 @@ The logs are attached only if there are errors. // attach error logs if len(errors) > 0 { - f, err := os.Create(fileName) + f, err := framework.CreateFile(fileName) Expect(err).ToNot(HaveOccurred()) defer f.Close() for _, e := range errors { - _, err = io.WriteString(f, fmt.Sprintf("%s\n", e)) + _, err = framework.WriteString(f, fmt.Sprintf("%s\n", e)) Expect(err).ToNot(HaveOccurred()) } } @@ -235,14 +243,19 @@ The logs are attached only if there are errors. q, getStartTime, modifyStartTime, + framework.WithLoggingDisabled(), ), ).WithTimeout(metricExistTimeout).WithPolling(metricExistPolling).Should(Succeed()) } test() - // We sleep for 2 scape intervals to ensure that Prometheus scrapes the metrics after the test() finishes + // We sleep for 2 scrape intervals to ensure that Prometheus scrapes the metrics after the test() finishes // before endTime, so that we don't lose any metric values like reloads. + GinkgoWriter.Printf( + "Sleeping for %v to ensure Prometheus scrapes the metrics after the test finishes\n", + 2*scrapeInterval, + ) time.Sleep(2 * scrapeInterval) endTime := time.Now() @@ -278,6 +291,7 @@ The logs are attached only if there are errors. q, getEndTime, noOpModifier, + framework.WithLoggingDisabled(), ), ).WithTimeout(metricExistTimeout).WithPolling(metricExistPolling).Should(Succeed()) } @@ -303,7 +317,7 @@ The logs are attached only if there are errors. framework.GenerateMemoryPNG(testResultsDir, memCSV, memPNG), ).To(Succeed()) - Expect(os.Remove(memCSV)).To(Succeed()) + Expect(framework.Remove(memCSV)).To(Succeed()) result, err = promInstance.QueryRange( fmt.Sprintf(`rate(container_cpu_usage_seconds_total{pod="%s",container="nginx-gateway"}[2m])`, ngfPodName), @@ -323,7 +337,7 @@ The logs are attached only if there are errors. framework.GenerateCPUPNG(testResultsDir, cpuCSV, cpuPNG), ).To(Succeed()) - Expect(os.Remove(cpuCSV)).To(Succeed()) + Expect(framework.Remove(cpuCSV)).To(Succeed()) eventsCount, err := framework.GetEventsCountWithStartTime(promInstance, ngfPodName, startTime) Expect(err).ToNot(HaveOccurred()) @@ -345,7 +359,12 @@ The logs are attached only if there are errors. []string{`"logger":"usageReporter`}, // ignore usageReporter errors ) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := framework.GetReadyNginxPodNames( + k8sClient, + namespace, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -371,10 +390,15 @@ The logs are attached only if there are errors. findRestarts := func(containerName string, pod *core.Pod) int { for _, containerStatus := range pod.Status.ContainerStatuses { if containerStatus.Name == containerName { + GinkgoWriter.Printf("INFO: container %q had %d restarts\n", containerName, containerStatus.RestartCount) + return int(containerStatus.RestartCount) } } - Fail(fmt.Sprintf("container %s not found", containerName)) + fail := fmt.Sprintf("container %s not found", containerName) + GinkgoWriter.Printf("FAIL: %v\n", fail) + + Fail(fail) return 0 } @@ -404,21 +428,29 @@ The logs are attached only if there are errors. Expect(err).ToNot(HaveOccurred()) defer ttrCsvFile.Close() - Expect(resourceManager.Apply(objects.BaseObjects)).To(Succeed()) + Expect(resourceManager.Apply(objects.BaseObjects, framework.WithLoggingDisabled())).To(Succeed()) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace, framework.WithLoggingDisabled())).To(Succeed()) for i := range len(objects.ScaleIterationGroups) { - Expect(resourceManager.Apply(objects.ScaleIterationGroups[i])).To(Succeed()) + Expect(resourceManager.Apply( + objects.ScaleIterationGroups[i], + framework.WithLoggingDisabled(), + )).To(Succeed()) if i == 0 { var nginxPodNames []string Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = framework.GetReadyNginxPodNames( + k8sClient, + namespace, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -442,7 +474,11 @@ The logs are attached only if there are errors. startCheck := time.Now() Eventually( - framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), + framework.CreateResponseChecker( + url, + address, + timeoutConfig.RequestTimeout, + framework.WithLoggingDisabled()), ).WithTimeout(6 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) ttr := time.Since(startCheck) @@ -450,7 +486,7 @@ The logs are attached only if there are errors. seconds := ttr.Seconds() record := []string{strconv.Itoa(i + 1), strconv.FormatFloat(seconds, 'f', -1, 64)} - Expect(writer.Write(record)).To(Succeed()) + Expect(framework.WriteCSVRecord(writer, record)).To(Succeed()) } writer.Flush() @@ -461,18 +497,23 @@ The logs are attached only if there are errors. framework.GenerateTTRPNG(testResultsDir, ttrCsvFile.Name(), ttrPNG), ).To(Succeed()) - Expect(os.Remove(ttrCsvFile.Name())).To(Succeed()) + Expect(framework.Remove(ttrCsvFile.Name())).To(Succeed()) } runScaleUpstreams := func() { Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) var nginxPodNames []string var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = framework.GetReadyNginxPodNames( + k8sClient, + namespace, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -491,7 +532,12 @@ The logs are attached only if there are errors. } Eventually( - framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), + framework.CreateResponseChecker( + url, + address, + timeoutConfig.RequestTimeout, + framework.WithLoggingDisabled(), + ), ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) Expect( @@ -501,10 +547,15 @@ The logs are attached only if there are errors. ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace, framework.WithLoggingDisabled())).To(Succeed()) Eventually( - framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), + framework.CreateResponseChecker( + url, + address, + timeoutConfig.RequestTimeout, + framework.WithLoggingDisabled(), + ), ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) } @@ -523,7 +574,7 @@ The logs are attached only if there are errors. const testName = "TestScale_Listeners" testResultsDir := filepath.Join(resultsDir, testName) - Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) objects, err := framework.GenerateScaleListenerObjects(httpListenerCount, false /*non-tls*/) Expect(err).ToNot(HaveOccurred()) @@ -547,7 +598,7 @@ The logs are attached only if there are errors. const testName = "TestScale_HTTPSListeners" testResultsDir := filepath.Join(resultsDir, testName) - Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) objects, err := framework.GenerateScaleListenerObjects(httpsListenerCount, true /*tls*/) Expect(err).ToNot(HaveOccurred()) @@ -571,7 +622,7 @@ The logs are attached only if there are errors. const testName = "TestScale_HTTPRoutes" testResultsDir := filepath.Join(resultsDir, testName) - Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) objects, err := framework.GenerateScaleHTTPRouteObjects(httpRouteCount) Expect(err).ToNot(HaveOccurred()) @@ -598,7 +649,7 @@ The logs are attached only if there are errors. const testName = "TestScale_UpstreamServers" testResultsDir := filepath.Join(resultsDir, testName) - Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + Expect(framework.MkdirAll(testResultsDir, 0o755)).To(Succeed()) runTestWithMetricsAndLogs( testName, @@ -613,13 +664,18 @@ The logs are attached only if there are errors. const testName = "TestScale_HTTPMatches" Expect(resourceManager.ApplyFromFiles(matchesManifests, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) var nginxPodNames []string var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = framework.GetReadyNginxPodNames( + k8sClient, + namespace, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -682,7 +738,11 @@ The logs are attached only if there are errors. }) AfterEach(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + namespace, + framework.WithLoggingDisabled(), + ) cleanUpPortForward() Expect(resourceManager.DeleteNamespace(namespace)).To(Succeed()) teardown(releaseName) @@ -879,13 +939,18 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim Expect(resourceManager.Apply([]client.Object{&ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(ns.Name, framework.WithLoggingDisabled())).To(Succeed()) var nginxPodNames []string var err error Eventually( func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err = framework.GetReadyNginxPodNames( + k8sClient, + ns.Name, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) return len(nginxPodNames) == 1 && err == nil }). WithTimeout(timeoutConfig.CreateTimeout). @@ -901,7 +966,11 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim }) AfterAll(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + ns.Name, + framework.WithLoggingDisabled(), + ) cleanUpPortForward() teardown(releaseName) @@ -941,7 +1010,12 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.UpdateTimeout) - Expect(resourceManager.WaitForPodsToBeReadyWithCount(ctx, ns.Name, i+numCoffeeAndTeaPods)).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReadyWithCount( + ctx, + ns.Name, + i+numCoffeeAndTeaPods, + framework.WithLoggingDisabled()), + ).To(Succeed()) Expect(resourceManager.WaitForGatewayObservedGeneration(ctx, ns.Name, "gateway", i)).To(Succeed()) cancel() @@ -1018,6 +1092,8 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim var gw v1.Gateway key := types.NamespacedName{Namespace: ns.Name, Name: "gateway"} if err := resourceManager.K8sClient.Get(ctx, key, &gw); err != nil { + GinkgoWriter.Printf("ERROR getting gateway %q in namespace %q: %v\n", key.Name, key.Namespace, err) + return err } diff --git a/tests/suite/snippets_filter_test.go b/tests/suite/snippets_filter_test.go index 7c91c93565..0df9667461 100644 --- a/tests/suite/snippets_filter_test.go +++ b/tests/suite/snippets_filter_test.go @@ -40,9 +40,13 @@ var _ = Describe("SnippetsFilter", Ordered, Label("functional", "snippets-filter Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed()) Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace, framework.WithLoggingDisabled())).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := framework.GetReadyNginxPodNames( + resourceManager.K8sClient, + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -270,30 +274,45 @@ func checkHTTPRouteToHaveGatewayNotProgrammedCond(httpRouteNsName types.Namespac var hr v1.HTTPRoute var err error - if err = k8sClient.Get(ctx, httpRouteNsName, &hr); err != nil { + if err = resourceManager.K8sClient.Get(ctx, httpRouteNsName, &hr); err != nil { return err } if len(hr.Status.Parents) != 1 { - return fmt.Errorf("httproute has %d parent statuses, expected 1", len(hr.Status.Parents)) + tooManyParentStatusesErr := fmt.Errorf("httproute has %d parent statuses, expected 1", len(hr.Status.Parents)) + GinkgoWriter.Printf("ERROR: %v\n", tooManyParentStatusesErr) + + return tooManyParentStatusesErr } parent := hr.Status.Parents[0] if parent.Conditions == nil { - return fmt.Errorf("expected parent conditions to not be nil") + nilConditionErr := fmt.Errorf("expected parent conditions to not be nil") + GinkgoWriter.Printf("ERROR: %v\n", nilConditionErr) + + return nilConditionErr } cond := parent.Conditions[1] if cond.Type != string(v1.GatewayConditionAccepted) { - return fmt.Errorf("expected condition type to be Accepted, got %s", cond.Type) + wrongTypeErr := fmt.Errorf("expected condition type to be Accepted, got %s", cond.Type) + GinkgoWriter.Printf("ERROR: %v\n", wrongTypeErr) + + return wrongTypeErr } if cond.Status != metav1.ConditionFalse { - return fmt.Errorf("expected condition status to be False, got %s", cond.Status) + wrongStatusErr := fmt.Errorf("expected condition status to be False, got %s", cond.Status) + GinkgoWriter.Printf("ERROR: %v\n", wrongStatusErr) + + return wrongStatusErr } if cond.Reason != string(conditions.RouteReasonGatewayNotProgrammed) { - return fmt.Errorf("expected condition reason to be GatewayNotProgrammed, got %s", cond.Reason) + wrongReasonErr := fmt.Errorf("expected condition reason to be GatewayNotProgrammed, got %s", cond.Reason) + GinkgoWriter.Printf("ERROR: %v\n", wrongReasonErr) + + return wrongReasonErr } return nil @@ -311,34 +330,49 @@ func checkForSnippetsFilterToBeAccepted(snippetsFilterNsNames types.NamespacedNa var sf ngfAPI.SnippetsFilter var err error - if err = k8sClient.Get(ctx, snippetsFilterNsNames, &sf); err != nil { + if err = resourceManager.K8sClient.Get(ctx, snippetsFilterNsNames, &sf); err != nil { return err } if len(sf.Status.Controllers) != 1 { - return fmt.Errorf("snippetsFilter has %d controller statuses, expected 1", len(sf.Status.Controllers)) + tooManyStatusesErr := fmt.Errorf("snippetsFilter has %d controller statuses, expected 1", len(sf.Status.Controllers)) + GinkgoWriter.Printf("ERROR: %v\n", tooManyStatusesErr) + + return tooManyStatusesErr } status := sf.Status.Controllers[0] if status.ControllerName != ngfControllerName { - return fmt.Errorf("expected controller name to be %s, got %s", ngfControllerName, status.ControllerName) + wrongNameErr := fmt.Errorf("expected controller name to be %s, got %s", ngfControllerName, status.ControllerName) + GinkgoWriter.Printf("ERROR: %v\n", wrongNameErr) + + return wrongNameErr } condition := status.Conditions[0] if condition.Type != string(ngfAPI.SnippetsFilterConditionTypeAccepted) { - return fmt.Errorf("expected condition type to be Accepted, got %s", condition.Type) + wrongTypeErr := fmt.Errorf("expected condition type to be Accepted, got %s", condition.Type) + GinkgoWriter.Printf("ERROR: %v\n", wrongTypeErr) + + return wrongTypeErr } if status.Conditions[0].Status != metav1.ConditionTrue { - return fmt.Errorf("expected condition status to be %s, got %s", metav1.ConditionTrue, condition.Status) + wrongStatusErr := fmt.Errorf("expected condition status to be %s, got %s", metav1.ConditionTrue, condition.Status) + GinkgoWriter.Printf("ERROR: %v\n", wrongStatusErr) + + return wrongStatusErr } if status.Conditions[0].Reason != string(ngfAPI.SnippetsFilterConditionReasonAccepted) { - return fmt.Errorf( + wrongReasonErr := fmt.Errorf( "expected condition reason to be %s, got %s", ngfAPI.SnippetsFilterConditionReasonAccepted, condition.Reason, ) + GinkgoWriter.Printf("ERROR: %v\n", wrongReasonErr) + + return wrongReasonErr } return nil diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index f404f13e55..92cb5866cd 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -70,7 +70,7 @@ var ( var ( //go:embed manifests/* manifests embed.FS - k8sClient client.Client + k8sClient framework.K8sClient resourceManager framework.ResourceManager portForwardStopCh chan struct{} portFwdPort int @@ -126,7 +126,7 @@ func setup(cfg setupConfig, extraInstallArgs ...string) { } var err error - k8sClient, err = client.New(k8sConfig, options) + k8sClient, err = framework.NewK8sClient(k8sConfig, options) Expect(err).ToNot(HaveOccurred()) clientGoClient, err := kubernetes.NewForConfig(k8sConfig) @@ -193,6 +193,7 @@ func setUpPortForward(nginxPodName, nginxNamespace string) { var err error if *serviceType != "LoadBalancer" { + GinkgoWriter.Printf("Service Type: %s\n", *serviceType) ports := []string{fmt.Sprintf("%d:80", ngfHTTPForwardedPort), fmt.Sprintf("%d:443", ngfHTTPSForwardedPort)} portForwardStopCh = make(chan struct{}) err = framework.PortForward(resourceManager.K8sConfig, nginxNamespace, nginxPodName, ports, portForwardStopCh) @@ -200,6 +201,7 @@ func setUpPortForward(nginxPodName, nginxNamespace string) { portFwdPort = ngfHTTPForwardedPort portFwdHTTPSPort = ngfHTTPSForwardedPort } else { + GinkgoWriter.Printf("Service Type: LoadBalancer\n") address, err = resourceManager.GetLBIPAddress(nginxNamespace) } Expect(err).ToNot(HaveOccurred()) @@ -208,6 +210,7 @@ func setUpPortForward(nginxPodName, nginxNamespace string) { // cleanUpPortForward closes the port forward channel and needs to be called before deleting any gateways or else // the logs will be flooded with port forward errors. func cleanUpPortForward() { + GinkgoWriter.Printf("Cleaning up port forward\n") if portFwdPort != 0 { close(portForwardStopCh) portFwdPort = 0 @@ -216,6 +219,7 @@ func cleanUpPortForward() { } func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framework.InstallationConfig { + GinkgoWriter.Printf("Creating NGF installation config\n") installCfg := framework.InstallationConfig{ ReleaseName: cfg.releaseName, Namespace: ngfNamespace, @@ -229,6 +233,7 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo switch { // if we aren't installing from the public charts, then set the custom images case !strings.HasPrefix(cfg.chartPath, "oci://"): + GinkgoWriter.Printf("Chart path doesn't have prefix 'oci://'\n") installCfg.NgfImageRepository = *ngfImageRepository installCfg.NginxImageRepository = *nginxImageRepository if *plusEnabled && cfg.nfr { @@ -237,6 +242,7 @@ func createNGFInstallConfig(cfg setupConfig, extraInstallArgs ...string) framewo installCfg.ImageTag = *imageTag installCfg.ImagePullPolicy = *imagePullPolicy case version == "edge": + GinkgoWriter.Printf("Using NGF image repository %q with version 'edge'\n", *ngfImageRepository) chartVersion = "0.0.0-edge" installCfg.ChartVersion = chartVersion if *plusEnabled && cfg.nfr { @@ -287,7 +293,11 @@ func teardown(relName string) { true, /* poll immediately */ func(ctx context.Context) (bool, error) { key := k8sTypes.NamespacedName{Name: ngfNamespace} - if err := k8sClient.Get(ctx, key, &core.Namespace{}); err != nil && apierrors.IsNotFound(err) { + if err := resourceManager.K8sClient.Get( + ctx, + key, + &core.Namespace{}, + ); err != nil && apierrors.IsNotFound(err) { return true, nil } diff --git a/tests/suite/tracing_test.go b/tests/suite/tracing_test.go index 2837a01cb3..bf9a302b34 100644 --- a/tests/suite/tracing_test.go +++ b/tests/suite/tracing_test.go @@ -52,11 +52,11 @@ var _ = Describe("Tracing", FlakeAttempts(2), Ordered, Label("functional", "trac key := types.NamespacedName{Name: "ngf-test-proxy-config", Namespace: "nginx-gateway"} var nginxProxy ngfAPIv1alpha2.NginxProxy - Expect(k8sClient.Get(ctx, key, &nginxProxy)).To(Succeed()) + Expect(resourceManager.K8sClient.Get(ctx, key, &nginxProxy)).To(Succeed()) nginxProxy.Spec.Telemetry = &telemetry - Expect(k8sClient.Update(ctx, &nginxProxy)).To(Succeed()) + Expect(resourceManager.K8sClient.Update(ctx, &nginxProxy, nil)).To(Succeed()) } BeforeAll(func() { @@ -92,7 +92,11 @@ var _ = Describe("Tracing", FlakeAttempts(2), Ordered, Label("functional", "trac Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := framework.GetReadyNginxPodNames( + resourceManager.K8sClient, + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -127,10 +131,18 @@ var _ = Describe("Tracing", FlakeAttempts(2), Ordered, Label("functional", "trac }) sendRequests := func(url string, count int) { + GinkgoWriter.Printf("Sending %d requests to %s\n", count, url) for range count { Eventually( func() error { - status, _, err := framework.Get(url, address, timeoutConfig.RequestTimeout, nil, nil) + status, _, err := framework.Get( + url, + address, + timeoutConfig.RequestTimeout, + nil, + nil, + framework.WithLoggingDisabled(), + ) if err != nil { return err } @@ -227,30 +239,42 @@ var _ = Describe("Tracing", FlakeAttempts(2), Ordered, Label("functional", "trac }) func verifyGatewayClassResolvedRefs() error { + GinkgoWriter.Println("Verifying GatewayClass ResolvedRefs condition is True\n") ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetTimeout) defer cancel() var gc gatewayv1.GatewayClass - if err := k8sClient.Get(ctx, types.NamespacedName{Name: gatewayClassName}, &gc); err != nil { + if err := resourceManager.K8sClient.Get(ctx, types.NamespacedName{Name: gatewayClassName}, &gc); err != nil { return err } for _, cond := range gc.Status.Conditions { + GinkgoWriter.Printf( + "GatewayClass condition: Type=%s, Status=%s. Expected condition: Type=%s, Status=%s\n", + cond.Type, + cond.Status, + string(conditions.GatewayClassResolvedRefs), + metav1.ConditionTrue, + ) if cond.Type == string(conditions.GatewayClassResolvedRefs) && cond.Status == metav1.ConditionTrue { + GinkgoWriter.Println("Success: GatewayClass ResolvedRefs condition is True\n") return nil } } + statusErr := errors.New("ResolvedRefs status not set to true on GatewayClass") + GinkgoWriter.Printf("ERROR: %v\n", statusErr) - return errors.New("ResolvedRefs status not set to true on GatewayClass") + return statusErr } func verifyPolicyStatus() error { + GinkgoWriter.Println("Verifying ObservabilityPolicy status\n") ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetTimeout) defer cancel() var pol ngfAPIv1alpha2.ObservabilityPolicy key := types.NamespacedName{Name: "test-observability-policy", Namespace: "helloworld"} - if err := k8sClient.Get(ctx, key, &pol); err != nil { + if err := resourceManager.K8sClient.Get(ctx, key, &pol); err != nil { return err } @@ -264,7 +288,14 @@ func verifyPolicyStatus() error { } if count != len(pol.Status.Ancestors) { - return fmt.Errorf("Policy not accepted; expected %d accepted conditions, got %d", len(pol.Status.Ancestors), count) + acceptedConditionsErr := fmt.Errorf( + "Policy not accepted; expected %d accepted conditions, got %d", + len(pol.Status.Ancestors), + count, + ) + GinkgoWriter.Printf("ERROR: %v\n", acceptedConditionsErr) + + return acceptedConditionsErr } return nil diff --git a/tests/suite/upgrade_test.go b/tests/suite/upgrade_test.go index cc864ff990..8fe3c0f659 100644 --- a/tests/suite/upgrade_test.go +++ b/tests/suite/upgrade_test.go @@ -67,7 +67,12 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, ns.Name, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := framework.GetReadyNginxPodNames( + k8sClient, + ns.Name, + timeoutConfig.GetStatusTimeout, + framework.WithLoggingDisabled(), + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -83,7 +88,11 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { }) AfterEach(func() { - framework.AddNginxLogsAndEventsToReport(resourceManager, ns.Name) + framework.AddNginxLogsAndEventsToReport( + resourceManager, + ns.Name, + framework.WithLoggingDisabled(), + ) cleanUpPortForward() Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed()) @@ -220,7 +229,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { true, /* poll immediately */ func(_ context.Context) (bool, error) { defer GinkgoRecover() - Expect(k8sClient.Get(leaseCtx, key, &lease)).To(Succeed()) + Expect(resourceManager.K8sClient.Get(leaseCtx, key, &lease)).To(Succeed()) if lease.Spec.HolderIdentity != nil { for _, podName := range podNames { @@ -245,7 +254,7 @@ var _ = Describe("Upgrade testing", Label("nfr", "upgrade"), func() { 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { - Expect(k8sClient.Get(ctx, key, &gw)).To(Succeed()) + Expect(resourceManager.K8sClient.Get(ctx, key, &gw)).To(Succeed()) expListenerName := "http-new" for _, listener := range gw.Status.Listeners { if listener.Name == v1.SectionName(expListenerName) { diff --git a/tests/suite/upstream_settings_test.go b/tests/suite/upstream_settings_test.go index 2f5c2bb408..3b1ac4e58f 100644 --- a/tests/suite/upstream_settings_test.go +++ b/tests/suite/upstream_settings_test.go @@ -51,7 +51,11 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic Expect(resourceManager.ApplyFromFiles(files, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - nginxPodNames, err := framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + nginxPodNames, err := framework.GetReadyNginxPodNames( + resourceManager.K8sClient, + namespace, + timeoutConfig.GetStatusTimeout, + ) Expect(err).ToNot(HaveOccurred()) Expect(nginxPodNames).To(HaveLen(1)) @@ -138,7 +142,7 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic DescribeTable("are set properly for", func(expCfgs []framework.ExpectedNginxField) { for _, expCfg := range expCfgs { - Expect(framework.ValidateNginxFieldExists(conf, expCfg)).To(Succeed()) + Expect(framework.ValidateNginxFieldExists(conf, expCfg, framework.WithLoggingDisabled())).To(Succeed()) } }, Entry("HTTP upstreams", []framework.ExpectedNginxField{ @@ -318,7 +322,7 @@ var _ = Describe("UpstreamSettingsPolicy", Ordered, Label("functional", "uspolic DescribeTable("are set properly for", func(expCfgs []framework.ExpectedNginxField) { for _, expCfg := range expCfgs { - Expect(framework.ValidateNginxFieldExists(conf, expCfg)).To(Succeed()) + Expect(framework.ValidateNginxFieldExists(conf, expCfg, framework.WithLoggingDisabled())).To(Succeed()) } }, Entry("Coffee upstream", []framework.ExpectedNginxField{ @@ -425,11 +429,19 @@ func usPolicyHasNoAncestors(usPolicyNsName types.NamespacedName) bool { var usPolicy ngfAPI.UpstreamSettingsPolicy if err := k8sClient.Get(ctx, usPolicyNsName, &usPolicy); err != nil { - GinkgoWriter.Printf("Failed to get UpstreamSettingsPolicy %q: %s", usPolicyNsName, err.Error()) + GinkgoWriter.Printf("ERROR: Failed to get UpstreamSettingsPolicy %q: %s", usPolicyNsName, err.Error()) return false } + isZeroAncestors := len(usPolicy.Status.Ancestors) == 0 + if !isZeroAncestors { + GinkgoWriter.Printf( + "UpstreamSettingsPolicy %q has %d ancestors in status\n", + usPolicyNsName, + len(usPolicy.Status.Ancestors), + ) + } - return len(usPolicy.Status.Ancestors) == 0 + return isZeroAncestors } func waitForUSPolicyStatus( @@ -457,6 +469,8 @@ func waitForUSPolicyStatus( var err error if err := k8sClient.Get(ctx, usPolicyNsName, &usPolicy); err != nil { + GinkgoWriter.Printf("ERROR: Failed to get UpstreamSettingsPolicy %q: %s", usPolicyNsName, err.Error()) + return false, err } @@ -467,13 +481,18 @@ func waitForUSPolicyStatus( } if len(usPolicy.Status.Ancestors) != 1 { - return false, fmt.Errorf("policy has %d ancestors, expected 1", len(usPolicy.Status.Ancestors)) + tooManyAncestorsErr := fmt.Errorf("policy has %d ancestors, expected 1", len(usPolicy.Status.Ancestors)) + GinkgoWriter.Printf("ERROR: %v\n", tooManyAncestorsErr) + + return false, tooManyAncestorsErr } ancestors := usPolicy.Status.Ancestors for _, ancestor := range ancestors { if err := ancestorMustEqualGatewayRef(ancestor, gatewayName, usPolicy.Namespace); err != nil { + GinkgoWriter.Printf("ERROR: %v\n", err) + return false, err }