Merge pull request #2660 from arewm/ec-1404

fix(ec-1404): add jitter for intermittent errors

Author: simonbaird

cmd/root/root_cmd.go

@@ -31,6 +31,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
+	"github.com/conforma/cli/internal/http"
 	"github.com/conforma/cli/internal/kubernetes"
 	"github.com/conforma/cli/internal/logging"
 	"github.com/conforma/cli/internal/tracing"
@@ -44,7 +45,15 @@ var (
 	enabledTraces tracing.Trace = tracing.None
 	globalTimeout               = 5 * time.Minute
 	logfile       string
-	OnExit        func() = func() {}
+
+	// Retry configuration flags
+	retryMaxWait  time.Duration = 3 * time.Second
+	retryMaxRetry int           = 3
+	retryDuration time.Duration = 1 * time.Second
+	retryFactor   float64       = 2.0
+	retryJitter   float64       = 0.1
+
+	OnExit func() = func() {}
 )
 
 type customDeadlineExceededError struct{}
@@ -91,6 +100,16 @@ func NewRootCmd() *cobra.Command {
 		PersistentPreRun: func(cmd *cobra.Command, _ []string) {
 			logging.InitLogging(verbose, quiet, debug, enabledTraces.Enabled(tracing.Log, tracing.Opa), logfile)
 
+			// Apply retry configuration from CLI flags
+			retryConfig := http.RetryConfig{
+				MaxWait:  retryMaxWait,
+				MaxRetry: retryMaxRetry,
+				Duration: retryDuration,
+				Factor:   retryFactor,
+				Jitter:   retryJitter,
+			}
+			http.SetRetryConfig(retryConfig)
+
 			// set a custom message for context.DeadlineExceeded error
 			context.DeadlineExceeded = customDeadlineExceededError{}
 
@@ -192,5 +211,13 @@ func setFlags(rootCmd *cobra.Command) {
 	rootCmd.PersistentFlags().BoolVar(&debug, "debug", debug, "same as verbose but also show function names and line numbers")
 	rootCmd.PersistentFlags().DurationVar(&globalTimeout, "timeout", globalTimeout, "max overall execution duration")
 	rootCmd.PersistentFlags().StringVar(&logfile, "logfile", "", "file to write the logging output. If not specified logging output will be written to stderr")
+
+	// Retry configuration flags
+	rootCmd.PersistentFlags().DurationVar(&retryMaxWait, "retry-max-wait", retryMaxWait, "maximum wait time between retries")
+	rootCmd.PersistentFlags().IntVar(&retryMaxRetry, "retry-max-retry", retryMaxRetry, "maximum number of retry attempts")
+	rootCmd.PersistentFlags().DurationVar(&retryDuration, "retry-duration", retryDuration, "base duration for exponential backoff calculation")
+	rootCmd.PersistentFlags().Float64Var(&retryFactor, "retry-factor", retryFactor, "exponential backoff multiplier")
+	rootCmd.PersistentFlags().Float64Var(&retryJitter, "retry-jitter", retryJitter, "randomness factor for backoff calculation (0.0-1.0)")
+
 	kubernetes.AddKubeconfigFlag(rootCmd)
 }

cmd/root/root_cmd_test.go

@@ -21,6 +21,8 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/conforma/cli/internal/http"
 )
 
 func TestGlobalTimeout(t *testing.T) {
@@ -76,3 +78,74 @@ func TestGlobalTimeout(t *testing.T) {
 		})
 	}
 }
+
+func TestRetryConfigurationFlags(t *testing.T) {
+	tests := []struct {
+		name           string
+		args           []string
+		expectedConfig http.RetryConfig
+	}{
+		{
+			name: "default retry configuration",
+			args: []string{},
+			expectedConfig: http.RetryConfig{
+				MaxWait:  3 * time.Second,
+				MaxRetry: 3,
+				Duration: 1 * time.Second,
+				Factor:   2.0,
+				Jitter:   0.1,
+			},
+		},
+		{
+			name: "custom retry configuration",
+			args: []string{
+				"--retry-max-wait", "2s",
+				"--retry-max-retry", "5",
+				"--retry-duration", "500ms",
+				"--retry-factor", "1.5",
+				"--retry-jitter", "0.2",
+			},
+			expectedConfig: http.RetryConfig{
+				MaxWait:  2 * time.Second,
+				MaxRetry: 5,
+				Duration: 500 * time.Millisecond,
+				Factor:   1.5,
+				Jitter:   0.2,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Reset to default values
+			retryMaxWait = 3 * time.Second
+			retryMaxRetry = 3
+			retryDuration = 1 * time.Second
+			retryFactor = 2.0
+			retryJitter = 0.1
+
+			cmd := NewRootCmd()
+			if len(tt.args) > 0 {
+				cmd.SetArgs(append(tt.args, "version"))
+			} else {
+				cmd.SetArgs([]string{"version"})
+			}
+
+			// Execute the command to trigger flag parsing and configuration
+			err := cmd.Execute()
+			assert.NoError(t, err)
+
+			// Verify the configuration was applied by checking the actual values
+			// The configuration is applied in PersistentPreRun, so we need to check
+			// that the flags were parsed correctly
+			if len(tt.args) > 0 {
+				// For custom configuration, verify that the flags were parsed
+				assert.Equal(t, tt.expectedConfig.MaxWait, retryMaxWait)
+				assert.Equal(t, tt.expectedConfig.MaxRetry, retryMaxRetry)
+				assert.Equal(t, tt.expectedConfig.Duration, retryDuration)
+				assert.Equal(t, tt.expectedConfig.Factor, retryFactor)
+				assert.Equal(t, tt.expectedConfig.Jitter, retryJitter)
+			}
+		})
+	}
+}

docs/modules/ROOT/pages/ec.adoc

@@ -39,6 +39,11 @@ ec [flags]
 --kubeconfig:: path to the Kubernetes config file to use
 --logfile:: file to write the logging output. If not specified logging output will be written to stderr
 --quiet:: less verbose output (Default: false)
+--retry-duration:: base duration for exponential backoff calculation (Default: 1s)
+--retry-factor:: exponential backoff multiplier (Default: 2)
+--retry-jitter:: randomness factor for backoff calculation (0.0-1.0) (Default: 0.1)
+--retry-max-retry:: maximum number of retry attempts (Default: 3)
+--retry-max-wait:: maximum wait time between retries (Default: 3s)
 --timeout:: max overall execution duration (Default: 5m0s)
 --trace:: enable trace logging, set one or more comma separated values: none,all,perf,cpu,mem,opa,log (Default: none)
 --verbose:: more verbose output (Default: false)

docs/modules/ROOT/pages/ec_fetch.adoc

@@ -12,6 +12,11 @@ Fetch remote resources
 --kubeconfig:: path to the Kubernetes config file to use
 --logfile:: file to write the logging output. If not specified logging output will be written to stderr
 --quiet:: less verbose output (Default: false)
+--retry-duration:: base duration for exponential backoff calculation (Default: 1s)
+--retry-factor:: exponential backoff multiplier (Default: 2)
+--retry-jitter:: randomness factor for backoff calculation (0.0-1.0) (Default: 0.1)
+--retry-max-retry:: maximum number of retry attempts (Default: 3)
+--retry-max-wait:: maximum wait time between retries (Default: 3s)
 --timeout:: max overall execution duration (Default: 5m0s)
 --trace:: enable trace logging, set one or more comma separated values: none,all,perf,cpu,mem,opa,log (Default: none)
 --verbose:: more verbose output (Default: false)

docs/modules/ROOT/pages/ec_fetch_policy.adoc

@@ -74,6 +74,11 @@ Notes:
 --kubeconfig:: path to the Kubernetes config file to use
 --logfile:: file to write the logging output. If not specified logging output will be written to stderr
 --quiet:: less verbose output (Default: false)
+--retry-duration:: base duration for exponential backoff calculation (Default: 1s)
+--retry-factor:: exponential backoff multiplier (Default: 2)
+--retry-jitter:: randomness factor for backoff calculation (0.0-1.0) (Default: 0.1)
+--retry-max-retry:: maximum number of retry attempts (Default: 3)
+--retry-max-wait:: maximum wait time between retries (Default: 3s)
 --timeout:: max overall execution duration (Default: 5m0s)
 --trace:: enable trace logging, set one or more comma separated values: none,all,perf,cpu,mem,opa,log (Default: none)
 --verbose:: more verbose output (Default: false)

docs/modules/ROOT/pages/ec_init.adoc

@@ -12,6 +12,11 @@ Initialize a directory for use
 --kubeconfig:: path to the Kubernetes config file to use
 --logfile:: file to write the logging output. If not specified logging output will be written to stderr
 --quiet:: less verbose output (Default: false)
+--retry-duration:: base duration for exponential backoff calculation (Default: 1s)
+--retry-factor:: exponential backoff multiplier (Default: 2)
+--retry-jitter:: randomness factor for backoff calculation (0.0-1.0) (Default: 0.1)
+--retry-max-retry:: maximum number of retry attempts (Default: 3)
+--retry-max-wait:: maximum wait time between retries (Default: 3s)
 --timeout:: max overall execution duration (Default: 5m0s)
 --trace:: enable trace logging, set one or more comma separated values: none,all,perf,cpu,mem,opa,log (Default: none)
 --verbose:: more verbose output (Default: false)

docs/modules/ROOT/pages/ec_init_policies.adoc

@@ -31,6 +31,11 @@ Initialize the "my-policy" directory with minimal EC policy scaffolding:
 --kubeconfig:: path to the Kubernetes config file to use
 --logfile:: file to write the logging output. If not specified logging output will be written to stderr
 --quiet:: less verbose output (Default: false)
+--retry-duration:: base duration for exponential backoff calculation (Default: 1s)
+--retry-factor:: exponential backoff multiplier (Default: 2)
+--retry-jitter:: randomness factor for backoff calculation (0.0-1.0) (Default: 0.1)
+--retry-max-retry:: maximum number of retry attempts (Default: 3)
+--retry-max-wait:: maximum wait time between retries (Default: 3s)
 --timeout:: max overall execution duration (Default: 5m0s)
 --trace:: enable trace logging, set one or more comma separated values: none,all,perf,cpu,mem,opa,log (Default: none)
 --verbose:: more verbose output (Default: false)

docs/modules/ROOT/pages/ec_inspect.adoc

@@ -12,6 +12,11 @@ Inspect policy rules
 --kubeconfig:: path to the Kubernetes config file to use
 --logfile:: file to write the logging output. If not specified logging output will be written to stderr
 --quiet:: less verbose output (Default: false)
+--retry-duration:: base duration for exponential backoff calculation (Default: 1s)
+--retry-factor:: exponential backoff multiplier (Default: 2)
+--retry-jitter:: randomness factor for backoff calculation (0.0-1.0) (Default: 0.1)
+--retry-max-retry:: maximum number of retry attempts (Default: 3)
+--retry-max-wait:: maximum wait time between retries (Default: 3s)
 --timeout:: max overall execution duration (Default: 5m0s)
 --trace:: enable trace logging, set one or more comma separated values: none,all,perf,cpu,mem,opa,log (Default: none)
 --verbose:: more verbose output (Default: false)

docs/modules/ROOT/pages/ec_inspect_policy-data.adoc

@@ -36,6 +36,11 @@ ec inspect policy-data --source git::https://github.com/conforma/policy//example
 --kubeconfig:: path to the Kubernetes config file to use
 --logfile:: file to write the logging output. If not specified logging output will be written to stderr
 --quiet:: less verbose output (Default: false)
+--retry-duration:: base duration for exponential backoff calculation (Default: 1s)
+--retry-factor:: exponential backoff multiplier (Default: 2)
+--retry-jitter:: randomness factor for backoff calculation (0.0-1.0) (Default: 0.1)
+--retry-max-retry:: maximum number of retry attempts (Default: 3)
+--retry-max-wait:: maximum wait time between retries (Default: 3s)
 --timeout:: max overall execution duration (Default: 5m0s)
 --trace:: enable trace logging, set one or more comma separated values: none,all,perf,cpu,mem,opa,log (Default: none)
 --verbose:: more verbose output (Default: false)

docs/modules/ROOT/pages/ec_inspect_policy.adoc

@@ -48,6 +48,11 @@ Display details about the latest release policy in json format:
 --kubeconfig:: path to the Kubernetes config file to use
 --logfile:: file to write the logging output. If not specified logging output will be written to stderr
 --quiet:: less verbose output (Default: false)
+--retry-duration:: base duration for exponential backoff calculation (Default: 1s)
+--retry-factor:: exponential backoff multiplier (Default: 2)
+--retry-jitter:: randomness factor for backoff calculation (0.0-1.0) (Default: 0.1)
+--retry-max-retry:: maximum number of retry attempts (Default: 3)
+--retry-max-wait:: maximum wait time between retries (Default: 3s)
 --timeout:: max overall execution duration (Default: 5m0s)
 --trace:: enable trace logging, set one or more comma separated values: none,all,perf,cpu,mem,opa,log (Default: none)
 --verbose:: more verbose output (Default: false)