add additional options for dialer retries

Author: ndyakov

hitless/example_hooks.go

@@ -21,6 +21,9 @@ type MetricsHook struct {
 	NotificationCounts map[string]int64
 	ProcessingTimes    map[string]time.Duration
 	ErrorCounts        map[string]int64
+	HandoffCounts      int64 // Total handoffs initiated
+	HandoffSuccesses   int64 // Successful handoffs
+	HandoffFailures    int64 // Failed handoffs
 }
 
 // NewMetricsHook creates a new metrics collection hook.

internal/pool/pool.go

@@ -99,6 +99,14 @@ type Options struct {
 	ConnMaxIdleTime          time.Duration
 	ConnMaxLifetime          time.Duration
 	PushNotificationsEnabled bool
+
+	// DialerRetries is the maximum number of retry attempts when dialing fails.
+	// Default: 5
+	DialerRetries int
+
+	// DialerRetryTimeout is the backoff duration between retry attempts.
+	// Default: 100ms
+	DialerRetryTimeout time.Duration
 }
 
 type lastDialErrorWrap struct {
@@ -315,8 +323,14 @@ func (p *ConnPool) dialConn(ctx context.Context, pooled bool) (*Conn, error) {
 	// Retry dialing with backoff
 	// the context timeout is already handled by the context passed in
 	// so we may never reach the max retries, higher values don't hurt
-	const maxRetries = 10
-	const backoffDuration = 100 * time.Millisecond
+	maxRetries := p.cfg.DialerRetries
+	if maxRetries <= 0 {
+		maxRetries = 5 // Default value
+	}
+	backoffDuration := p.cfg.DialerRetryTimeout
+	if backoffDuration <= 0 {
+		backoffDuration = 100 * time.Millisecond // Default value
+	}
 
 	var lastErr error
 	shouldLoop := true

internal/pool/pool_test.go

@@ -437,6 +437,70 @@ var _ = Describe("race", func() {
 	})
 })
 
+// TestDialerRetryConfiguration tests the new DialerRetries and DialerRetryTimeout options
+func TestDialerRetryConfiguration(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("CustomDialerRetries", func(t *testing.T) {
+		attempts := 0
+		failingDialer := func(ctx context.Context) (net.Conn, error) {
+			attempts++
+			return nil, errors.New("dial failed")
+		}
+
+		connPool := pool.NewConnPool(&pool.Options{
+			Dialer:             failingDialer,
+			PoolSize:           1,
+			PoolTimeout:        time.Second,
+			DialTimeout:        time.Second,
+			DialerRetries:      3, // Custom retry count
+			DialerRetryTimeout: 10 * time.Millisecond, // Fast retries for testing
+		})
+		defer connPool.Close()
+
+		_, err := connPool.Get(ctx)
+		if err == nil {
+			t.Error("Expected error from failing dialer")
+		}
+
+		// Should have attempted at least 3 times (DialerRetries = 3)
+		// There might be additional attempts due to pool logic
+		if attempts < 3 {
+			t.Errorf("Expected at least 3 dial attempts, got %d", attempts)
+		}
+		if attempts > 6 {
+			t.Errorf("Expected around 3 dial attempts, got %d (too many)", attempts)
+		}
+	})
+
+	t.Run("DefaultDialerRetries", func(t *testing.T) {
+		attempts := 0
+		failingDialer := func(ctx context.Context) (net.Conn, error) {
+			attempts++
+			return nil, errors.New("dial failed")
+		}
+
+		connPool := pool.NewConnPool(&pool.Options{
+			Dialer:      failingDialer,
+			PoolSize:    1,
+			PoolTimeout: time.Second,
+			DialTimeout: time.Second,
+			// DialerRetries and DialerRetryTimeout not set - should use defaults
+		})
+		defer connPool.Close()
+
+		_, err := connPool.Get(ctx)
+		if err == nil {
+			t.Error("Expected error from failing dialer")
+		}
+
+		// Should have attempted 5 times (default DialerRetries = 5)
+		if attempts != 5 {
+			t.Errorf("Expected 5 dial attempts (default), got %d", attempts)
+		}
+	})
+}
+
 func init() {
 	logging.Disable()
 }

options.go

@@ -112,6 +112,16 @@ type Options struct {
 	// default: 10 seconds
 	DialTimeout time.Duration
 
+	// DialerRetries is the maximum number of retry attempts when dialing fails.
+	//
+	// default: 5
+	DialerRetries int
+
+	// DialerRetryTimeout is the backoff duration between retry attempts.
+	//
+	// default: 100 milliseconds
+	DialerRetryTimeout time.Duration
+
 	// ReadTimeout for socket reads. If reached, commands will fail
 	// with a timeout instead of blocking. Supported values:
 	//
@@ -277,6 +287,12 @@ func (opt *Options) init() {
 	if opt.DialTimeout == 0 {
 		opt.DialTimeout = 10 * time.Second
 	}
+	if opt.DialerRetries == 0 {
+		opt.DialerRetries = 5
+	}
+	if opt.DialerRetryTimeout == 0 {
+		opt.DialerRetryTimeout = 100 * time.Millisecond
+	}
 	if opt.Dialer == nil {
 		opt.Dialer = NewDialer(opt)
 	}
@@ -678,6 +694,8 @@ func newConnPool(
 		PoolSize:                 poolSize,
 		PoolTimeout:              opt.PoolTimeout,
 		DialTimeout:              opt.DialTimeout,
+		DialerRetries:            opt.DialerRetries,
+		DialerRetryTimeout:       opt.DialerRetryTimeout,
 		MinIdleConns:             minIdleConns,
 		MaxIdleConns:             maxIdleConns,
 		MaxActiveConns:           maxActiveConns,
@@ -716,6 +734,8 @@ func newPubSubPool(opt *Options, dialer func(ctx context.Context, network, addr
 		PoolSize:                 poolSize,
 		PoolTimeout:              opt.PoolTimeout,
 		DialTimeout:              opt.DialTimeout,
+		DialerRetries:            opt.DialerRetries,
+		DialerRetryTimeout:       opt.DialerRetryTimeout,
 		MinIdleConns:             minIdleConns,
 		MaxIdleConns:             maxIdleConns,
 		MaxActiveConns:           maxActiveConns,