Increase handover drain timeout from 5 minutes to 24 hours (#234)

* Increase handover drain timeout from 5 minutes to 24 hours

The 5-minute drain timeout was too aggressive — long-running queries
were being forcibly terminated during rolling deployments. Increase to
24 hours so existing connections can finish naturally.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Make handover drain timeout configurable via --handover-drain-timeout

Adds 3-tier config support (CLI flag, env var, YAML) following the
existing pattern for worker timeouts. Default remains 24h.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: fuziontech

config_resolution.go

@@ -30,16 +30,18 @@ type configCLIInputs struct {
 	MinWorkers                int
 	WorkerQueueTimeout        string
 	WorkerIdleTimeout         string
+	HandoverDrainTimeout      string
 	ACMEDomain                string
 	ACMEEmail                 string
 	ACMECacheDir              string
 	MaxConnections            int
 }
 
 type resolvedConfig struct {
-	Server             server.Config
-	WorkerQueueTimeout time.Duration
-	WorkerIdleTimeout  time.Duration
+	Server               server.Config
+	WorkerQueueTimeout   time.Duration
+	WorkerIdleTimeout    time.Duration
+	HandoverDrainTimeout time.Duration
 }
 
 func defaultServerConfig() server.Config {
@@ -75,6 +77,7 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 	cfg := defaultServerConfig()
 	var workerQueueTimeout time.Duration
 	var workerIdleTimeout time.Duration
+	var handoverDrainTimeout time.Duration
 
 	if fileCfg != nil {
 		if fileCfg.Host != "" {
@@ -230,6 +233,13 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 				warn("Invalid worker_idle_timeout duration: " + err.Error())
 			}
 		}
+		if fileCfg.HandoverDrainTimeout != "" {
+			if d, err := time.ParseDuration(fileCfg.HandoverDrainTimeout); err == nil {
+				handoverDrainTimeout = d
+			} else {
+				warn("Invalid handover_drain_timeout duration: " + err.Error())
+			}
+		}
 		if len(fileCfg.PassthroughUsers) > 0 {
 			cfg.PassthroughUsers = make(map[string]bool, len(fileCfg.PassthroughUsers))
 			for _, u := range fileCfg.PassthroughUsers {
@@ -399,6 +409,13 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 			warn("Invalid DUCKGRES_WORKER_IDLE_TIMEOUT duration: " + err.Error())
 		}
 	}
+	if v := getenv("DUCKGRES_HANDOVER_DRAIN_TIMEOUT"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil {
+			handoverDrainTimeout = d
+		} else {
+			warn("Invalid DUCKGRES_HANDOVER_DRAIN_TIMEOUT duration: " + err.Error())
+		}
+	}
 	if v := getenv("DUCKGRES_ACME_DOMAIN"); v != "" {
 		cfg.ACMEDomain = v
 	}
@@ -504,6 +521,13 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 			warn("Invalid --worker-idle-timeout duration: " + err.Error())
 		}
 	}
+	if cli.Set["handover-drain-timeout"] {
+		if d, err := time.ParseDuration(cli.HandoverDrainTimeout); err == nil {
+			handoverDrainTimeout = d
+		} else {
+			warn("Invalid --handover-drain-timeout duration: " + err.Error())
+		}
+	}
 	if cli.Set["acme-domain"] {
 		cfg.ACMEDomain = cli.ACMEDomain
 	}
@@ -530,8 +554,9 @@ func resolveEffectiveConfig(fileCfg *FileConfig, cli configCLIInputs, getenv fun
 	}
 
 	return resolvedConfig{
-		Server:             cfg,
-		WorkerQueueTimeout: workerQueueTimeout,
-		WorkerIdleTimeout:  workerIdleTimeout,
+		Server:               cfg,
+		WorkerQueueTimeout:   workerQueueTimeout,
+		WorkerIdleTimeout:    workerIdleTimeout,
+		HandoverDrainTimeout: handoverDrainTimeout,
 	}
 }

controlplane/control.go

@@ -31,9 +31,10 @@ type ControlPlaneConfig struct {
 	ConfigPath          string // Path to config file, passed to workers
 	HandoverSocket      string
 	HealthCheckInterval time.Duration
-	WorkerQueueTimeout  time.Duration // How long to wait for an available worker slot (default: 5m)
-	WorkerIdleTimeout   time.Duration // How long to keep an idle worker alive (default: 5m)
-	MetricsServer       *http.Server  // Optional metrics server to shut down during handover
+	WorkerQueueTimeout   time.Duration // How long to wait for an available worker slot (default: 5m)
+	WorkerIdleTimeout    time.Duration // How long to keep an idle worker alive (default: 5m)
+	HandoverDrainTimeout time.Duration // How long to wait for connections to drain during handover (default: 24h)
+	MetricsServer        *http.Server  // Optional metrics server to shut down during handover
 }
 
 // ControlPlane manages the TCP listener and routes connections to Flight SQL workers.
@@ -74,6 +75,9 @@ func RunControlPlane(cfg ControlPlaneConfig) {
 	if cfg.WorkerIdleTimeout == 0 {
 		cfg.WorkerIdleTimeout = 5 * time.Minute
 	}
+	if cfg.HandoverDrainTimeout == 0 {
+		cfg.HandoverDrainTimeout = 24 * time.Hour
+	}
 
 	// Enforce secure defaults for control-plane mode.
 	if err := validateControlPlaneSecurity(cfg); err != nil {

controlplane/handover.go

@@ -190,8 +190,8 @@ func (cp *ControlPlane) handleHandoverRequest(conn net.Conn, handoverLn net.List
 	select {
 	case <-drainDone:
 		slog.Info("All connections drained after handover.")
-	case <-time.After(5 * time.Minute):
-		slog.Warn("Handover drain timeout after 5 minutes, forcing exit.")
+	case <-time.After(cp.cfg.HandoverDrainTimeout):
+		slog.Warn("Handover drain timeout, forcing exit.", "timeout", cp.cfg.HandoverDrainTimeout)
 	}
 
 	// Shut down workers

main.go

@@ -43,8 +43,9 @@ type FileConfig struct {
 	MemoryRebalance           *bool               `yaml:"memory_rebalance"`  // Enable dynamic per-connection memory reallocation
 	MaxWorkers                int                 `yaml:"max_workers"`           // Max worker processes (control-plane mode)
 	MinWorkers                int                 `yaml:"min_workers"`           // Pre-warm worker count (control-plane mode)
-	WorkerQueueTimeout        string              `yaml:"worker_queue_timeout"`  // e.g., "5m"
-	WorkerIdleTimeout         string              `yaml:"worker_idle_timeout"`   // e.g., "5m"
+	WorkerQueueTimeout        string              `yaml:"worker_queue_timeout"`        // e.g., "5m"
+	WorkerIdleTimeout         string              `yaml:"worker_idle_timeout"`         // e.g., "5m"
+	HandoverDrainTimeout      string              `yaml:"handover_drain_timeout"`      // e.g., "24h"
 	PassthroughUsers          []string            `yaml:"passthrough_users"` // Users that bypass transpiler + pg_catalog
 }
 
@@ -178,6 +179,7 @@ func main() {
 	maxWorkers := flag.Int("max-workers", 0, "Max worker processes, 0=unlimited (control-plane mode) (env: DUCKGRES_MAX_WORKERS)")
 	workerQueueTimeout := flag.String("worker-queue-timeout", "", "How long to wait for an available worker slot (e.g., '5m') (env: DUCKGRES_WORKER_QUEUE_TIMEOUT)")
 	workerIdleTimeout := flag.String("worker-idle-timeout", "", "How long to keep an idle worker alive (e.g., '5m') (env: DUCKGRES_WORKER_IDLE_TIMEOUT)")
+	handoverDrainTimeout := flag.String("handover-drain-timeout", "", "How long to wait for connections to drain during handover (default: '24h') (env: DUCKGRES_HANDOVER_DRAIN_TIMEOUT)")
 	socketDir := flag.String("socket-dir", "/var/run/duckgres", "Unix socket directory (control-plane mode)")
 	handoverSocket := flag.String("handover-socket", "", "Handover socket for graceful deployment (control-plane mode)")
 
@@ -217,6 +219,7 @@ func main() {
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_MIN_WORKERS        Pre-warm worker count (control-plane mode)\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_MAX_WORKERS        Max worker processes (control-plane mode)\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_WORKER_QUEUE_TIMEOUT  Worker queue timeout (default: 5m)\n")
+		fmt.Fprintf(os.Stderr, "  DUCKGRES_HANDOVER_DRAIN_TIMEOUT  Handover drain timeout (default: 24h)\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_ACME_DOMAIN        Domain for ACME/Let's Encrypt certificate\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_ACME_EMAIL         Contact email for Let's Encrypt notifications\n")
 		fmt.Fprintf(os.Stderr, "  DUCKGRES_ACME_CACHE_DIR     Directory for ACME certificate cache\n")
@@ -303,6 +306,7 @@ func main() {
 		MaxWorkers:                *maxWorkers,
 		WorkerQueueTimeout:        *workerQueueTimeout,
 		WorkerIdleTimeout:         *workerIdleTimeout,
+		HandoverDrainTimeout:      *handoverDrainTimeout,
 		ACMEDomain:                *acmeDomain,
 		ACMEEmail:                 *acmeEmail,
 		ACMECacheDir:              *acmeCacheDir,
@@ -425,9 +429,10 @@ func main() {
 			SocketDir:          *socketDir,
 			ConfigPath:         *configFile,
 			HandoverSocket:     *handoverSocket,
-			WorkerQueueTimeout: resolved.WorkerQueueTimeout,
-			WorkerIdleTimeout:  resolved.WorkerIdleTimeout,
-			MetricsServer:      metricsSrv,
+			WorkerQueueTimeout:   resolved.WorkerQueueTimeout,
+			WorkerIdleTimeout:    resolved.WorkerIdleTimeout,
+			HandoverDrainTimeout: resolved.HandoverDrainTimeout,
+			MetricsServer:        metricsSrv,
 		}
 		controlplane.RunControlPlane(cpCfg)
 		return