Allow configuring the kubernetes rest client timeout for draining nodes

Author: cnmcavoy

controllers/alias.go

@@ -18,6 +18,7 @@ package controllers
 
 import (
 	"context"
+	"time"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -67,6 +68,9 @@ type MachineReconciler struct {
 
 	// WatchFilterValue is the label value used to filter events prior to reconciliation.
 	WatchFilterValue string
+
+	// NodeDrainClientTimeout timeout of the client used for draining nodes.
+	NodeDrainClientTimeout time.Duration
 }
 
 func (r *MachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
@@ -76,6 +80,7 @@ func (r *MachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manag
 		APIReader:                 r.APIReader,
 		Tracker:                   r.Tracker,
 		WatchFilterValue:          r.WatchFilterValue,
+		NodeDrainClientTimeout:    r.NodeDrainClientTimeout,
 	}).SetupWithManager(ctx, mgr, options)
 }
 

internal/controllers/machine/machine_controller.go

@@ -30,6 +30,7 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	kubedrain "k8s.io/kubectl/pkg/drain"
@@ -81,6 +82,9 @@ type Reconciler struct {
 	// WatchFilterValue is the label value used to filter events prior to reconciliation.
 	WatchFilterValue string
 
+	// NodeDrainClientTimeout timeout of the client used for draining nodes.
+	NodeDrainClientTimeout time.Duration
+
 	controller      controller.Controller
 	recorder        record.EventRecorder
 	externalTracker external.ObjectTracker
@@ -602,6 +606,8 @@ func (r *Reconciler) drainNode(ctx context.Context, cluster *clusterv1.Cluster,
 		log.Error(err, "Error creating a remote client while deleting Machine, won't retry")
 		return ctrl.Result{}, nil
 	}
+	restConfig = rest.CopyConfig(restConfig)
+	restConfig.Timeout = r.NodeDrainClientTimeout
 	kubeClient, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
 		log.Error(err, "Error creating a remote client while deleting Machine, won't retry")

main.go

@@ -19,6 +19,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"os"
@@ -103,6 +104,7 @@ var (
 	syncPeriod                    time.Duration
 	restConfigQPS                 float32
 	restConfigBurst               int
+	nodeDrainClientTimeout        time.Duration
 	webhookPort                   int
 	webhookCertDir                string
 	healthAddr                    string
@@ -205,6 +207,9 @@ func InitFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&restConfigBurst, "kube-api-burst", 30,
 		"Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default 30")
 
+	fs.DurationVar(&nodeDrainClientTimeout, "node-drain-client-timeout-duration", time.Second*10,
+		"The timeout of the client used for draining nodes. Defaults to 10s")
+
 	fs.IntVar(&webhookPort, "webhook-port", 9443,
 		"Webhook Server port")
 
@@ -238,6 +243,11 @@ func main() {
 	restConfig.Burst = restConfigBurst
 	restConfig.UserAgent = remote.DefaultClusterAPIUserAgent(controllerName)
 
+	if nodeDrainClientTimeout <= 0 {
+		setupLog.Error(errors.New("node drain client timeout must be greater than zero"), "unable to start manager")
+		os.Exit(1)
+	}
+
 	minVer := version.MinimumKubernetesVersion
 	if feature.Gates.Enabled(feature.ClusterTopology) {
 		minVer = version.MinimumKubernetesVersionClusterTopology
@@ -475,6 +485,7 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) {
 		APIReader:                 mgr.GetAPIReader(),
 		Tracker:                   tracker,
 		WatchFilterValue:          watchFilterValue,
+		NodeDrainClientTimeout:    nodeDrainClientTimeout,
 	}).SetupWithManager(ctx, mgr, concurrency(machineConcurrency)); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Machine")
 		os.Exit(1)