Merge remote-tracking branch 'origin/eks_overhead_friendly_benchmarks' into eks_overhead_friendly_benchmarks

Author: nathangeology

pkg/operator/operator.go

@@ -48,6 +48,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+	ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -179,6 +180,12 @@ func NewOperator(o ...option.Function[Options]) (context.Context, *Operator) {
 				},
 			},
 		},
+		Controller: ctrlconfig.Controller{
+			// EnableWarmup allows controllers to start their sources (watches/informers) before leader election
+			// is won. This pre-populates caches and improves leader failover time. Only effective when leader
+			// election is enabled, so we only set it when both conditions are true.
+			EnableWarmup: lo.ToPtr(!options.FromContext(ctx).DisableLeaderElection && !options.FromContext(ctx).DisableControllerWarmup),
+		},
 	}
 	if options.FromContext(ctx).EnableProfiling {
 		// TODO @joinnis: Investigate the mgrOpts.PprofBindAddress that would allow native support for pprof

pkg/operator/options/options.go

@@ -71,6 +71,7 @@ type Options struct {
 	KubeClientQPS                    int
 	KubeClientBurst                  int
 	EnableProfiling                  bool
+	DisableControllerWarmup          bool
 	DisableLeaderElection            bool
 	DisableClusterStateObservability bool
 	LeaderElectionName               string
@@ -114,6 +115,7 @@ func (o *Options) AddFlags(fs *FlagSet) {
 	fs.IntVar(&o.KubeClientQPS, "kube-client-qps", env.WithDefaultInt("KUBE_CLIENT_QPS", 200), "The smoothed rate of qps to kube-apiserver")
 	fs.IntVar(&o.KubeClientBurst, "kube-client-burst", env.WithDefaultInt("KUBE_CLIENT_BURST", 300), "The maximum allowed burst of queries to the kube-apiserver")
 	fs.BoolVarWithEnv(&o.EnableProfiling, "enable-profiling", "ENABLE_PROFILING", false, "Enable the profiling on the metric endpoint")
+	fs.BoolVarWithEnv(&o.DisableControllerWarmup, "disable-controller-warmup", "DISABLE_CONTROLLER_WARMUP", true, "Disable controller warmup which starts controller sources before leader election is won. Controller warmup pre-populates caches and improves leader failover time.")
 	fs.BoolVarWithEnv(&o.DisableLeaderElection, "disable-leader-election", "DISABLE_LEADER_ELECTION", false, "Disable the leader election client before executing the main loop. Disable when running replicated components for high availability is not desired.")
 	fs.BoolVarWithEnv(&o.DisableClusterStateObservability, "disable-cluster-state-observability", "DISABLE_CLUSTER_STATE_OBSERVABILITY", false, "Disable cluster state metrics and events")
 	fs.StringVar(&o.LeaderElectionName, "leader-election-name", env.WithDefaultString("LEADER_ELECTION_NAME", "karpenter-leader-election"), "Leader election name to create and monitor the lease if running outside the cluster")

pkg/operator/options/suite_test.go

@@ -52,6 +52,7 @@ var _ = Describe("Options", func() {
 		"KUBE_CLIENT_QPS",
 		"KUBE_CLIENT_BURST",
 		"ENABLE_PROFILING",
+		"DISABLE_CONTROLLER_WARMUP",
 		"DISABLE_LEADER_ELECTION",
 		"DISABLE_CLUSTER_STATE_OBSERVABILITY",
 		"LEADER_ELECTION_NAMESPACE",
@@ -106,6 +107,7 @@ var _ = Describe("Options", func() {
 				KubeClientQPS:                    lo.ToPtr(200),
 				KubeClientBurst:                  lo.ToPtr(300),
 				EnableProfiling:                  lo.ToPtr(false),
+				DisableControllerWarmup:          lo.ToPtr(true),
 				DisableLeaderElection:            lo.ToPtr(false),
 				DisableClusterStateObservability: lo.ToPtr(false),
 				LeaderElectionName:               lo.ToPtr("karpenter-leader-election"),
@@ -140,6 +142,7 @@ var _ = Describe("Options", func() {
 				"--kube-client-qps", "0",
 				"--kube-client-burst", "0",
 				"--enable-profiling",
+				"--disable-controller-warmup=false",
 				"--disable-leader-election=true",
 				"--disable-cluster-state-observability=true",
 				"--leader-election-name=karpenter-controller",
@@ -162,6 +165,7 @@ var _ = Describe("Options", func() {
 				KubeClientQPS:                    lo.ToPtr(0),
 				KubeClientBurst:                  lo.ToPtr(0),
 				EnableProfiling:                  lo.ToPtr(true),
+				DisableControllerWarmup:          lo.ToPtr(false),
 				DisableLeaderElection:            lo.ToPtr(true),
 				DisableClusterStateObservability: lo.ToPtr(true),
 				LeaderElectionName:               lo.ToPtr("karpenter-controller"),
@@ -192,6 +196,7 @@ var _ = Describe("Options", func() {
 			os.Setenv("KUBE_CLIENT_QPS", "0")
 			os.Setenv("KUBE_CLIENT_BURST", "0")
 			os.Setenv("ENABLE_PROFILING", "true")
+			os.Setenv("DISABLE_CONTROLLER_WARMUP", "false")
 			os.Setenv("DISABLE_LEADER_ELECTION", "true")
 			os.Setenv("DISABLE_CLUSTER_STATE_OBSERVABILITY", "true")
 			os.Setenv("LEADER_ELECTION_NAME", "karpenter-controller")
@@ -218,6 +223,7 @@ var _ = Describe("Options", func() {
 				KubeClientQPS:                    lo.ToPtr(0),
 				KubeClientBurst:                  lo.ToPtr(0),
 				EnableProfiling:                  lo.ToPtr(true),
+				DisableControllerWarmup:          lo.ToPtr(false),
 				DisableLeaderElection:            lo.ToPtr(true),
 				DisableClusterStateObservability: lo.ToPtr(true),
 				LeaderElectionName:               lo.ToPtr("karpenter-controller"),
@@ -247,6 +253,7 @@ var _ = Describe("Options", func() {
 			os.Setenv("KUBE_CLIENT_QPS", "0")
 			os.Setenv("KUBE_CLIENT_BURST", "0")
 			os.Setenv("ENABLE_PROFILING", "true")
+			os.Setenv("DISABLE_CONTROLLER_WARMUP", "false")
 			os.Setenv("DISABLE_LEADER_ELECTION", "true")
 			os.Setenv("DISABLE_CLUSTER_STATE_OBSERVABILITY", "true")
 			os.Setenv("MEMORY_LIMIT", "0")
@@ -276,6 +283,7 @@ var _ = Describe("Options", func() {
 				KubeClientQPS:                    lo.ToPtr(0),
 				KubeClientBurst:                  lo.ToPtr(0),
 				EnableProfiling:                  lo.ToPtr(true),
+				DisableControllerWarmup:          lo.ToPtr(false),
 				DisableLeaderElection:            lo.ToPtr(true),
 				DisableClusterStateObservability: lo.ToPtr(true),
 				LeaderElectionName:               lo.ToPtr("karpenter-leader-election"),
@@ -381,6 +389,7 @@ func expectOptionsMatch(optsA, optsB *options.Options) {
 	Expect(optsA.KubeClientQPS).To(Equal(optsB.KubeClientQPS))
 	Expect(optsA.KubeClientBurst).To(Equal(optsB.KubeClientBurst))
 	Expect(optsA.EnableProfiling).To(Equal(optsB.EnableProfiling))
+	Expect(optsA.DisableControllerWarmup).To(Equal(optsB.DisableControllerWarmup))
 	Expect(optsA.DisableLeaderElection).To(Equal(optsB.DisableLeaderElection))
 	Expect(optsA.DisableClusterStateObservability).To(Equal(optsB.DisableClusterStateObservability))
 	Expect(optsA.MemoryLimit).To(Equal(optsB.MemoryLimit))

pkg/test/options.go

@@ -34,6 +34,7 @@ type OptionsFields struct {
 	KubeClientQPS                    *int
 	KubeClientBurst                  *int
 	EnableProfiling                  *bool
+	DisableControllerWarmup          *bool
 	DisableLeaderElection            *bool
 	DisableClusterStateObservability *bool
 	LeaderElectionName               *string
@@ -74,6 +75,7 @@ func Options(overrides ...OptionsFields) *options.Options {
 		KubeClientQPS:                    lo.FromPtrOr(opts.KubeClientQPS, 200),
 		KubeClientBurst:                  lo.FromPtrOr(opts.KubeClientBurst, 300),
 		EnableProfiling:                  lo.FromPtrOr(opts.EnableProfiling, false),
+		DisableControllerWarmup:          lo.FromPtrOr(opts.DisableControllerWarmup, true),
 		DisableLeaderElection:            lo.FromPtrOr(opts.DisableLeaderElection, false),
 		DisableClusterStateObservability: lo.FromPtrOr(opts.DisableClusterStateObservability, false),
 		MemoryLimit:                      lo.FromPtrOr(opts.MemoryLimit, -1),