Add controller manager structured config

Author: astefanutti

config/manager/manager.yaml

@@ -34,9 +34,6 @@ spec:
       containers:
       - command:
         - /manager
-        args:
-        - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=0.0.0.0:8082"
         image: controller:latest
         imagePullPolicy: Always
         name: manager

main.go

@@ -32,6 +32,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
 	configv1alpha1 "k8s.io/component-base/config/v1alpha1"
 	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -56,12 +57,6 @@ func init() {
 }
 
 func main() {
-	var metricsAddr string
-	var probeAddr string
-
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
-	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-
 	zapOptions := zap.Options{
 		Development: true,
 		TimeEncoder: zapcore.TimeEncoderOfLayout(time.RFC3339),
@@ -73,15 +68,27 @@ func main() {
 	ctx := ctrl.SetupSignalHandler()
 
 	cfg := config.CodeFlareOperatorConfiguration{
-		LeaderElection: &configv1alpha1.LeaderElectionConfiguration{},
-		MCAD:           &mcadconfig.MCADConfiguration{},
-		InstaScale:     &config.InstaScaleConfiguration{},
+		ControllerManager: config.ControllerManager{
+			LeaderElection: &configv1alpha1.LeaderElectionConfiguration{},
+		},
+		MCAD:       &mcadconfig.MCADConfiguration{},
+		InstaScale: &config.InstaScaleConfiguration{},
 	}
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	kubeConfig, err := ctrl.GetConfig()
+	exitOnError(err, "unable to get client config")
+
+	if kubeConfig.UserAgent == "" {
+		kubeConfig.UserAgent = "codeflare-operator"
+	}
+	kubeConfig.Burst = int(pointer.Int32Deref(cfg.ClientConnection.Burst, int32(rest.DefaultBurst)))
+	kubeConfig.QPS = pointer.Float32Deref(cfg.ClientConnection.QPS, rest.DefaultQPS)
+	setupLog.V(2).Info("REST client", "qps", kubeConfig.QPS, "burst", kubeConfig.Burst)
+
+	mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{
 		Scheme:                     scheme,
-		MetricsBindAddress:         metricsAddr,
-		HealthProbeBindAddress:     probeAddr,
+		MetricsBindAddress:         cfg.Metrics.BindAddress,
+		HealthProbeBindAddress:     cfg.Health.BindAddress,
 		LeaderElection:             pointer.BoolDeref(cfg.LeaderElection.LeaderElect, false),
 		LeaderElectionID:           cfg.LeaderElection.ResourceName,
 		LeaderElectionNamespace:    cfg.LeaderElection.ResourceNamespace,
@@ -90,10 +97,7 @@ func main() {
 		RetryPeriod:                &cfg.LeaderElection.RetryPeriod.Duration,
 		RenewDeadline:              &cfg.LeaderElection.RenewDeadline.Duration,
 	})
-	if err != nil {
-		setupLog.Error(err, "unable to start manager")
-		os.Exit(1)
-	}
+	exitOnError(err, "unable to start manager")
 
 	mcadQueueController := mcad.NewJobController(mgr.GetConfig(), cfg.MCAD, &mcadconfig.MCADConfigurationExtended{})
 	if mcadQueueController == nil {
@@ -111,20 +115,11 @@ func main() {
 		exitOnError(instaScaleController.SetupWithManager(mgr), "Error setting up InstaScale controller")
 	}
 
-	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
-		setupLog.Error(err, "unable to set up health check")
-		os.Exit(1)
-	}
-	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
-		setupLog.Error(err, "unable to set up ready check")
-		os.Exit(1)
-	}
+	exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check")
+	exitOnError(mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, healthz.Ping), "unable to set up ready check")
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "problem running manager")
-		os.Exit(1)
-	}
+	exitOnError(mgr.Start(ctx), "error running manager")
 }
 
 func exitOnError(err error, msg string) {

pkg/config/config.go

@@ -24,9 +24,12 @@ import (
 )
 
 type CodeFlareOperatorConfiguration struct {
-	// LeaderElection is the LeaderElection config to be used when configuring
-	// the manager.Manager leader election
-	LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"`
+	// ClientConnection provides additional configuration options for Kubernetes
+	// API server client.
+	ClientConnection *ClientConnection `json:"clientConnection,omitempty"`
+
+	// ControllerManager returns the configurations for controllers
+	ControllerManager `json:",inline"`
 
 	// The MCAD controller configuration
 	MCAD *mcad.MCADConfiguration `json:"mcad,omitempty"`
@@ -44,3 +47,52 @@ type InstaScaleConfiguration struct {
 	// The InstaScale controller configuration
 	instascale.InstaScaleConfiguration `json:",inline,omitempty"`
 }
+
+type ControllerManager struct {
+	// Metrics contains the controller metrics configuration
+	// +optional
+	Metrics MetricsConfiguration `json:"metrics,omitempty"`
+
+	// Health contains the controller health configuration
+	// +optional
+	Health HealthConfiguration `json:"health,omitempty"`
+
+	// LeaderElection is the LeaderElection config to be used when configuring
+	// the manager.Manager leader election
+	LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"`
+}
+
+type ClientConnection struct {
+	// QPS controls the number of queries per second allowed before client-side throttling
+	// connection to the API server.
+	QPS *float32 `json:"qps,omitempty"`
+
+	// Burst allows extra queries to accumulate when a client is exceeding its rate.
+	Burst *int32 `json:"burst,omitempty"`
+}
+
+// MetricsConfiguration defines the metrics configuration.
+type MetricsConfiguration struct {
+	// BindAddress is the TCP address that the controller should bind to
+	// for serving Prometheus metrics.
+	// It can be set to "0" to disable the metrics serving.
+	// +optional
+	BindAddress string `json:"bindAddress,omitempty"`
+}
+
+// HealthConfiguration defines the health configuration.
+type HealthConfiguration struct {
+	// BindAddress is the TCP address that the controller should bind to
+	// for serving health probes.
+	// It can be set to "0" or "" to disable serving the health probe.
+	// +optional
+	BindAddress string `json:"bindAddress,omitempty"`
+
+	// ReadinessEndpointName, defaults to "readyz"
+	// +optional
+	ReadinessEndpointName string `json:"readinessEndpointName,omitempty"`
+
+	// LivenessEndpointName, defaults to "healthz"
+	// +optional
+	LivenessEndpointName string `json:"livenessEndpointName,omitempty"`
+}