Replace kube-rbac-proxy to ensure the same level of protection with controller-runtime feature

Utilise Controller-Runtime's WithAuthenticationAndAuthorization feature to protect the metrics endpoint. This approach provides access control, similar to the functionality of kube-rbac-proxy. kube-rbac-proxy image from gcr.io/kubebuilder/kube-rbac-proxy is deprecated and should no longer be used

More info: kubernetes-sigs/kubebuilder#3907

Author: camilamacedo86

cmd/manager/main.go

@@ -18,8 +18,10 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"flag"
 	"fmt"
+	"log"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -41,9 +43,11 @@ import (
 	"k8s.io/klog/v2/textlogger"
 	ctrl "sigs.k8s.io/controller-runtime"
 	crcache "sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	crfinalizer "sigs.k8s.io/controller-runtime/pkg/finalizer"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	catalogd "github.com/operator-framework/catalogd/api/v1"
@@ -70,6 +74,7 @@ import (
 var (
 	setupLog               = ctrl.Log.WithName("setup")
 	defaultSystemNamespace = "olmv1-system"
+	certWatcher            *certwatcher.CertWatcher
 )
 
 const authFilePrefix = "operator-controller-global-pull-secrets"
@@ -89,17 +94,25 @@ func podNamespace() string {
 func main() {
 	var (
 		metricsAddr               string
+		certFile                  string
+		keyFile                   string
 		enableLeaderElection      bool
+		disableMetricsTLS         bool
 		probeAddr                 string
 		cachePath                 string
 		operatorControllerVersion bool
 		systemNamespace           string
 		caCertDir                 string
 		globalPullSecret          string
 	)
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
+		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.StringVar(&caCertDir, "ca-certs-dir", "", "The directory of TLS certificate to use for verifying HTTPS connections to the Catalogd and docker-registry web servers.")
+	flag.BoolVar(&disableMetricsTLS, "disable-metrics-tls", false,
+		"Disables TLS for the metrics endpoint, making it insecure. Use with caution. Defaults to secure (false).")
+	flag.StringVar(&certFile, "tls-cert", "", "The certificate file used for serving metrics contents over HTTPS. Requires tls-key.")
+	flag.StringVar(&keyFile, "tls-key", "", "The key file used for serving metrics contents over HTTPS. Requires tls-cert.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -119,6 +132,11 @@ func main() {
 		os.Exit(0)
 	}
 
+	if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") {
+		setupLog.Error(nil, "unable to configure TLS certificates: tls-cert and tls-key flags must be used together")
+		os.Exit(1)
+	}
+
 	ctrl.SetLogger(textlogger.NewLogger(textlogger.NewConfig()))
 
 	setupLog.Info("starting up the controller", "version info", version.String())
@@ -161,9 +179,46 @@ func main() {
 			},
 		}
 	}
+
+	metricsServerOptions := server.Options{
+		BindAddress:   metricsAddr,
+		SecureServing: !disableMetricsTLS,
+	}
+
+	if !disableMetricsTLS {
+		setupLog.Info("Metrics endpoint will be protected by TLS.")
+
+		// FilterProvider is used to protect the metrics endpoint with authn/authz.
+		// These configurations ensure that only authorized users and service accounts
+		// can access the metrics endpoint.
+		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+
+		// If the certificate files are provided, the metrics server will use them to serve the metrics endpoint.
+		// Otherwise, the metrics server will use the default certificate provided by the controller-runtime which
+		// is not recommended for production use.
+		if len(certFile) > 0 && len(keyFile) > 0 {
+			setupLog.Info("Using provided TLS certificate and key files for the metrics server.",
+				"certFile", certFile, "keyFile", keyFile)
+
+			// If the certificate files change, the watcher will reload them.
+			var err error
+			certWatcher, err = certwatcher.New(certFile, keyFile)
+			if err != nil {
+				log.Fatalf("Failed to initialize certificate watcher: %v", err)
+			}
+			metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) {
+				config.GetCertificate = certWatcher.GetCertificate
+			})
+		} else {
+			setupLog.Info("WARNING: No certificate files provided. (not recommended for production).")
+		}
+	} else {
+		setupLog.Info("WARNING: Metrics endpoint is not protected by TLS. Ensure this is intentional.")
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme.Scheme,
-		Metrics:                server.Options{BindAddress: metricsAddr},
+		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "9c4404e7.operatorframework.io",
@@ -337,6 +392,14 @@ func main() {
 
 	//+kubebuilder:scaffold:builder
 
+	if !disableMetricsTLS && certWatcher != nil {
+		setupLog.Info("Adding certificate watcher to manager")
+		if err := mgr.Add(certWatcher); err != nil {
+			setupLog.Error(err, "unable to add certificate watcher to manager")
+			os.Exit(1)
+		}
+	}
+
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)

config/base/manager/manager.yaml

@@ -52,7 +52,7 @@ spec:
         - /manager
         args:
         - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
+        - "--metrics-bind-address=0.0.0.0:8443"
         - "--leader-elect"
         image: controller:latest
         imagePullPolicy: IfNotPresent
@@ -84,27 +84,6 @@ spec:
             cpu: 10m
             memory: 64Mi
         terminationMessagePolicy: FallbackToLogsOnError
-      - name: kube-rbac-proxy
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-              - "ALL"
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
-        args:
-        - --secure-listen-address=0.0.0.0:8443
-        - --http2-disable
-        - --upstream=http://127.0.0.1:8080/
-        - --logtostderr=true
-        ports:
-        - containerPort: 8443
-          protocol: TCP
-          name: https
-        resources:
-          requests:
-            cpu: 5m
-            memory: 64Mi
-        terminationMessagePolicy: FallbackToLogsOnError
       serviceAccountName: operator-controller-controller-manager
       terminationGracePeriodSeconds: 10
       volumes:

config/base/rbac/kustomization.yaml

@@ -17,10 +17,14 @@ resources:
 - extension_editor_role.yaml
 - extension_viewer_role.yaml
 
-# Comment the following 4 lines if you want to disable
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-# which protects your /metrics endpoint.
+# The following RBAC configurations are used to protect
+# the metrics endpoint with authn/authz. These configurations
+# ensure that only authorized users and service accounts
+# can access the metrics endpoint. Comment the following
+# permissions if you want to disable this protection.
+# More info: https://book.kubebuilder.io/reference/metrics.html
 - auth_proxy_service.yaml
 - auth_proxy_role.yaml
 - auth_proxy_role_binding.yaml
 - auth_proxy_client_clusterrole.yaml
+

config/components/coverage/manager_e2e_coverage_patch.yaml

@@ -7,7 +7,6 @@ spec:
   template:
     spec:
       containers:
-      - name: kube-rbac-proxy
       - name: manager
         env:
         - name: GOCOVERDIR

config/components/registries-conf/manager_e2e_registries_conf_patch.yaml

@@ -7,7 +7,6 @@ spec:
   template:
     spec:
       containers:
-      - name: kube-rbac-proxy
       - name: manager
         volumeMounts:
         - name: e2e-registries-conf

config/components/tls/patches/manager_deployment_cert.yaml

@@ -1,9 +1,15 @@
 - op: add
   path: /spec/template/spec/volumes/-
-  value: {"name":"olmv1-certificate", "secret":{"secretName":"olmv1-cert", "optional": false, "items": [{"key": "ca.crt", "path": "olm-ca.crt"}]}}
+  value: {"name":"olmv1-certificate", "secret":{"secretName":"olmv1-cert", "optional": false, "items": [{"key": "ca.crt", "path": "olm-ca.crt"}, {"key": "tls.crt", "path": "tls.cert"}, {"key": "tls.key", "path": "tls.key"}]}}
 - op: add
   path: /spec/template/spec/containers/0/volumeMounts/-
   value: {"name":"olmv1-certificate", "readOnly": true, "mountPath":"/var/certs/"}
 - op: add
   path: /spec/template/spec/containers/0/args/-
   value: "--ca-certs-dir=/var/certs"
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: "--tls-cert=/var/certs/tls.cert"
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: "--tls-key=/var/certs/tls.key"

config/components/tls/resources/manager_cert.yaml

@@ -7,6 +7,8 @@ spec:
   dnsNames:
     - operator-controller.olmv1-system.svc
     - operator-controller.olmv1-system.svc.cluster.local
+    - controller-manager-metrics-service.olmv1-system.svc
+    - controller-manager-metrics-service.olmv1-system.svc.cluster.local
   privateKey:
     algorithm: ECDSA
     size: 256

testdata/build-test-registry.sh

@@ -42,6 +42,8 @@ spec:
   dnsNames:
     - ${name}.${namespace}.svc
     - ${name}.${namespace}.svc.cluster.local
+    - controller-manager-metrics-service.${namespace}.svc
+    - controller-manager-metrics-service.${namespace}.svc.cluster.local
   privateKey:
     algorithm: ECDSA
     size: 256