✨ Add protection to metrics endpoint using authn/authz via controller-runtime feature (#4003)

Add protection to metrics endpoint using authn/authz via controller-runtime feature

This PR re-introduce authn/authz protection for the endpoint but without
use the kube-rbac-proxy project and image.

Signed-off-by: Joe Lanford <[email protected]>
Co-authored-by: Joe Lanford <[email protected]>

Author: camilamacedo86

.github/workflows/test-sample-go.yml

@@ -24,7 +24,7 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
-          sed -i '47s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '46s/^#//' $KUSTOMIZATION_FILE_PATH
 
       - name: Test
         run: |

docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go

@@ -32,6 +32,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
@@ -76,14 +77,14 @@ func main() {
 	var probeAddr string
 	var secureMetrics bool
 	var enableHTTP2 bool
-	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+
-		"Use the port :8080. If not set, it will be 0 in order to disable the metrics server")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
+		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	flag.BoolVar(&secureMetrics, "metrics-secure", false,
-		"If set the metrics endpoint is served securely")
+	flag.BoolVar(&secureMetrics, "metrics-secure", true,
+		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	opts := zap.Options{
@@ -116,10 +117,25 @@ func main() {
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme: scheme,
+		// Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server.
+		// More info:
+		// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/server
+		// - https://book.kubebuilder.io/reference/metrics.html
 		Metrics: metricsserver.Options{
 			BindAddress:   metricsAddr,
 			SecureServing: secureMetrics,
-			TLSOpts:       tlsOpts,
+			// TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are
+			// not provided, self-signed certificates will be generated by default. This option is not recommended for
+			// production environments as self-signed certificates do not offer the same level of trust and security
+			// as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing
+			// unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName
+			// to provide certificates, ensuring the server communicates using trusted and secure certificates.
+			TLSOpts: tlsOpts,
+			// FilterProvider is used to protect the metrics endpoint with authn/authz.
+			// These configurations ensure that only authorized users and service accounts
+			// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
+			// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
+			FilterProvider: filters.WithAuthenticationAndAuthorization,
 		},
 		WebhookServer:          webhookServer,
 		HealthProbeBindAddress: probeAddr,

docs/book/src/cronjob-tutorial/testdata/project/config/default/kustomization.yaml

@@ -25,17 +25,16 @@ resources:
 - ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 - ../prometheus
-# [METRICS] To enable the controller manager metrics service, uncomment the following line.
-#- metrics_service.yaml
+# [METRICS] Expose the controller manager metrics service.
+- metrics_service.yaml
 
 # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager
 patches:
-# [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint.
+# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
 # More info: https://book.kubebuilder.io/reference/metrics
-# If you want to expose the metric endpoint of your controller-manager uncomment the following line.
-#- path: manager_metrics_patch.yaml
-#  target:
-#    kind: Deployment
+- path: manager_metrics_patch.yaml
+  target:
+    kind: Deployment
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml

docs/book/src/cronjob-tutorial/testdata/project/config/default/manager_metrics_patch.yaml

@@ -1,4 +1,4 @@
-# This patch adds the args to allow exposing the metrics endpoint securely
+# This patch adds the args to allow exposing the metrics endpoint using HTTPS
 - op: add
   path: /spec/template/spec/containers/0/args/0
-  value: --metrics-bind-address=:8080
+  value: --metrics-bind-address=:8443

docs/book/src/cronjob-tutorial/testdata/project/config/default/metrics_service.yaml

@@ -9,9 +9,9 @@ metadata:
   namespace: system
 spec:
   ports:
-  - name: http
-    port: 8080
+  - name: https
+    port: 8443
     protocol: TCP
-    targetPort: 8080
+    targetPort: 8443
   selector:
     control-plane: controller-manager

docs/book/src/cronjob-tutorial/testdata/project/config/prometheus/monitor.yaml

@@ -11,8 +11,20 @@ metadata:
 spec:
   endpoints:
     - path: /metrics
-      port: http # Ensure this is the name of the port that exposes HTTP metrics
-      scheme: http
+      port: https # Ensure this is the name of the port that exposes HTTPS metrics
+      scheme: https
+      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      tlsConfig:
+        # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
+        # certificate verification. This poses a significant security risk by making the system vulnerable to
+        # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
+        # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
+        # compromising the integrity and confidentiality of the information.
+        # Please use the following options for secure configurations:
+        # caFile: /etc/metrics-certs/ca.crt
+        # certFile: /etc/metrics-certs/tls.crt
+        # keyFile: /etc/metrics-certs/tls.key
+        insecureSkipVerify: true
   selector:
     matchLabels:
       control-plane: controller-manager

docs/book/src/cronjob-tutorial/testdata/project/config/rbac/kustomization.yaml

@@ -9,6 +9,15 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
+# The following RBAC configurations are used to protect
+# the metrics endpoint with authn/authz. These configurations
+# ensure that only authorized users and service accounts
+# can access the metrics endpoint. Comment the following
+# permissions if you want to disable this protection.
+# More info: https://book.kubebuilder.io/reference/metrics.html
+- metrics_auth_role.yaml
+- metrics_auth_role_binding.yaml
+- metrics_reader_role.yaml
 # For each CRD, "Editor" and "Viewer" roles are scaffolded by
 # default, aiding admins in cluster management. Those roles are
 # not used by the Project itself. You can comment the following lines

docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_auth_role.yaml

@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-auth-role
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create

docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_auth_role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-auth-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-auth-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system

docs/book/src/cronjob-tutorial/testdata/project/config/rbac/metrics_reader_role.yaml

@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-reader
+rules:
+- nonResourceURLs:
+  - "/metrics"
+  verbs:
+  - get