✨ (Blocked by #1475) - Ensure that metrics is protected with TLS for Prometheus integration.

cmd/manager/main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"flag"
 	"fmt"
 	"net/http"
@@ -44,6 +45,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	crfinalizer "sigs.k8s.io/controller-runtime/pkg/finalizer"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	catalogd "github.com/operator-framework/catalogd/api/v1"
@@ -89,6 +91,9 @@ func podNamespace() string {
 func main() {
 	var (
 		metricsAddr               string
+		secureMetrics             bool
+		tlsOpts                   []func(*tls.Config)
+		enableHTTP2               bool
 		enableLeaderElection      bool
 		probeAddr                 string
 		cachePath                 string
@@ -97,7 +102,11 @@ func main() {
 		caCertDir                 string
 		globalPullSecret          string
 	)
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8443", "The address the metric endpoint binds to.")
+	flag.BoolVar(&secureMetrics, "metrics-secure", true,
+		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
+	flag.BoolVar(&enableHTTP2, "enable-http2", false,
+		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.StringVar(&caCertDir, "ca-certs-dir", "", "The directory of TLS certificate to use for verifying HTTPS connections to the Catalogd and docker-registry web servers.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
@@ -161,9 +170,43 @@ func main() {
 			},
 		}
 	}
+
+	// if the enable-http2 flag is false (the default), http/2 should be disabled
+	// due to its vulnerabilities. More specifically, disabling http/2 will
+	// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
+	// Rapid Reset CVEs. For more information see:
+	// - https://github.com/advisories/GHSA-qppj-fm5r-hxr3
+	// - https://github.com/advisories/GHSA-4374-p667-p6c8
+	disableHTTP2 := func(c *tls.Config) {
+		setupLog.Info("disabling http/2")
+		c.NextProtos = []string{"http/1.1"}
+	}
+
+	if !enableHTTP2 {
+		tlsOpts = append(tlsOpts, disableHTTP2)
+	}
+
+	metricsServerOptions := server.Options{
+		BindAddress:   metricsAddr,
+		SecureServing: secureMetrics,
+		TLSOpts:       tlsOpts,
+	}
+
+	if secureMetrics {
+		// FilterProvider is used to protect the metrics endpoint with authn/authz.
+		// These configurations ensure that only authorized users and service accounts
+		// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
+		// https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/metrics/filters#WithAuthenticationAndAuthorization
+		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+
+		metricsServerOptions.CertDir = "/var/metrics/certs"
+		metricsServerOptions.CertName = "tls.crt"
+		metricsServerOptions.KeyName = "tls.key"
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme.Scheme,
-		Metrics:                server.Options{BindAddress: metricsAddr},
+		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "9c4404e7.operatorframework.io",

config/base/manager/manager.yaml

@@ -52,7 +52,7 @@ spec:
         - /manager
         args:
         - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
+        - "--metrics-bind-address=127.0.0.1:8443"
         - "--leader-elect"
         image: controller:latest
         imagePullPolicy: IfNotPresent
@@ -84,27 +84,6 @@ spec:
             cpu: 10m
             memory: 64Mi
         terminationMessagePolicy: FallbackToLogsOnError
-      - name: kube-rbac-proxy
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-              - "ALL"
-        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
-        args:
-        - --secure-listen-address=0.0.0.0:8443
-        - --http2-disable
-        - --upstream=http://127.0.0.1:8080/
-        - --logtostderr=true
-        ports:
-        - containerPort: 8443
-          protocol: TCP
-          name: https
-        resources:
-          requests:
-            cpu: 5m
-            memory: 64Mi
-        terminationMessagePolicy: FallbackToLogsOnError
       serviceAccountName: operator-controller-controller-manager
       terminationGracePeriodSeconds: 10
       volumes:

config/base/prometheus/kustomization.yaml

@@ -1,2 +1,11 @@
 resources:
 - monitor.yaml
+
+# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
+# to securely reference certificates created and managed by cert-manager.
+# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
+# to mount the "metrics-server-cert" secret in the Manager Deployment.
+patches:
+  - path: monitor_tls_patch.yaml
+    target:
+      kind: ServiceMonitor

config/base/prometheus/monitor.yaml

@@ -17,3 +17,4 @@ spec:
   selector:
     matchLabels:
       control-plane: operator-controller-controller-manager
+

config/base/prometheus/monitor_tls_patch.yaml

@@ -0,0 +1,22 @@
+# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
+# using certificates managed by cert-manager
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: controller-manager-metrics-monitor
+  namespace: system
+spec:
+  endpoints:
+    - tlsConfig:
+        insecureSkipVerify: false
+        ca:
+          secret:
+            name: olmv1-metrics-ca
+            key: ca.crt
+        cert:
+          secret:
+            name: olmv1-metrics-ca
+            key: tls.crt
+        keySecret:
+          name: olmv1-metrics-ca
+          key: tls.key

config/base/rbac/kustomization.yaml

@@ -17,10 +17,12 @@ resources:
 - extension_editor_role.yaml
 - extension_viewer_role.yaml
 
-# Comment the following 4 lines if you want to disable
-# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
-# which protects your /metrics endpoint.
-- auth_proxy_service.yaml
-- auth_proxy_role.yaml
-- auth_proxy_role_binding.yaml
-- auth_proxy_client_clusterrole.yaml
+# The following RBAC configurations are used to protect
+# the metrics endpoint with authn/authz. These configurations
+# ensure that only authorized users and service accounts
+# can access the metrics endpoint. Comment the following
+# permissions if you want to disable this protection.
+# More info: https://book.kubebuilder.io/reference/metrics.html
+- metrics_auth_role.yaml
+- metrics_auth_role_binding.yaml
+- metrics_reader_role.yaml

config/base/rbac/metrics_auth_role.yaml

@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-auth-role
+rules:
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create

config/base/rbac/auth_proxy_role_binding.yaml → config/base/rbac/metrics_auth_role_binding.yaml

@@ -1,12 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: proxy-rolebinding
+  name: metrics-auth-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: proxy-role
+  name: metrics-auth-role
 subjects:
-- kind: ServiceAccount
-  name: controller-manager
-  namespace: system
+  - kind: ServiceAccount
+    name: controller-manager
+    namespace: system

config/components/ca/issuers.yaml

@@ -30,3 +30,28 @@ metadata:
 spec:
   ca:
     secretName: olmv1-ca
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: olmv1-metrics-ca
+  namespace: cert-manager
+spec:
+  isCA: true
+  commonName: olmv1-metrics-ca
+  secretName: olmv1-metrics-ca
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: self-sign-issuer
+    kind: Issuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: olmv1-metrics-ca
+spec:
+  ca:
+    secretName: olmv1-metrics-ca

config/components/coverage/manager_e2e_coverage_patch.yaml

@@ -7,7 +7,6 @@ spec:
   template:
     spec:
       containers:
-      - name: kube-rbac-proxy
       - name: manager
         env:
         - name: GOCOVERDIR

config/components/registries-conf/manager_e2e_registries_conf_patch.yaml

@@ -7,7 +7,6 @@ spec:
   template:
     spec:
       containers:
-      - name: kube-rbac-proxy
       - name: manager
         volumeMounts:
         - name: e2e-registries-conf

config/components/tls/kustomization.yaml

@@ -3,8 +3,13 @@ kind: Component
 namespace: olmv1-system
 resources:
 - resources/manager_cert.yaml
+- resources/manager_metrics_cert.yaml
 patches:
 - target:
     kind: Deployment
     name: controller-manager
   path: patches/manager_deployment_cert.yaml
+- target:
+    kind: Deployment
+    name: controller-manager
+  path: patches/manager_metrics_deployment_cert.yaml

config/components/tls/patches/manager_metrics_deployment_cert.yaml

@@ -0,0 +1,12 @@
+- op: add
+  path: /spec/template/spec/volumes/-
+  value:
+    name: olmv1-metrics-ca
+    secret:
+      secretName: olmv1-metrics-ca
+- op: add
+  path: /spec/template/spec/containers/0/volumeMounts/-
+  value:
+    name: olmv1-metrics-ca
+    mountPath: /var/metrics/certs
+    readOnly: true

config/components/tls/resources/manager_metrics_cert.yaml

@@ -0,0 +1,16 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: olmv1-metrics-ca
+spec:
+  secretName: olmv1-metrics-ca
+  dnsNames:
+    - controller-manager-metrics-service.olmv1-system.svc
+    - controller-manager-metrics-service.olmv1-system.svc.cluster.local
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: olmv1-metrics-ca
+    kind: ClusterIssuer
+    group: cert-manager.io