Add an option to enable Prometheus with real certificates

While the install scripts do not enable Prometheus integration by default, solutions running upstream may want to use and enable it with Prometheus. This addition offers a way for upstream users to understand how to properly configure Prometheus using real certificates.

At the very least, it serves as documentation and provides an option for those installing from source who want to implement secure Prometheus integration.

Author: camilamacedo86

cmd/manager/main.go

@@ -21,6 +21,7 @@ import (
 	"crypto/tls"
 	"flag"
 	"fmt"
+	"log"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -42,6 +43,7 @@ import (
 	"k8s.io/klog/v2/textlogger"
 	ctrl "sigs.k8s.io/controller-runtime"
 	crcache "sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	crfinalizer "sigs.k8s.io/controller-runtime/pkg/finalizer"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -98,11 +100,15 @@ func main() {
 		operatorControllerVersion bool
 		systemNamespace           string
 		caCertDir                 string
+		certFile                  string
+		keyFile                   string
 		globalPullSecret          string
 	)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8443", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.StringVar(&caCertDir, "ca-certs-dir", "", "The directory of TLS certificate to use for verifying HTTPS connections to the Catalogd and docker-registry web servers.")
+	flag.StringVar(&certFile, "tls-cert", "", "The certificate file used for serving catalog contents over HTTPS. Requires tls-key.")
+	flag.StringVar(&keyFile, "tls-key", "", "The key file used for serving catalog contents over HTTPS. Requires tls-cert.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -122,6 +128,11 @@ func main() {
 		os.Exit(0)
 	}
 
+	if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") {
+		setupLog.Error(nil, "unable to configure TLS certificates: tls-cert and tls-key flags must be used together")
+		os.Exit(1)
+	}
+
 	ctrl.SetLogger(textlogger.NewLogger(textlogger.NewConfig()))
 
 	setupLog.Info("starting up the controller", "version info", version.String())
@@ -177,6 +188,21 @@ func main() {
 
 	tlsOpts = append(tlsOpts, disableHTTP2)
 
+	// Create a new certificate watcher to watch for changes in the certificate files.
+	// If the certificate files change, the certificate watcher will reload the certificate
+	// and key files and update the TLS configuration.
+	cw, err := certwatcher.New(certFile, keyFile)
+	if err != nil {
+		log.Fatalf("Failed to initialize certificate watcher: %v", err)
+	}
+
+	// Ensure that metrics is protected with certs managed by cert-manager
+	// If not informed, the metrics service provided by controller-runtime will generate
+	// and use its own self-assigned certs which is not recommended for production envs.
+	tlsOpts = append(tlsOpts, func(cfg *tls.Config) {
+		cfg.GetCertificate = cw.GetCertificate
+	})
+
 	metricsServerOptions := server.Options{
 		BindAddress:   metricsAddr,
 		SecureServing: true,
@@ -186,13 +212,6 @@ func main() {
 		// These configurations ensure that only authorized users and service accounts
 		// can access the metrics endpoint.
 		FilterProvider: filters.WithAuthenticationAndAuthorization,
-
-		// Ensure that metrics is protected with certs managed by cert-manager
-		// If not informed, the metrics service provided by controller-runtime will generate
-		// and use its own self-assigned certs which is not recommended for production envs.
-		CertDir:  "/var/certs/",
-		CertName: "olm-ca.crt",
-		KeyName:  "ca.crt",
 	}
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{

config/base/prometheus/monitor.yaml

@@ -1,19 +1,22 @@
-# Prometheus Monitor Service (Metrics)
+# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
+# using certificates managed by cert-manager
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
-  labels:
-    control-plane: operator-controller-controller-manager
   name: controller-manager-metrics-monitor
   namespace: system
 spec:
   endpoints:
-    - path: /metrics
-      port: https
-      scheme: https
-      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-      tlsConfig:
-        insecureSkipVerify: true
-  selector:
-    matchLabels:
-      control-plane: operator-controller-controller-manager
+    - tlsConfig:
+        insecureSkipVerify: false
+        ca:
+          secret:
+            name: olmv1-ca
+            key: ca.crt
+        cert:
+          secret:
+            name: olmv1-ca
+            key: olm-ca.crt
+        keySecret:
+          name: olmv1-ca
+          key: ca.crt

config/components/tls/patches/manager_deployment_cert.yaml

@@ -7,3 +7,9 @@
 - op: add
   path: /spec/template/spec/containers/0/args/-
   value: "--ca-certs-dir=/var/certs"
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: "--tls-cert=olm-ca.crt"
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: "--tls-key=ca.crt"