demonstrating native metrics authn/authz using controller-runtime

Signed-off-by: Joe Lanford <[email protected]>

Author: joelanford

Makefile

@@ -157,7 +157,16 @@ local-build: IMAGE_TAG = local
 local-build: image
 
 .PHONY: run-local
-run-local: local-build kind-create deploy
+run-local: local-build kind-create deploy tmp-certs
+
+.PHONY: tmp-certs
+tmp-certs:
+	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
+	kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-webhook --timeout=60s
+	kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-cainjector --timeout=60s
+	kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager --timeout=60s
+	kubectl apply -f issuer.yaml
+	kubectl apply -f certificate.yaml
 
 .PHONY: clean
 clean: #HELP Clean up build artifacts
@@ -233,14 +242,19 @@ deploy: $(KIND) $(HELM) #HELP Deploy OLM to kind cluster $KIND_CLUSTER_NAME (def
 		--set debug=true \
 		--set olm.image.ref=$(OLM_IMAGE) \
 		--set olm.image.pullPolicy=IfNotPresent \
+		--set olm.tlsSecret=olm-cert \
+		--set olm.clientCASecret=olm-cert \
+		--set olm.service.internalPort=8443 \
 		--set catalog.image.ref=$(OLM_IMAGE) \
 		--set catalog.image.pullPolicy=IfNotPresent \
+		--set catalog.tlsSecret=olm-cert \
+		--set catalog.clientCASecret=olm-cert \
+		--set catalog.service.internalPort=8443 \
 		--set catalog.commandArgs=--configmapServerImage=$(CONFIGMAP_SERVER_IMAGE) \
 		--set catalog.opmImageArgs=--opmImage=$(OPERATOR_REGISTRY_IMAGE) \
 		--set package.image.ref=$(OLM_IMAGE) \
 		--set package.image.pullPolicy=IfNotPresent \
-		$(HELM_INSTALL_OPTS) \
-		--wait;
+		$(HELM_INSTALL_OPTS);
 
 .PHONY: undeploy
 undeploy: $(KIND) $(HELM) #HELP Uninstall OLM from kind cluster $KIND_CLUSTER_NAME (default: kind-olmv0)

certificate.yaml

@@ -0,0 +1,22 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: olm
+  namespace: operator-lifecycle-manager
+spec:
+  secretName: olm-cert
+  isCA: false
+  usages:
+    - server auth
+  dnsNames:
+    - localhost
+    - catalog-operator.operator-lifecycle-manager.svc
+    - olm-operator.operator-lifecycle-manager.svc
+  issuerRef:
+    name: issuer
+    # We can reference ClusterIssuers by changing the kind here.
+    # The default value is Issuer (i.e. a locally namespaced Issuer)
+    kind: Issuer
+    # This is optional since cert-manager will default to this value however
+    # if you are using an external issuer, change this to that issuer group.
+    group: cert-manager.io

cmd/catalog/main.go

@@ -57,9 +57,16 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error {
 		o.catalogNamespace = catalogNamespaceEnvVarValue
 	}
 
+	// create a config client for operator status
+	config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig)
+	if err != nil {
+		return fmt.Errorf("error configuring client: %s", err.Error())
+	}
+
 	listenAndServe, err := server.GetListenAndServeFunc(
 		server.WithLogger(logger),
 		server.WithTLS(&o.tlsCertPath, &o.tlsKeyPath, &o.clientCAPath),
+		server.WithKubeConfig(config),
 		server.WithDebug(o.debug),
 	)
 	if err != nil {
@@ -72,11 +79,6 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error {
 		}
 	}()
 
-	// create a config client for operator status
-	config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig)
-	if err != nil {
-		return fmt.Errorf("error configuring client: %s", err.Error())
-	}
 	configClient, err := configv1client.NewForConfig(config)
 	if err != nil {
 		return fmt.Errorf("error configuring client: %s", err.Error())

cmd/olm/main.go

@@ -123,7 +123,18 @@ func main() {
 	}
 	logger.Infof("log level %s", logger.Level)
 
-	listenAndServe, err := server.GetListenAndServeFunc(server.WithLogger(logger), server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath), server.WithDebug(*debug))
+	mgr, err := Manager(ctx, *debug)
+	if err != nil {
+		logger.WithError(err).Fatal("error configuring controller manager")
+	}
+	config := mgr.GetConfig()
+
+	listenAndServe, err := server.GetListenAndServeFunc(
+		server.WithLogger(logger),
+		server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath),
+		server.WithKubeConfig(config),
+		server.WithDebug(*debug),
+	)
 	if err != nil {
 		logger.Fatalf("Error setting up health/metric/pprof service: %v", err)
 	}
@@ -134,12 +145,6 @@ func main() {
 		}
 	}()
 
-	mgr, err := Manager(ctx, *debug)
-	if err != nil {
-		logger.WithError(err).Fatal("error configuring controller manager")
-	}
-	config := mgr.GetConfig()
-
 	// create a config that validates we're creating objects with labels
 	validatingConfig := validatingroundtripper.Wrap(config, mgr.GetScheme())
 

issuer.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: issuer
+  namespace: operator-lifecycle-manager
+spec:
+  selfSigned: {}

pkg/lib/server/server.go

@@ -6,11 +6,15 @@ import (
 	"fmt"
 	"net/http"
 	"path/filepath"
+	"time"
 
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/filemonitor"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 )
 
 // Option applies a configuration option to the given config.
@@ -43,11 +47,18 @@ func WithDebug(debug bool) Option {
 	}
 }
 
+func WithKubeConfig(config *rest.Config) Option {
+	return func(sc *serverConfig) {
+		sc.kubeConfig = config
+	}
+}
+
 type serverConfig struct {
 	logger       *logrus.Logger
 	tlsCertPath  *string
 	tlsKeyPath   *string
 	clientCAPath *string
+	kubeConfig   *rest.Config
 	debug        bool
 }
 
@@ -62,6 +73,7 @@ func defaultServerConfig() serverConfig {
 		tlsCertPath:  nil,
 		tlsKeyPath:   nil,
 		clientCAPath: nil,
+		kubeConfig:   nil,
 		logger:       nil,
 		debug:        false,
 	}
@@ -90,12 +102,53 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) {
 	}
 
 	mux := http.NewServeMux()
-	mux.Handle("/metrics", promhttp.Handler())
 	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 	})
 	profile.RegisterHandlers(mux, profile.WithTLS(tlsEnabled || !sc.debug))
 
+	// Set up authenticated metrics endpoint if kubeConfig is provided
+	sc.logger.Infof("DEBUG: Checking authentication setup - kubeConfig != nil: %v, tlsEnabled: %v", sc.kubeConfig != nil, tlsEnabled)
+	if sc.kubeConfig != nil && tlsEnabled {
+		sc.logger.Info("DEBUG: Setting up authenticated metrics endpoint")
+		// Create authentication filter using controller-runtime
+		sc.logger.Info("DEBUG: Creating authentication filter with controller-runtime")
+		filter, err := filters.WithAuthenticationAndAuthorization(sc.kubeConfig, &http.Client{
+			Timeout: 30 * time.Second,
+		})
+		if err != nil {
+			sc.logger.Errorf("DEBUG: Failed to create authentication filter: %v", err)
+			return nil, fmt.Errorf("failed to create authentication filter: %w", err)
+		}
+		sc.logger.Info("DEBUG: Authentication filter created successfully")
+		// Create authenticated metrics handler
+		sc.logger.Info("DEBUG: Wrapping metrics handler with authentication")
+		logger := log.FromContext(context.Background())
+		authenticatedMetricsHandler, err := filter(logger, promhttp.Handler())
+		if err != nil {
+			sc.logger.Errorf("DEBUG: Failed to wrap metrics handler: %v", err)
+			return nil, fmt.Errorf("failed to wrap metrics handler with authentication: %w", err)
+		}
+		sc.logger.Info("DEBUG: Metrics handler wrapped successfully")
+		// Add debugging wrapper to log authentication attempts
+		debugAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			sc.logger.Infof("DEBUG: Metrics request from %s, Auth header present: %v, User-Agent: %s",
+				r.RemoteAddr, r.Header.Get("Authorization") != "", r.Header.Get("User-Agent"))
+			authenticatedMetricsHandler.ServeHTTP(w, r)
+		})
+		mux.Handle("/metrics", debugAuthHandler)
+		sc.logger.Info("Metrics endpoint configured with authentication and authorization")
+	} else {
+		// Fallback to unprotected metrics (for development/testing)
+		sc.logger.Warnf("DEBUG: Using unprotected metrics - kubeConfig != nil: %v, tlsEnabled: %v", sc.kubeConfig != nil, tlsEnabled)
+		mux.Handle("/metrics", promhttp.Handler())
+		if sc.kubeConfig == nil {
+			sc.logger.Warn("No Kubernetes config provided - metrics endpoint will be unprotected")
+		} else if !tlsEnabled {
+			sc.logger.Warn("TLS not enabled - metrics endpoint will be unprotected")
+		}
+	}
+
 	s := http.Server{
 		Handler: mux,
 		Addr:    sc.getAddress(tlsEnabled),
@@ -141,6 +194,7 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) {
 				ClientAuth:   tls.VerifyClientCertIfGiven,
 			}, nil
 		},
+		NextProtos: []string{"http/1.1"}, // Disable HTTP/2 for security
 	}
 	return func() error {
 		return s.ListenAndServeTLS("", "")

vendor/modules.txt

@@ -1873,6 +1873,7 @@ sigs.k8s.io/controller-runtime/pkg/log/zap
 sigs.k8s.io/controller-runtime/pkg/manager
 sigs.k8s.io/controller-runtime/pkg/manager/signals
 sigs.k8s.io/controller-runtime/pkg/metrics
+sigs.k8s.io/controller-runtime/pkg/metrics/filters
 sigs.k8s.io/controller-runtime/pkg/metrics/server
 sigs.k8s.io/controller-runtime/pkg/predicate
 sigs.k8s.io/controller-runtime/pkg/reconcile