Skip to content

Commit e33627e

Browse files
committed
demonstrating native metrics authn/authz using controller-runtime
Signed-off-by: Joe Lanford <[email protected]>
1 parent ccf0c4c commit e33627e

File tree

8 files changed

+244
-16
lines changed

8 files changed

+244
-16
lines changed

Makefile

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,16 @@ local-build: IMAGE_TAG = local
157157
local-build: image
158158

159159
.PHONY: run-local
160-
run-local: local-build kind-create deploy
160+
run-local: local-build kind-create deploy tmp-certs
161+
162+
.PHONY: tmp-certs
163+
tmp-certs:
164+
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.yaml
165+
kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-webhook --timeout=60s
166+
kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-cainjector --timeout=60s
167+
kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager --timeout=60s
168+
kubectl apply -f issuer.yaml
169+
kubectl apply -f certificate.yaml
161170

162171
.PHONY: clean
163172
clean: #HELP Clean up build artifacts
@@ -233,14 +242,19 @@ deploy: $(KIND) $(HELM) #HELP Deploy OLM to kind cluster $KIND_CLUSTER_NAME (def
233242
--set debug=true \
234243
--set olm.image.ref=$(OLM_IMAGE) \
235244
--set olm.image.pullPolicy=IfNotPresent \
245+
--set olm.tlsSecret=olm-cert \
246+
--set olm.clientCASecret=olm-cert \
247+
--set olm.service.internalPort=8443 \
236248
--set catalog.image.ref=$(OLM_IMAGE) \
237249
--set catalog.image.pullPolicy=IfNotPresent \
250+
--set catalog.tlsSecret=olm-cert \
251+
--set catalog.clientCASecret=olm-cert \
252+
--set catalog.service.internalPort=8443 \
238253
--set catalog.commandArgs=--configmapServerImage=$(CONFIGMAP_SERVER_IMAGE) \
239254
--set catalog.opmImageArgs=--opmImage=$(OPERATOR_REGISTRY_IMAGE) \
240255
--set package.image.ref=$(OLM_IMAGE) \
241256
--set package.image.pullPolicy=IfNotPresent \
242-
$(HELM_INSTALL_OPTS) \
243-
--wait;
257+
$(HELM_INSTALL_OPTS);
244258

245259
.PHONY: undeploy
246260
undeploy: $(KIND) $(HELM) #HELP Uninstall OLM from kind cluster $KIND_CLUSTER_NAME (default: kind-olmv0)

certificate.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Certificate
3+
metadata:
4+
name: olm
5+
namespace: operator-lifecycle-manager
6+
spec:
7+
secretName: olm-cert
8+
isCA: false
9+
usages:
10+
- server auth
11+
dnsNames:
12+
- localhost
13+
- catalog-operator.operator-lifecycle-manager.svc
14+
- olm-operator.operator-lifecycle-manager.svc
15+
issuerRef:
16+
name: issuer
17+
# We can reference ClusterIssuers by changing the kind here.
18+
# The default value is Issuer (i.e. a locally namespaced Issuer)
19+
kind: Issuer
20+
# This is optional since cert-manager will default to this value however
21+
# if you are using an external issuer, change this to that issuer group.
22+
group: cert-manager.io

cmd/catalog/main.go

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,16 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error {
5757
o.catalogNamespace = catalogNamespaceEnvVarValue
5858
}
5959

60+
// create a config client for operator status
61+
config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig)
62+
if err != nil {
63+
return fmt.Errorf("error configuring client: %s", err.Error())
64+
}
65+
6066
listenAndServe, err := server.GetListenAndServeFunc(
6167
server.WithLogger(logger),
6268
server.WithTLS(&o.tlsCertPath, &o.tlsKeyPath, &o.clientCAPath),
69+
server.WithKubeConfig(config),
6370
server.WithDebug(o.debug),
6471
)
6572
if err != nil {
@@ -72,11 +79,6 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error {
7279
}
7380
}()
7481

75-
// create a config client for operator status
76-
config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig)
77-
if err != nil {
78-
return fmt.Errorf("error configuring client: %s", err.Error())
79-
}
8082
configClient, err := configv1client.NewForConfig(config)
8183
if err != nil {
8284
return fmt.Errorf("error configuring client: %s", err.Error())

cmd/olm/main.go

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,18 @@ func main() {
123123
}
124124
logger.Infof("log level %s", logger.Level)
125125

126-
listenAndServe, err := server.GetListenAndServeFunc(server.WithLogger(logger), server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath), server.WithDebug(*debug))
126+
mgr, err := Manager(ctx, *debug)
127+
if err != nil {
128+
logger.WithError(err).Fatal("error configuring controller manager")
129+
}
130+
config := mgr.GetConfig()
131+
132+
listenAndServe, err := server.GetListenAndServeFunc(
133+
server.WithLogger(logger),
134+
server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath),
135+
server.WithKubeConfig(config),
136+
server.WithDebug(*debug),
137+
)
127138
if err != nil {
128139
logger.Fatalf("Error setting up health/metric/pprof service: %v", err)
129140
}
@@ -134,12 +145,6 @@ func main() {
134145
}
135146
}()
136147

137-
mgr, err := Manager(ctx, *debug)
138-
if err != nil {
139-
logger.WithError(err).Fatal("error configuring controller manager")
140-
}
141-
config := mgr.GetConfig()
142-
143148
// create a config that validates we're creating objects with labels
144149
validatingConfig := validatingroundtripper.Wrap(config, mgr.GetScheme())
145150

issuer.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
apiVersion: cert-manager.io/v1
3+
kind: Issuer
4+
metadata:
5+
name: issuer
6+
namespace: operator-lifecycle-manager
7+
spec:
8+
selfSigned: {}

pkg/lib/server/server.go

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,15 @@ import (
66
"fmt"
77
"net/http"
88
"path/filepath"
9+
"time"
910

1011
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/filemonitor"
1112
"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile"
1213
"github.com/prometheus/client_golang/prometheus/promhttp"
1314
"github.com/sirupsen/logrus"
15+
"k8s.io/client-go/rest"
16+
"sigs.k8s.io/controller-runtime/pkg/log"
17+
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
1418
)
1519

1620
// Option applies a configuration option to the given config.
@@ -43,11 +47,18 @@ func WithDebug(debug bool) Option {
4347
}
4448
}
4549

50+
func WithKubeConfig(config *rest.Config) Option {
51+
return func(sc *serverConfig) {
52+
sc.kubeConfig = config
53+
}
54+
}
55+
4656
type serverConfig struct {
4757
logger *logrus.Logger
4858
tlsCertPath *string
4959
tlsKeyPath *string
5060
clientCAPath *string
61+
kubeConfig *rest.Config
5162
debug bool
5263
}
5364

@@ -62,6 +73,7 @@ func defaultServerConfig() serverConfig {
6273
tlsCertPath: nil,
6374
tlsKeyPath: nil,
6475
clientCAPath: nil,
76+
kubeConfig: nil,
6577
logger: nil,
6678
debug: false,
6779
}
@@ -90,12 +102,53 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) {
90102
}
91103

92104
mux := http.NewServeMux()
93-
mux.Handle("/metrics", promhttp.Handler())
94105
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
95106
w.WriteHeader(http.StatusOK)
96107
})
97108
profile.RegisterHandlers(mux, profile.WithTLS(tlsEnabled || !sc.debug))
98109

110+
// Set up authenticated metrics endpoint if kubeConfig is provided
111+
sc.logger.Infof("DEBUG: Checking authentication setup - kubeConfig != nil: %v, tlsEnabled: %v", sc.kubeConfig != nil, tlsEnabled)
112+
if sc.kubeConfig != nil && tlsEnabled {
113+
sc.logger.Info("DEBUG: Setting up authenticated metrics endpoint")
114+
// Create authentication filter using controller-runtime
115+
sc.logger.Info("DEBUG: Creating authentication filter with controller-runtime")
116+
filter, err := filters.WithAuthenticationAndAuthorization(sc.kubeConfig, &http.Client{
117+
Timeout: 30 * time.Second,
118+
})
119+
if err != nil {
120+
sc.logger.Errorf("DEBUG: Failed to create authentication filter: %v", err)
121+
return nil, fmt.Errorf("failed to create authentication filter: %w", err)
122+
}
123+
sc.logger.Info("DEBUG: Authentication filter created successfully")
124+
// Create authenticated metrics handler
125+
sc.logger.Info("DEBUG: Wrapping metrics handler with authentication")
126+
logger := log.FromContext(context.Background())
127+
authenticatedMetricsHandler, err := filter(logger, promhttp.Handler())
128+
if err != nil {
129+
sc.logger.Errorf("DEBUG: Failed to wrap metrics handler: %v", err)
130+
return nil, fmt.Errorf("failed to wrap metrics handler with authentication: %w", err)
131+
}
132+
sc.logger.Info("DEBUG: Metrics handler wrapped successfully")
133+
// Add debugging wrapper to log authentication attempts
134+
debugAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
135+
sc.logger.Infof("DEBUG: Metrics request from %s, Auth header present: %v, User-Agent: %s",
136+
r.RemoteAddr, r.Header.Get("Authorization") != "", r.Header.Get("User-Agent"))
137+
authenticatedMetricsHandler.ServeHTTP(w, r)
138+
})
139+
mux.Handle("/metrics", debugAuthHandler)
140+
sc.logger.Info("Metrics endpoint configured with authentication and authorization")
141+
} else {
142+
// Fallback to unprotected metrics (for development/testing)
143+
sc.logger.Warnf("DEBUG: Using unprotected metrics - kubeConfig != nil: %v, tlsEnabled: %v", sc.kubeConfig != nil, tlsEnabled)
144+
mux.Handle("/metrics", promhttp.Handler())
145+
if sc.kubeConfig == nil {
146+
sc.logger.Warn("No Kubernetes config provided - metrics endpoint will be unprotected")
147+
} else if !tlsEnabled {
148+
sc.logger.Warn("TLS not enabled - metrics endpoint will be unprotected")
149+
}
150+
}
151+
99152
s := http.Server{
100153
Handler: mux,
101154
Addr: sc.getAddress(tlsEnabled),
@@ -141,6 +194,7 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) {
141194
ClientAuth: tls.VerifyClientCertIfGiven,
142195
}, nil
143196
},
197+
NextProtos: []string{"http/1.1"}, // Disable HTTP/2 for security
144198
}
145199
return func() error {
146200
return s.ListenAndServeTLS("", "")

vendor/modules.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1873,6 +1873,7 @@ sigs.k8s.io/controller-runtime/pkg/log/zap
18731873
sigs.k8s.io/controller-runtime/pkg/manager
18741874
sigs.k8s.io/controller-runtime/pkg/manager/signals
18751875
sigs.k8s.io/controller-runtime/pkg/metrics
1876+
sigs.k8s.io/controller-runtime/pkg/metrics/filters
18761877
sigs.k8s.io/controller-runtime/pkg/metrics/server
18771878
sigs.k8s.io/controller-runtime/pkg/predicate
18781879
sigs.k8s.io/controller-runtime/pkg/reconcile

vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go

Lines changed: 122 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)