Skip to content

Commit e926f00

Browse files
authored
remove deprecated kube-rbac-proxy (#294)
* remove deprecated kube-rbac-proxy * add metric to count appwrapper phase transitions per namespace * normalize config files * run Metrics tests in CI
1 parent ed8bee9 commit e926f00

23 files changed

+286
-126
lines changed

.github/workflows/CI-standalone.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,4 @@ jobs:
5959
make deploy -e GIT_BRANCH=${{ env.GIT_BRANCH }} TAG=${{ env.GIT_BRANCH }}-${{ env.TAG }} ENV=standalone
6060
6161
- name: Run E2E tests
62-
run: LABEL_FILTER="Standalone,Webhook" ./hack/run-tests-on-cluster.sh
62+
run: LABEL_FILTER="Metrics,Standalone,Webhook" ./hack/run-tests-on-cluster.sh

cmd/main.go

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,15 @@ import (
4242
ctrl "sigs.k8s.io/controller-runtime"
4343
"sigs.k8s.io/controller-runtime/pkg/client"
4444
"sigs.k8s.io/controller-runtime/pkg/log/zap"
45+
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
4546
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
4647
"sigs.k8s.io/controller-runtime/pkg/webhook"
4748
"sigs.k8s.io/yaml"
4849

4950
kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
5051

5152
workloadv1beta2 "github.com/project-codeflare/appwrapper/api/v1beta2"
53+
"github.com/project-codeflare/appwrapper/internal/metrics"
5254
"github.com/project-codeflare/appwrapper/pkg/config"
5355
"github.com/project-codeflare/appwrapper/pkg/controller"
5456
"github.com/project-codeflare/appwrapper/pkg/logger"
@@ -123,15 +125,19 @@ func main() {
123125
tlsOpts = append(tlsOpts, disableHTTP2)
124126
}
125127

128+
metrics.Register()
129+
126130
mgr, err := ctrl.NewManager(k8sConfig, ctrl.Options{
127131
Scheme: scheme,
128132
Metrics: metricsserver.Options{
129-
BindAddress: cfg.ControllerManager.Metrics.BindAddress,
130-
SecureServing: cfg.ControllerManager.Metrics.SecureServing,
131-
TLSOpts: tlsOpts,
133+
BindAddress: cfg.ControllerManager.Metrics.BindAddress,
134+
FilterProvider: filters.WithAuthenticationAndAuthorization,
135+
SecureServing: true,
136+
TLSOpts: tlsOpts,
132137
},
133138
WebhookServer: webhook.NewServer(webhook.Options{
134139
TLSOpts: tlsOpts,
140+
Port: 9443,
135141
}),
136142
HealthProbeBindAddress: cfg.ControllerManager.Health.BindAddress,
137143
LeaderElection: cfg.ControllerManager.LeaderElection,

config/default/config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@ data:
1010
health:
1111
bindAddress: ":8081"
1212
metrics:
13-
bindAddress: "127.0.0.1:8080"
13+
bindAddress: ":8443"
1414
leaderElection: true

config/default/kustomization.yaml

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,30 +8,30 @@ namespace: appwrapper-system
88
# field above.
99
namePrefix: appwrapper-
1010

11-
# Labels to add to all resources and selectors.
12-
#labels:
13-
#- includeSelectors: true
14-
# pairs:
15-
# someName: someValue
11+
labels:
12+
- pairs:
13+
app.kubernetes.io/name: appwrapper
14+
app.kubernetes.io/component: controller
15+
includeTemplates: true
16+
- pairs:
17+
control-plane: controller-manager
18+
includeSelectors: true
1619

1720
resources:
1821
- config.yaml
1922
- ../crd
2023
- ../rbac
2124
- ../manager
25+
- ../internalcert
2226
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
2327
# crd/kustomization.yaml
2428
- ../webhook
25-
- ../internalcert
2629
# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
2730
#- ../prometheus
31+
# [METRICS] Expose the controller manager metrics service.
32+
- metrics_service.yaml
2833

2934
patches:
30-
# Protect the /metrics endpoint by putting it behind auth.
31-
# If you want your controller-manager to expose the /metrics
32-
# endpoint w/o any authn/z, please comment the following line.
33-
- path: manager_auth_proxy_patch.yaml
34-
3535
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
3636
# crd/kustomization.yaml
3737
- path: manager_webhook_patch.yaml

config/default/manager_auth_proxy_patch.yaml

Lines changed: 0 additions & 34 deletions
This file was deleted.
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
labels:
5+
control-plane: controller-manager
6+
app.kubernetes.io/name: appwrapper
7+
name: controller-manager-metrics-service
8+
namespace: system
9+
spec:
10+
ports:
11+
- name: https
12+
port: 8443
13+
protocol: TCP
14+
targetPort: 8443
15+
selector:
16+
control-plane: controller-manager

config/manager/manager.yaml

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,19 @@
11
apiVersion: v1
22
kind: Namespace
33
metadata:
4-
labels:
5-
control-plane: controller-manager
64
name: system
75
---
86
apiVersion: apps/v1
97
kind: Deployment
108
metadata:
119
name: controller-manager
1210
namespace: system
13-
labels:
14-
control-plane: controller-manager
1511
spec:
16-
selector:
17-
matchLabels:
18-
control-plane: controller-manager
1912
replicas: 1
2013
template:
2114
metadata:
2215
annotations:
2316
kubectl.kubernetes.io/default-container: manager
24-
labels:
25-
control-plane: controller-manager
2617
spec:
2718
affinity:
2819
nodeAffinity:
@@ -47,6 +38,8 @@ spec:
4738
containers:
4839
- command:
4940
- /manager
41+
args:
42+
- "--zap-log-level=2"
5043
image: controller:latest
5144
name: manager
5245
securityContext:
@@ -68,10 +61,10 @@ spec:
6861
periodSeconds: 10
6962
resources:
7063
limits:
71-
cpu: 500m
64+
cpu: "2"
7265
memory: 128Mi
7366
requests:
74-
cpu: 10m
67+
cpu: 100m
7568
memory: 64Mi
7669
serviceAccountName: controller-manager
7770
terminationGracePeriodSeconds: 10

config/rbac/kustomization.yaml

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,13 @@ resources:
1212
- user_role.yaml
1313
- editor_role.yaml
1414
- viewer_role.yaml
15-
# Comment the following 4 lines if you want to disable
16-
# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
17-
# which protects your /metrics endpoint.
18-
- auth_proxy_service.yaml
19-
- auth_proxy_role.yaml
20-
- auth_proxy_role_binding.yaml
21-
- auth_proxy_client_clusterrole.yaml
15+
16+
# The following RBAC configurations are used to protect
17+
# the metrics endpoint with authn/authz. These configurations
18+
# ensure that only authorized users and service accounts
19+
# can access the metrics endpoint. Comment the following
20+
# permissions if you want to disable this protection.
21+
# More info: https://book.kubebuilder.io/reference/metrics.html
22+
- metrics_auth_role.yaml
23+
- metrics_auth_role_binding.yaml
24+
- metrics_reader_role.yaml

config/rbac/auth_proxy_role.yaml renamed to config/rbac/metrics_auth_role.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
apiVersion: rbac.authorization.k8s.io/v1
22
kind: ClusterRole
33
metadata:
4-
name: proxy-role
4+
name: metrics-auth-role
55
rules:
66
- apiGroups:
77
- authentication.k8s.io

config/rbac/auth_proxy_role_binding.yaml renamed to config/rbac/metrics_auth_role_binding.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
apiVersion: rbac.authorization.k8s.io/v1
22
kind: ClusterRoleBinding
33
metadata:
4-
name: proxy-rolebinding
4+
name: metrics-auth-rolebinding
55
roleRef:
66
apiGroup: rbac.authorization.k8s.io
77
kind: ClusterRole
8-
name: proxy-role
8+
name: metrics-auth-role
99
subjects:
1010
- kind: ServiceAccount
1111
name: controller-manager

0 commit comments

Comments
 (0)