Skip to content

Commit 0437d4c

Browse files
machine424slashpai
machine424
and
slashpai
committed
MON-3866: create separate metrics client cert for metrics server
Signed-off-by: Ayoub Mrini <[email protected]> Co-authored-by: Jayapriya Pai <[email protected]>
1 parent ec07655 commit 0437d4c

File tree

9 files changed

+110
-49
lines changed

9 files changed

+110
-49
lines changed

‎assets/cluster-monitoring-operator/metrics-server-client-certs.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: v1
2+
data: {}
3+
kind: Secret
4+
metadata:
5+
labels:
6+
app.kubernetes.io/managed-by: cluster-monitoring-operator
7+
app.kubernetes.io/part-of: openshift-monitoring
8+
name: metrics-server-client-certs
9+
namespace: openshift-monitoring
10+
type: Opaque

‎assets/metrics-server/deployment.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ spec:
4646
- --kubelet-use-node-status-port
4747
- --metric-resolution=15s
4848
- --kubelet-certificate-authority=/etc/tls/kubelet-serving-ca-bundle/ca-bundle.crt
49-
- --kubelet-client-certificate=/etc/tls/metrics-client-certs/tls.crt
50-
- --kubelet-client-key=/etc/tls/metrics-client-certs/tls.key
49+
- --kubelet-client-certificate=/etc/tls/metrics-server-client-certs/tls.crt
50+
- --kubelet-client-key=/etc/tls/metrics-server-client-certs/tls.key
5151
- --tls-cert-file=/etc/tls/private/tls.crt
5252
- --tls-private-key-file=/etc/tls/private/tls.key
5353
- --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
@@ -88,8 +88,8 @@ spec:
8888
volumeMounts:
8989
- mountPath: /etc/tls/private
9090
name: secret-metrics-server-tls
91-
- mountPath: /etc/tls/metrics-client-certs
92-
name: secret-metrics-client-certs
91+
- mountPath: /etc/tls/metrics-server-client-certs
92+
name: secret-metrics-server-client-certs
9393
- mountPath: /etc/tls/kubelet-serving-ca-bundle
9494
name: configmap-kubelet-serving-ca-bundle
9595
- mountPath: /etc/audit
@@ -104,9 +104,9 @@ spec:
104104
serviceAccountName: metrics-server
105105
terminationGracePeriodSeconds: 170
106106
volumes:
107-
- name: secret-metrics-client-certs
107+
- name: secret-metrics-server-client-certs
108108
secret:
109-
secretName: metrics-client-certs
109+
secretName: metrics-server-client-certs
110110
- name: secret-metrics-server-tls
111111
secret:
112112
secretName: metrics-server-tls

‎jsonnet/components/cluster-monitoring-operator.libsonnet‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,17 @@ function(params) {
8080
},
8181
},
8282

83+
metricsServerClientCerts: {
84+
apiVersion: 'v1',
85+
kind: 'Secret',
86+
metadata: {
87+
name: 'metrics-server-client-certs',
88+
namespace: cfg.namespace,
89+
},
90+
type: 'Opaque',
91+
data: {},
92+
},
93+
8394
metricsClientCerts: {
8495
apiVersion: 'v1',
8596
kind: 'Secret',

‎jsonnet/components/metrics-server.libsonnet‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -197,8 +197,8 @@ function(params) {
197197
'--kubelet-use-node-status-port',
198198
'--metric-resolution=15s',
199199
'--kubelet-certificate-authority=/etc/tls/kubelet-serving-ca-bundle/ca-bundle.crt',
200-
'--kubelet-client-certificate=/etc/tls/metrics-client-certs/tls.crt',
201-
'--kubelet-client-key=/etc/tls/metrics-client-certs/tls.key',
200+
'--kubelet-client-certificate=/etc/tls/metrics-server-client-certs/tls.crt',
201+
'--kubelet-client-key=/etc/tls/metrics-server-client-certs/tls.key',
202202
'--tls-cert-file=/etc/tls/private/tls.crt',
203203
'--tls-private-key-file=/etc/tls/private/tls.key',
204204
'--tls-cipher-suites=' + cfg.tlsCipherSuites,
@@ -260,8 +260,8 @@ function(params) {
260260
name: 'secret-metrics-server-tls',
261261
},
262262
{
263-
mountPath: '/etc/tls/metrics-client-certs',
264-
name: 'secret-metrics-client-certs',
263+
mountPath: '/etc/tls/metrics-server-client-certs',
264+
name: 'secret-metrics-server-client-certs',
265265
},
266266
{
267267
mountPath: '/etc/tls/kubelet-serving-ca-bundle',
@@ -278,9 +278,9 @@ function(params) {
278278
terminationGracePeriodSeconds: 170,
279279
volumes: [
280280
{
281-
name: 'secret-metrics-client-certs',
281+
name: 'secret-metrics-server-client-certs',
282282
secret: {
283-
secretName: 'metrics-client-certs',
283+
secretName: 'metrics-server-client-certs',
284284
},
285285
},
286286
{

‎pkg/manifests/manifests.go‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,7 @@ var (
242242
ClusterMonitoringGrpcTLSSecret = "cluster-monitoring-operator/grpc-tls-secret.yaml"
243243
ClusterMonitoringOperatorPrometheusRule = "cluster-monitoring-operator/prometheus-rule.yaml"
244244
ClusterMonitoringMetricsClientCertsSecret = "cluster-monitoring-operator/metrics-client-certs.yaml"
245+
ClusterMonitoringMetricsServerClientCertsSecret = "cluster-monitoring-operator/metrics-server-client-certs.yaml"
245246
ClusterMonitoringFederateClientCertsSecret = "cluster-monitoring-operator/federate-client-certs.yaml"
246247
ClusterMonitoringMetricsClientCACM = "cluster-monitoring-operator/metrics-client-ca.yaml"
247248

@@ -1889,7 +1890,7 @@ func (f *Factory) MetricsServerRoleBindingAuthReader() (*rbacv1.RoleBinding, err
18891890
return f.NewRoleBinding(f.assets.MustNewAssetSlice(MetricsServerRoleBindingAuthReader))
18901891
}
18911892

1892-
func (f *Factory) MetricsServerDeployment(apiAuthSecretName string, kubeletCABundle *v1.ConfigMap, servingCASecret, metricsClientCert *v1.Secret, requestheader map[string]string) (*appsv1.Deployment, error) {
1893+
func (f *Factory) MetricsServerDeployment(apiAuthSecretName string, kubeletCABundle *v1.ConfigMap, servingCASecret, metricsServerClientCerts *v1.Secret, requestheader map[string]string) (*appsv1.Deployment, error) {
18931894
dep, err := f.NewDeployment(f.assets.MustNewAssetSlice(MetricsServerDeployment))
18941895
if err != nil {
18951896
return nil, err
@@ -1934,10 +1935,10 @@ func (f *Factory) MetricsServerDeployment(apiAuthSecretName string, kubeletCABun
19341935
// are rotated.
19351936
dep.Spec.Template.Annotations["monitoring.openshift.io/serving-ca-secret-hash"] = hashByteMap(servingCASecret.Data)
19361937

1937-
// Hash the metrics client cert and propagate it as an annotation to the
1938-
// deployment's pods to trigger a new rollout when the metrics client cert
1938+
// Hash the metrics server client cert and propagate it as an annotation to the
1939+
// deployment's pods to trigger a new rollout when the metrics server client cert
19391940
// is rotated.
1940-
dep.Spec.Template.Annotations["monitoring.openshift.io/metrics-client-cert-hash"] = hashByteMap(metricsClientCert.Data)
1941+
dep.Spec.Template.Annotations["monitoring.openshift.io/metrics-server-client-certs-hash"] = hashByteMap(metricsServerClientCerts.Data)
19411942

19421943
config := f.config.ClusterMonitoringConfiguration.MetricsServerConfig
19431944
if config == nil {
@@ -2017,13 +2018,9 @@ func (f *Factory) MetricsServerDeployment(apiAuthSecretName string, kubeletCABun
20172018
return dep, nil
20182019
}
20192020

2020-
func (f *Factory) MetricsServerSecret(tlsSecret *v1.Secret, apiAuthConfigmap *v1.ConfigMap) (*v1.Secret, error) {
2021+
func (f *Factory) MetricsServerClientCASecret(apiAuthConfigmap *v1.ConfigMap) (*v1.Secret, error) {
20212022
data := make(map[string]string)
20222023

2023-
for k, v := range tlsSecret.Data {
2024-
data[k] = string(v)
2025-
}
2026-
20272024
for k, v := range apiAuthConfigmap.Data {
20282025
data[k] = v
20292026
}
@@ -2033,16 +2030,14 @@ func (f *Factory) MetricsServerSecret(tlsSecret *v1.Secret, apiAuthConfigmap *v1
20332030
var (
20342031
clientCA = r.value("client-ca-file")
20352032
requestheaderClientCA = r.value("requestheader-client-ca-file")
2036-
tlsCA = r.value("tls.crt")
2037-
tlsKey = r.value("tls.key")
20382033
)
20392034

20402035
if r.Error() != nil {
20412036
return nil, fmt.Errorf("value not found in extension api server authentication configmap: %w", r.err)
20422037
}
20432038

20442039
h := fnv.New64()
2045-
h.Write([]byte(clientCA + requestheaderClientCA + tlsCA + tlsKey))
2040+
h.Write([]byte(clientCA + requestheaderClientCA))
20462041
hash := strconv.FormatUint(h.Sum64(), 32)
20472042

20482043
return &v1.Secret{
@@ -2057,8 +2052,6 @@ func (f *Factory) MetricsServerSecret(tlsSecret *v1.Secret, apiAuthConfigmap *v1
20572052
Data: map[string][]byte{
20582053
"client-ca-file": []byte(clientCA),
20592054
"requestheader-client-ca-file": []byte(requestheaderClientCA),
2060-
"tls.crt": []byte(tlsCA),
2061-
"tls.key": []byte(tlsKey),
20622055
},
20632056
}, nil
20642057
}

‎pkg/manifests/manifests_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2577,7 +2577,7 @@ metricsServer:
25772577
}
25782578
metricsClientSecret := &v1.Secret{
25792579
ObjectMeta: metav1.ObjectMeta{
2580-
Name: "metrics-client-cert",
2580+
Name: "metrics-server-client-certs",
25812581
Namespace: "openshift-monitoring",
25822582
},
25832583
Data: map[string][]byte{
@@ -2637,7 +2637,7 @@ metricsServer:
26372637

26382638
podAnnotations := d.Spec.Template.Annotations
26392639
require.Equal(t, "eplue2a9srfkb", podAnnotations["monitoring.openshift.io/kubelet-serving-ca-bundle-hash"])
2640-
require.Equal(t, "arprfan3mk728", podAnnotations["monitoring.openshift.io/metrics-client-cert-hash"])
2640+
require.Equal(t, "arprfan3mk728", podAnnotations["monitoring.openshift.io/metrics-server-client-certs-hash"])
26412641
require.Equal(t, "383c7cmidrae2", podAnnotations["monitoring.openshift.io/serving-ca-secret-hash"])
26422642
}
26432643

‎pkg/manifests/tls.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,19 @@ func (f *Factory) MetricsClientCerts() (*v1.Secret, error) {
6666
return s, nil
6767
}
6868

69+
func (f *Factory) MetricsServerClientCerts() (*v1.Secret, error) {
70+
s, err := f.NewSecret(f.assets.MustNewAssetSlice(ClusterMonitoringMetricsServerClientCertsSecret))
71+
if err != nil {
72+
return nil, err
73+
}
74+
75+
s.Namespace = f.namespace
76+
s.Data = make(map[string][]byte)
77+
s.Annotations = make(map[string]string)
78+
79+
return s, nil
80+
}
81+
6982
func (f *Factory) FederateClientCerts() (*v1.Secret, error) {
7083
s, err := f.NewSecret(f.assets.MustNewAssetSlice(ClusterMonitoringFederateClientCertsSecret))
7184
if err != nil {

‎pkg/operator/operator.go‎

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,8 @@ type InfrastructureConfig struct {
6262

6363
var (
6464
// The cluster-policy-controller will automatically approve the
65-
// CertificateSigningRequest resources issued for the prometheus-k8s
66-
// service account.
65+
// CertificateSigningRequest resources issued for the prometheus and metrics-server
66+
// service accounts.
6767
// See https://github.com/openshift/cluster-policy-controller/blob/cc787e1b1e177696817b66689a03471914083a67/pkg/cmd/controller/csr.go#L21-L46.
6868
csrOption = csr.CSROption{
6969
ObjectMeta: metav1.ObjectMeta{
@@ -76,6 +76,17 @@ var (
7676
SignerName: certapiv1.KubeAPIServerClientSignerName,
7777
}
7878

79+
csrMetricsServerOption = csr.CSROption{
80+
ObjectMeta: metav1.ObjectMeta{
81+
GenerateName: "system:openshift:openshift-monitoring-",
82+
Labels: map[string]string{
83+
"metrics.openshift.io/csr.subject": "metrics-server",
84+
},
85+
},
86+
Subject: &pkix.Name{CommonName: "system:serviceaccount:openshift-monitoring:metrics-server"},
87+
SignerName: certapiv1.KubeAPIServerClientSignerName,
88+
}
89+
7990
// To identify "invalid UWM config only" failures
8091
ErrUserWorkloadInvalidConfiguration = fmt.Errorf("invalid UWM configuration")
8192
)
@@ -154,6 +165,7 @@ const (
154165
alertmanagerCABundleConfigMap = "openshift-monitoring/alertmanager-trusted-ca-bundle"
155166
grpcTLS = "openshift-monitoring/grpc-tls"
156167
metricsClientCerts = "openshift-monitoring/metrics-client-certs"
168+
metricsServerClientCerts = "openshift-monitoring/metrics-server-client-certs"
157169
federateClientCerts = "openshift-monitoring/federate-client-certs"
158170

159171
// Canonical name of the cluster-wide infrastructure resource.
@@ -527,9 +539,35 @@ func New(
527539
return nil, fmt.Errorf("failed to create federate certificate controller: %w", err)
528540
}
529541

530-
o.controllersToRunFunc = append(o.controllersToRunFunc, csrFederateController.Run, csrController.Run)
542+
csrMetricsServerController, err := csr.NewClientCertificateController(
543+
csr.ClientCertOption{
544+
SecretNamespace: "openshift-monitoring",
545+
SecretName: "metrics-server-client-certs",
546+
AdditionalAnnotations: certrotation.AdditionalAnnotations{
547+
JiraComponent: "Monitoring",
548+
},
549+
},
550+
csrMetricsServerOption,
551+
kubeInformersOperatorNS.Certificates().V1().CertificateSigningRequests(),
552+
o.client.KubernetesInterface().CertificatesV1().CertificateSigningRequests(),
553+
kubeInformersOperatorNS.Core().V1().Secrets(),
554+
o.client.KubernetesInterface().CoreV1(),
555+
o.client.EventRecorder(),
556+
"OpenShiftMonitoringMetricsServerClientCertRequester",
557+
)
558+
559+
if err != nil {
560+
return nil, fmt.Errorf("failed to create client certificate controller: %w", err)
561+
}
531562

532-
o.controllersToRunFunc = append(o.controllersToRunFunc, o.ruleController.Run, o.relabelController.Run)
563+
o.controllersToRunFunc = append(
564+
o.controllersToRunFunc,
565+
csrFederateController.Run,
566+
csrController.Run,
567+
csrMetricsServerController.Run,
568+
o.ruleController.Run,
569+
o.relabelController.Run,
570+
)
533571

534572
return o, nil
535573
}
@@ -663,6 +701,7 @@ func (o *Operator) handleEvent(obj interface{}) {
663701
case cmoConfigMap:
664702
case apiAuthenticationConfigMap:
665703
case kubeletServingCAConfigMap:
704+
case metricsServerClientCerts:
666705
case telemeterCABundleConfigMap:
667706
case alertmanagerCABundleConfigMap:
668707
case grpcTLS:

‎pkg/tasks/metricsserver.go‎

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -122,20 +122,20 @@ func (t *MetricsServerTask) Run(ctx context.Context) error {
122122
return fmt.Errorf("reconciling MetricsServer Service failed: %w", err)
123123
}
124124

125-
cacm, err := t.factory.PrometheusK8sKubeletServingCABundle(map[string]string{})
125+
kubeletServingCA, err := t.factory.PrometheusK8sKubeletServingCABundle(map[string]string{})
126126
if err != nil {
127127
return fmt.Errorf("initializing kubelet serving CA Bundle ConfigMap failed: %w", err)
128128
}
129129

130-
cacm, err = t.client.WaitForConfigMap(
130+
kubeletServingCA, err = t.client.WaitForConfigMap(
131131
ctx,
132-
cacm,
132+
kubeletServingCA,
133133
)
134134
if err != nil {
135135
return err
136136
}
137137

138-
scas, err := t.client.WaitForSecretByNsName(
138+
servingCASecret, err := t.client.WaitForSecretByNsName(
139139
ctx,
140140
types.NamespacedName{
141141
Namespace: s.Namespace,
@@ -146,17 +146,17 @@ func (t *MetricsServerTask) Run(ctx context.Context) error {
146146
return err
147147
}
148148

149-
mcs, err := t.factory.MetricsClientCerts()
149+
metricsServerClientCerts, err := t.factory.MetricsServerClientCerts()
150150
if err != nil {
151151
return fmt.Errorf("initializing metrics-client-cert failed: %w", err)
152152
}
153153

154-
mcs, err = t.client.WaitForSecret(
154+
metricsServerClientCerts, err = t.client.WaitForSecret(
155155
ctx,
156-
mcs,
156+
metricsServerClientCerts,
157157
)
158158
if err != nil {
159-
return err
159+
return fmt.Errorf("waiting for metrics-server-client-certs secret failed: %w", err)
160160
}
161161

162162
{
@@ -171,19 +171,14 @@ func (t *MetricsServerTask) Run(ctx context.Context) error {
171171
}
172172
}
173173

174-
tlsSecret, err := t.client.WaitForSecretByNsName(ctx, types.NamespacedName{Namespace: t.namespace, Name: "metrics-server-tls"})
175-
if err != nil {
176-
return fmt.Errorf("failed to wait for metrics-server-tls secret: %w", err)
177-
}
178-
179174
apiAuthConfigmap, err := t.client.WaitForConfigMapByNsName(ctx, types.NamespacedName{Namespace: "kube-system", Name: "extension-apiserver-authentication"})
180175
if err != nil {
181176
return fmt.Errorf("failed to wait for kube-system/extension-apiserver-authentication configmap: %w", err)
182177
}
183178

184-
secret, err := t.factory.MetricsServerSecret(tlsSecret, apiAuthConfigmap)
179+
secret, err := t.factory.MetricsServerClientCASecret(apiAuthConfigmap)
185180
if err != nil {
186-
return fmt.Errorf("failed to create metrics-server secret: %w", err)
181+
return fmt.Errorf("failed to create metrics-server client-ca secret: %w", err)
187182
}
188183

189184
err = t.deleteOldMetricsServerSecrets(secret.Labels["monitoring.openshift.io/hash"])
@@ -193,10 +188,10 @@ func (t *MetricsServerTask) Run(ctx context.Context) error {
193188

194189
err = t.client.CreateOrUpdateSecret(ctx, secret)
195190
if err != nil {
196-
return fmt.Errorf("reconciling MetricsServer Secret failed: %w", err)
191+
return fmt.Errorf("reconciling metrics-server client-ca secret failed: %w", err)
197192
}
198193

199-
dep, err := t.factory.MetricsServerDeployment(secret.Name, cacm, scas, mcs, apiAuthConfigmap.Data)
194+
dep, err := t.factory.MetricsServerDeployment(secret.Name, kubeletServingCA, servingCASecret, metricsServerClientCerts, apiAuthConfigmap.Data)
200195
if err != nil {
201196
return fmt.Errorf("initializing MetricsServer Deployment failed: %w", err)
202197
}

0 commit comments

Comments
 (0)