Skip to content

Commit 9f9063e

Browse files
irbekrmirbekrm
authored
cmd/k8s-operator,k8s-operator,go.mod: optionally create ServiceMonitor (tailscale#14248)
* cmd/k8s-operator,k8s-operator,go.mod: optionally create ServiceMonitor Adds a new spec.metrics.serviceMonitor field to ProxyClass. If that's set to true (and metrics are enabled), the operator will create a Prometheus ServiceMonitor for each proxy to which the ProxyClass applies. Additionally, create a metrics Service for each proxy that has metrics enabled. Updates tailscale#11292 Signed-off-by: Irbe Krumina <[email protected]>
1 parent eabb424 commit 9f9063e

21 files changed

+877
-22
lines changed

‎cmd/k8s-operator/connector.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
189189
isExitNode: cn.Spec.ExitNode,
190190
},
191191
ProxyClassName: proxyClass,
192+
proxyType: proxyTypeConnector,
192193
}
193194

194195
if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {
@@ -253,7 +254,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
253254
}
254255

255256
func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger *zap.SugaredLogger, cn *tsapi.Connector) (bool, error) {
256-
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector")); err != nil {
257+
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(cn.Name, a.tsnamespace, "connector"), proxyTypeConnector); err != nil {
257258
return false, fmt.Errorf("failed to cleanup Connector resources: %w", err)
258259
} else if !done {
259260
logger.Debugf("Connector cleanup not done yet, waiting for next reconcile")

‎cmd/k8s-operator/depaware.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
378378
k8s.io/api/storage/v1beta1 from k8s.io/client-go/applyconfigurations/storage/v1beta1+
379379
k8s.io/api/storagemigration/v1alpha1 from k8s.io/client-go/applyconfigurations/storagemigration/v1alpha1+
380380
k8s.io/apiextensions-apiserver/pkg/apis/apiextensions from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
381-
💣 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 from sigs.k8s.io/controller-runtime/pkg/webhook/conversion
381+
💣 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 from sigs.k8s.io/controller-runtime/pkg/webhook/conversion+
382382
k8s.io/apimachinery/pkg/api/equality from k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1+
383383
k8s.io/apimachinery/pkg/api/errors from k8s.io/apimachinery/pkg/util/managedfields/internal+
384384
k8s.io/apimachinery/pkg/api/meta from k8s.io/apimachinery/pkg/api/validation+

‎cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ rules:
3030
- apiGroups: ["tailscale.com"]
3131
resources: ["recorders", "recorders/status"]
3232
verbs: ["get", "list", "watch", "update"]
33+
- apiGroups: ["apiextensions.k8s.io"]
34+
resources: ["customresourcedefinitions"]
35+
verbs: ["get", "list", "watch"]
36+
resourceNames: ["servicemonitors.monitoring.coreos.com"]
3337
---
3438
apiVersion: rbac.authorization.k8s.io/v1
3539
kind: ClusterRoleBinding
@@ -65,6 +69,9 @@ rules:
6569
- apiGroups: ["rbac.authorization.k8s.io"]
6670
resources: ["roles", "rolebindings"]
6771
verbs: ["get", "create", "patch", "update", "list", "watch"]
72+
- apiGroups: ["monitoring.coreos.com"]
73+
resources: ["servicemonitors"]
74+
verbs: ["get", "list", "update", "create", "delete"]
6875
---
6976
apiVersion: rbac.authorization.k8s.io/v1
7077
kind: RoleBinding

‎cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,13 +74,34 @@ spec:
7474
description: |-
7575
Setting enable to true will make the proxy serve Tailscale metrics
7676
at <pod-ip>:9002/metrics.
77+
A metrics Service named <proxy-statefulset>-metrics will also be created in the operator's namespace and will
78+
serve the metrics at <service-ip>:9002/metrics.
7779
7880
In 1.78.x and 1.80.x, this field also serves as the default value for
7981
.spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
8082
fields will independently default to false.
8183
8284
Defaults to false.
8385
type: boolean
86+
serviceMonitor:
87+
description: |-
88+
Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics.
89+
The ServiceMonitor will select the metrics Service that gets created when metrics are enabled.
90+
The ingested metrics for each Service monitor will have labels to identify the proxy:
91+
ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup
92+
ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup)
93+
ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped)
94+
job: ts_<proxy type>_[<parent namespace>]_<parent_name>
95+
type: object
96+
required:
97+
- enable
98+
properties:
99+
enable:
100+
description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
101+
type: boolean
102+
x-kubernetes-validations:
103+
- rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable && !self.enable)'
104+
message: ServiceMonitor can only be enabled if metrics are enabled
84105
statefulSet:
85106
description: |-
86107
Configuration parameters for the proxy's StatefulSet. Tailscale

‎cmd/k8s-operator/deploy/manifests/operator.yaml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -541,16 +541,37 @@ spec:
541541
description: |-
542542
Setting enable to true will make the proxy serve Tailscale metrics
543543
at <pod-ip>:9002/metrics.
544+
A metrics Service named <proxy-statefulset>-metrics will also be created in the operator's namespace and will
545+
serve the metrics at <service-ip>:9002/metrics.
544546
545547
In 1.78.x and 1.80.x, this field also serves as the default value for
546548
.spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both
547549
fields will independently default to false.
548550
549551
Defaults to false.
550552
type: boolean
553+
serviceMonitor:
554+
description: |-
555+
Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics.
556+
The ServiceMonitor will select the metrics Service that gets created when metrics are enabled.
557+
The ingested metrics for each Service monitor will have labels to identify the proxy:
558+
ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup
559+
ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup)
560+
ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped)
561+
job: ts_<proxy type>_[<parent namespace>]_<parent_name>
562+
properties:
563+
enable:
564+
description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled.
565+
type: boolean
566+
required:
567+
- enable
568+
type: object
551569
required:
552570
- enable
553571
type: object
572+
x-kubernetes-validations:
573+
- message: ServiceMonitor can only be enabled if metrics are enabled
574+
rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable && !self.enable)'
554575
statefulSet:
555576
description: |-
556577
Configuration parameters for the proxy's StatefulSet. Tailscale
@@ -4648,6 +4669,16 @@ rules:
46484669
- list
46494670
- watch
46504671
- update
4672+
- apiGroups:
4673+
- apiextensions.k8s.io
4674+
resourceNames:
4675+
- servicemonitors.monitoring.coreos.com
4676+
resources:
4677+
- customresourcedefinitions
4678+
verbs:
4679+
- get
4680+
- list
4681+
- watch
46514682
---
46524683
apiVersion: rbac.authorization.k8s.io/v1
46534684
kind: ClusterRoleBinding
@@ -4728,6 +4759,16 @@ rules:
47284759
- update
47294760
- list
47304761
- watch
4762+
- apiGroups:
4763+
- monitoring.coreos.com
4764+
resources:
4765+
- servicemonitors
4766+
verbs:
4767+
- get
4768+
- list
4769+
- update
4770+
- create
4771+
- delete
47314772
---
47324773
apiVersion: rbac.authorization.k8s.io/v1
47334774
kind: Role

‎cmd/k8s-operator/ingress.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ func (a *IngressReconciler) maybeCleanup(ctx context.Context, logger *zap.Sugare
9090
return nil
9191
}
9292

93-
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress")); err != nil {
93+
if done, err := a.ssr.Cleanup(ctx, logger, childResourceLabels(ing.Name, ing.Namespace, "ingress"), proxyTypeIngressResource); err != nil {
9494
return fmt.Errorf("failed to cleanup: %w", err)
9595
} else if !done {
9696
logger.Debugf("cleanup not done yet, waiting for next reconcile")
@@ -268,6 +268,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
268268
Tags: tags,
269269
ChildResourceLabels: crl,
270270
ProxyClassName: proxyClass,
271+
proxyType: proxyTypeIngressResource,
271272
}
272273

273274
if val := ing.GetAnnotations()[AnnotationExperimentalForwardClusterTrafficViaL7IngresProxy]; val == "true" {

‎cmd/k8s-operator/ingress_test.go‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
appsv1 "k8s.io/api/apps/v1"
1313
corev1 "k8s.io/api/core/v1"
1414
networkingv1 "k8s.io/api/networking/v1"
15+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1516
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1617
"k8s.io/apimachinery/pkg/types"
1718
"sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -271,3 +272,124 @@ func TestTailscaleIngressWithProxyClass(t *testing.T) {
271272
opts.proxyClass = ""
272273
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
273274
}
275+
276+
func TestTailscaleIngressWithServiceMonitor(t *testing.T) {
277+
pc := &tsapi.ProxyClass{
278+
ObjectMeta: metav1.ObjectMeta{Name: "metrics", Generation: 1},
279+
Spec: tsapi.ProxyClassSpec{
280+
Metrics: &tsapi.Metrics{
281+
Enable: true,
282+
ServiceMonitor: &tsapi.ServiceMonitor{Enable: true},
283+
},
284+
},
285+
Status: tsapi.ProxyClassStatus{
286+
Conditions: []metav1.Condition{{
287+
Status: metav1.ConditionTrue,
288+
Type: string(tsapi.ProxyClassReady),
289+
ObservedGeneration: 1,
290+
}}},
291+
}
292+
crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
293+
tsIngressClass := &networkingv1.IngressClass{ObjectMeta: metav1.ObjectMeta{Name: "tailscale"}, Spec: networkingv1.IngressClassSpec{Controller: "tailscale.com/ts-ingress"}}
294+
fc := fake.NewClientBuilder().
295+
WithScheme(tsapi.GlobalScheme).
296+
WithObjects(pc, tsIngressClass).
297+
WithStatusSubresource(pc).
298+
Build()
299+
ft := &fakeTSClient{}
300+
fakeTsnetServer := &fakeTSNetServer{certDomains: []string{"foo.com"}}
301+
zl, err := zap.NewDevelopment()
302+
if err != nil {
303+
t.Fatal(err)
304+
}
305+
ingR := &IngressReconciler{
306+
Client: fc,
307+
ssr: &tailscaleSTSReconciler{
308+
Client: fc,
309+
tsClient: ft,
310+
tsnetServer: fakeTsnetServer,
311+
defaultTags: []string{"tag:k8s"},
312+
operatorNamespace: "operator-ns",
313+
proxyImage: "tailscale/tailscale",
314+
},
315+
logger: zl.Sugar(),
316+
}
317+
// 1. Enable metrics- expect metrics Service to be created
318+
ing := &networkingv1.Ingress{
319+
TypeMeta: metav1.TypeMeta{Kind: "Ingress", APIVersion: "networking.k8s.io/v1"},
320+
ObjectMeta: metav1.ObjectMeta{
321+
Name: "test",
322+
Namespace: "default",
323+
// The apiserver is supposed to set the UID, but the fake client
324+
// doesn't. So, set it explicitly because other code later depends
325+
// on it being set.
326+
UID: types.UID("1234-UID"),
327+
Labels: map[string]string{
328+
"tailscale.com/proxy-class": "metrics",
329+
},
330+
},
331+
Spec: networkingv1.IngressSpec{
332+
IngressClassName: ptr.To("tailscale"),
333+
DefaultBackend: &networkingv1.IngressBackend{
334+
Service: &networkingv1.IngressServiceBackend{
335+
Name: "test",
336+
Port: networkingv1.ServiceBackendPort{
337+
Number: 8080,
338+
},
339+
},
340+
},
341+
TLS: []networkingv1.IngressTLS{
342+
{Hosts: []string{"default-test"}},
343+
},
344+
},
345+
}
346+
mustCreate(t, fc, ing)
347+
mustCreate(t, fc, &corev1.Service{
348+
ObjectMeta: metav1.ObjectMeta{
349+
Name: "test",
350+
Namespace: "default",
351+
},
352+
Spec: corev1.ServiceSpec{
353+
ClusterIP: "1.2.3.4",
354+
Ports: []corev1.ServicePort{{
355+
Port: 8080,
356+
Name: "http"},
357+
},
358+
},
359+
})
360+
361+
expectReconciled(t, ingR, "default", "test")
362+
363+
fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
364+
opts := configOpts{
365+
stsName: shortName,
366+
secretName: fullName,
367+
namespace: "default",
368+
tailscaleNamespace: "operator-ns",
369+
parentType: "ingress",
370+
hostname: "default-test",
371+
app: kubetypes.AppIngressResource,
372+
enableMetrics: true,
373+
namespaced: true,
374+
proxyType: proxyTypeIngressResource,
375+
}
376+
serveConfig := &ipn.ServeConfig{
377+
TCP: map[uint16]*ipn.TCPPortHandler{443: {HTTPS: true}},
378+
Web: map[ipn.HostPort]*ipn.WebServerConfig{"${TS_CERT_DOMAIN}:443": {Handlers: map[string]*ipn.HTTPHandler{"/": {Proxy: "http://1.2.3.4:8080/"}}}},
379+
}
380+
opts.serveConfig = serveConfig
381+
382+
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
383+
expectEqual(t, fc, expectedHeadlessService(shortName, "ingress"), nil)
384+
expectEqual(t, fc, expectedMetricsService(opts), nil)
385+
expectEqual(t, fc, expectedSTSUserspace(t, fc, opts), removeHashAnnotation)
386+
// 2. Enable ServiceMonitor - should not error when there is no ServiceMonitor CRD in cluster
387+
mustUpdate(t, fc, "", "metrics", func(pc *tsapi.ProxyClass) {
388+
pc.Spec.Metrics.ServiceMonitor = &tsapi.ServiceMonitor{Enable: true}
389+
})
390+
expectReconciled(t, ingR, "default", "test")
391+
// 3. Create ServiceMonitor CRD and reconcile- ServiceMonitor should get created
392+
mustCreate(t, fc, crd)
393+
expectReconciled(t, ingR, "default", "test")
394+
expectEqualUnstructured(t, fc, expectedServiceMonitor(t, opts))
395+
}

0 commit comments

Comments
 (0)