Merge pull request #9390 from chrischdi/pr-ksm-sidecar

🌱 observability: move metrics to config and use sidecar in kube-state-metrics

Author: k8s-ci-robot

Makefile

@@ -554,16 +554,16 @@ generate-e2e-templates-main: $(KUSTOMIZE)
 	$(KUSTOMIZE) build $(INMEMORY_TEMPLATES)/main/cluster-template --load-restrictor LoadRestrictionsNone > $(INMEMORY_TEMPLATES)/main/cluster-template.yaml
 
 .PHONY: generate-metrics-config
-generate-metrics-config: $(ENVSUBST_BIN) ## Generate ./hack/observability/kube-state-metrics/crd-config.yaml
-	OUTPUT_FILE="${OBSERVABILITY_DIR}/kube-state-metrics/crd-config.yaml"; \
-	METRICS_DIR="${OBSERVABILITY_DIR}/kube-state-metrics/metrics"; \
+generate-metrics-config: $(ENVSUBST_BIN) ## Generate ./config/metrics/crd-metrics-config.yaml
+	OUTPUT_FILE="./config/metrics/crd-metrics-config.yaml"; \
+	METRIC_TEMPLATES_DIR="./config/metrics/templates"; \
 	echo "# This file was auto-generated via: make generate-metrics-config" > "$${OUTPUT_FILE}"; \
-	cat "$${METRICS_DIR}/header.yaml" >> "$${OUTPUT_FILE}"; \
+	cat "$${METRIC_TEMPLATES_DIR}/header.yaml" >> "$${OUTPUT_FILE}"; \
 	for resource in clusterclass cluster kubeadmcontrolplane kubeadmconfig machine machinedeployment machinehealthcheck machineset machinepool; do \
-		cat "$${METRICS_DIR}/$${resource}.yaml"; \
-		RESOURCE="$${resource}" ${ENVSUBST_BIN} < "$${METRICS_DIR}/common_metrics.yaml"; \
+		cat "$${METRIC_TEMPLATES_DIR}/$${resource}.yaml"; \
+		RESOURCE="$${resource}" ${ENVSUBST_BIN} < "$${METRIC_TEMPLATES_DIR}/common_metrics.yaml"; \
 		if [[ "$${resource}" != "cluster" ]]; then \
-			cat "$${METRICS_DIR}/owner_metric.yaml"; \
+			cat "$${METRIC_TEMPLATES_DIR}/owner_metric.yaml"; \
 		fi \
 	done >> "$${OUTPUT_FILE}"; \
 

Tiltfile

@@ -482,6 +482,19 @@ def deploy_observability():
             objects = ["capi-visualizer:serviceaccount"],
         )
 
+def deploy_kustomizations():
+    for name in settings.get("deploy_kustomizations", []):
+        yaml = read_file("./.tiltbuild/yaml/{}.kustomization.yaml".format(name))
+        k8s_yaml(yaml)
+        objs = decode_yaml_stream(yaml)
+        print("objects")
+        print(find_all_objects_names(objs))
+        k8s_resource(
+            new_name = name,
+            objects = find_all_objects_names(objs),
+            labels = ["kustomization"],
+        )
+
 def prepare_all():
     tools_arg = "--tools kustomize,envsubst,clusterctl "
     tilt_settings_file_arg = "--tilt-settings-file " + tilt_file
@@ -640,6 +653,8 @@ deploy_provider_crds()
 
 deploy_observability()
 
+deploy_kustomizations()
+
 enable_providers()
 
 cluster_templates()

config/metrics/crd-clusterrole.yaml

@@ -0,0 +1,38 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics-custom-resource-capi
+  labels:
+    kube-state-metrics/aggregate-to-manager: "true"
+rules:
+- apiGroups:
+  - cluster.x-k8s.io
+  resources:
+  - clusterclasses
+  - clusters
+  - machinedeployments
+  - machinepools
+  - machinesets
+  - machines
+  - machinehealthchecks
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - controlplane.cluster.x-k8s.io
+  resources:
+  - kubeadmcontrolplanes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - bootstrap.cluster.x-k8s.io
+  resources:
+  - kubeadmconfigs
+  verbs:
+  - get
+  - list
+  - watch

hack/observability/kube-state-metrics/crd-config.yaml renamed to config/metrics/crd-metrics-config.yaml

@@ -0,0 +1,13 @@
+resources:
+  - ./crd-clusterrole.yaml
+
+namespace: observability
+
+configMapGenerator:
+- name: kube-state-metrics-crd-config-capi
+  files:
+  - capi.yaml=crd-metrics-config.yaml
+  options:
+    disableNameSuffixHash: true
+    labels:
+      kube-state-metrics/custom-resource: "true"

config/metrics/kustomization.yaml

@@ -4,4 +4,4 @@
 
 The make target `generate-metrics-config` is used to generate a single file which contains the Cluster API specific custom resource configuration for kube-state-metrics.
 
-To regenerate the file `../crd-config.yaml`, execute the `make generate-metrics-config` command.
+To regenerate the file `../crd-metrics-config.yaml`, execute the `make generate-metrics-config` command.