Integrate kube-state-metrics and CR config into tilt.

Author: chrischdi

Makefile

@@ -59,6 +59,7 @@ TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR))
 E2E_FRAMEWORK_DIR := $(TEST_DIR)/framework
 CAPD_DIR := $(TEST_DIR)/infrastructure/docker
 GO_INSTALL := ./scripts/go_install.sh
+OBSERVABILITY_DIR := hack/observability
 
 export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH)
 
@@ -209,7 +210,7 @@ ALL_GENERATE_MODULES = core kubeadm-bootstrap kubeadm-control-plane capd
 
 .PHONY: generate
 generate: ## Run all generate-manifests-*, generate-go-deepcopy-*, generate-go-conversions-* and generate-go-openapi targets
-	$(MAKE) generate-modules generate-manifests generate-go-deepcopy generate-go-conversions generate-go-openapi
+	$(MAKE) generate-modules generate-manifests generate-go-deepcopy generate-go-conversions generate-go-openapi generate-metrics-config
 
 .PHONY: generate-manifests
 generate-manifests: $(addprefix generate-manifests-,$(ALL_GENERATE_MODULES)) ## Run all generate-manifests-* targets
@@ -434,6 +435,20 @@ generate-modules: ## Run go mod tidy to ensure modules are up to date
 	cd $(TOOLS_DIR); go mod tidy
 	cd $(TEST_DIR); go mod tidy
 
+.PHONY: generate-metrics-config
+generate-metrics-config: $(ENVSUBST_BIN) ## Generate ./hack/observability/kube-state-metrics/crd-config.yaml
+	OUTPUT_FILE="${OBSERVABILITY_DIR}/kube-state-metrics/crd-config.yaml"; \
+	METRICS_DIR="${OBSERVABILITY_DIR}/kube-state-metrics/metrics"; \
+	echo "# This file was auto-generated via: make generate-metrics-config" > "$${OUTPUT_FILE}"; \
+	cat "$${METRICS_DIR}/header.yaml" >> "$${OUTPUT_FILE}"; \
+	for resource in cluster kubeadmcontrolplane machine machinedeployment machinehealthcheck machineset; do \
+		cat "$${METRICS_DIR}/$${resource}.yaml"; \
+		RESOURCE="$${resource}" ${ENVSUBST_BIN} < "$${METRICS_DIR}/common_metrics.yaml"; \
+		if [[ "$${resource}" != "cluster" ]]; then \
+			cat "$${METRICS_DIR}/owner_metric.yaml"; \
+		fi \
+	done >> "$${OUTPUT_FILE}"; \
+
 .PHONY: generate-diagrams
 generate-diagrams: ## Generate diagrams for *.plantuml files
 	$(MAKE) -C docs diagrams

Tiltfile

@@ -352,6 +352,10 @@ def deploy_observability():
         k8s_yaml(read_file("./.tiltbuild/yaml/prometheus.observability.yaml"), allow_duplicates = True)
         k8s_resource(workload = "prometheus-server", new_name = "prometheus", port_forwards = "9090", extra_pod_selectors = [{"app": "prometheus"}], labels = ["observability"])
 
+    if "kube-state-metrics" in settings.get("deploy_observability", []):
+        k8s_yaml(read_file("./.tiltbuild/yaml/kube-state-metrics.observability.yaml"), allow_duplicates = True)
+        k8s_resource(workload = "kube-state-metrics", new_name = "kube-state-metrics", extra_pod_selectors = [{"app": "kube-state-metrics"}], labels = ["observability"])
+
     if "visualizer" in settings.get("deploy_observability", []):
         k8s_yaml(read_file("./.tiltbuild/yaml/visualizer.observability.yaml"), allow_duplicates = True)
         k8s_resource(

docs/book/src/developer/tilt.md

@@ -173,9 +173,20 @@ kustomize_substitutions:
 {{#/tabs }}
 
 **deploy_observability** ([string], default=[]): If set, installs on the dev cluster one of more observability
-tools. Supported values are `grafana`, `loki`, `visualizer`, `promtail` and/or `prometheus` (Note: the UI for `grafana`, `prometheus`, and `visualizer` will be accessible via a link in the tilt console).
+tools. 
 Important! This feature requires the `helm` command to be available in the user's path.
 
+Supported values are:
+
+  * `grafana`*: To create dashboards and query `loki` as well as `prometheus`.
+  * `kube-state-metrics`: For exposing metrics for kubernetes and CAPI resources to `prometheus`.
+  * `loki`: To receive and store logs.
+  * `prometheus`*: For collecting metrics from Kubernetes.
+  * `promtail`: For providing pod logs to `loki`.
+  * `visualizer`*: Visualize Cluster API resources for each cluster, provide quick access to the specs and status of any resource.
+
+\*: Note: the UI will be accessible via a link in the tilt console
+
 **debug** (Map{string: Map} default{}): A map of named configurations for the provider. The key is the name of the provider.
 
 Supported settings:

hack/observability/kube-state-metrics/chart/kustomization.yaml

@@ -0,0 +1,6 @@
+helmCharts:
+  - name: kube-state-metrics
+    repo: https://prometheus-community.github.io/helm-charts
+    namespace: observability
+    releaseName: kube-state-metrics
+    valuesFile: values.yaml

hack/observability/kube-state-metrics/chart/values.yaml

@@ -0,0 +1,77 @@
+image:
+  tag: v2.6.0
+  pullPolicy: IfNotPresent
+
+# Add the CR configuration from the config map.
+volumeMounts:
+ - mountPath: /etc/config
+   name: config-volume
+
+volumes:
+ - configMap:
+     name: kube-state-metrics-crd-config
+   name: config-volume
+
+extraArgs:
+- "--custom-resource-state-config-file=/etc/config/crd-config.yaml"
+
+rbac:
+  extraRules:
+  - apiGroups:
+    - cluster.x-k8s.io
+    resources:
+    - clusters
+    - machinedeployments
+    - machinesets
+    - machines
+    - machinehealthchecks
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - controlplane.cluster.x-k8s.io
+    resources:
+    - kubeadmcontrolplanes
+    verbs:
+    - get
+    - list
+    - watch
+
+collectors:
+  # CAPI CRs
+  - clusters
+  - machinedeployments
+  - machinesets
+  - machines
+  - machinehealthchecks
+  - kubeadmcontrolplanes
+  # We need to define all default collectors too, otherwise the helm chart does not include this resources in rbac
+  - certificatesigningrequests
+  - configmaps
+  - cronjobs
+  - daemonsets
+  - deployments
+  - endpoints
+  - horizontalpodautoscalers
+  - ingresses
+  - jobs
+  - limitranges
+  - mutatingwebhookconfigurations
+  - namespaces
+  - networkpolicies
+  - nodes
+  - persistentvolumeclaims
+  - persistentvolumes
+  - poddisruptionbudgets
+  - pods
+  - replicasets
+  - replicationcontrollers
+  - resourcequotas
+  - secrets
+  - services
+  - statefulsets
+  - storageclasses
+  - validatingwebhookconfigurations
+  - volumeattachments
+  # - verticalpodautoscalers # not a default resource, see also: https://github.com/kubernetes/kube-state-metrics#enabling-verticalpodautoscalers