Merge pull request #6117 from apricote/tilt-prometheus

✨ Tilt: Add prometheus to observability deployment

Author: k8s-ci-robot

Tiltfile

@@ -336,6 +336,10 @@ def deploy_observability():
         k8s_yaml(read_file("./.tiltbuild/yaml/grafana.observability.yaml"), allow_duplicates = True)
         k8s_resource(workload = "grafana", port_forwards = "3001:3000", extra_pod_selectors = [{"app": "grafana"}], labels = ["observability"])
 
+    if "prometheus" in settings.get("deploy_observability", []):
+        k8s_yaml(read_file("./.tiltbuild/yaml/prometheus.observability.yaml"), allow_duplicates = True)
+        k8s_resource(workload = "prometheus-server", new_name = "prometheus", port_forwards = "9090", extra_pod_selectors = [{"app": "prometheus"}], labels = ["observability"])
+
 def prepare_all():
     allow_k8s_arg = ""
     if settings.get("allowed_contexts"):

docs/book/src/developer/tilt.md

@@ -76,7 +76,7 @@ for more details.
 provider's yaml.
 
 **deploy_observability** ([string], default=[]): If set, installs on the dev cluster one of more observability
-tools. Supported values are `grafana`, `loki` and/or `promtail` (Note: the UI for `grafana` will be accessible via a link in the tilt console).
+tools. Supported values are `grafana`, `loki`, `promtail` and/or `prometheus` (Note: the UI for `grafana` and `prometheus` will be accessible via a link in the tilt console).
 Important! This feature requires the `helm` command to be available in the user's path.
 
 **debug** (Map{string: Map} default{}): A map of named configurations for the provider. The key is the name of the provider.

hack/observability/grafana/values.yaml

@@ -18,6 +18,9 @@ datasources:
     - name: Loki
       type: loki
       url: http://loki:3100
+    - name: Prometheus
+      type: prometheus
+      url: http://prometheus-server
 
 # Disable grafana test framework
 testFramework:

hack/observability/prometheus/kustomization.yaml

@@ -0,0 +1,9 @@
+resources:
+  - ../namespace.yaml
+
+helmCharts:
+  - name: prometheus
+    repo: https://prometheus-community.github.io/helm-charts
+    releaseName: prometheus
+    namespace: observability
+    valuesFile: values.yaml

hack/observability/prometheus/values.yaml

@@ -0,0 +1,36 @@
+# Configuration for prometheus chart, see https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus
+
+# These components do not make sense in a local dev environment
+# and can be disabled.
+alertmanager:
+  enabled: false
+kubeStateMetrics:
+  enabled: false
+nodeExporter:
+  enabled: false
+pushgateway:
+  enabled: false
+
+# Scrape metrics from deployed providers
+extraScrapeConfigs: |
+  - job_name: 'capi-providers'
+    metrics_path: /metrics
+    kubernetes_sd_configs:
+      - role: pod
+
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_pod_labelpresent_cluster_x_k8s_io_provider]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_pod_container_port_name]
+        action: keep
+        regex: metrics
+      - source_labels: [__meta_kubernetes_pod_label_cluster_x_k8s_io_provider]
+        action: replace
+        target_label: provider
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        target_label: namespace
+      - source_labels: [__meta_kubernetes_pod_name]
+        action: replace
+        target_label: pod

hack/observability/promtail/values.yaml

@@ -1,4 +1,4 @@
-# Configuration for grafana chart, see https://github.com/grafana/helm-charts/tree/main/charts/grafana
+# Configuration for promtail chart, see https://github.com/grafana/helm-charts/tree/main/charts/promtail
 
 config:
   # publish data to loki

hack/tools/tilt-prepare/main.go

@@ -33,6 +33,7 @@ import (
 	"github.com/spf13/pflag"
 	"golang.org/x/net/context"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
@@ -415,6 +416,11 @@ func providerTask(path, out string, debug bool) taskFunction {
 			}
 		}
 
+		if err := prepareDeploymentForObservability(prefix, objs); err != nil {
+			errCh <- err
+			return
+		}
+
 		providerObj, err := getProviderObj(prefix, objs)
 		if err != nil {
 			errCh <- err
@@ -467,20 +473,12 @@ func writeIfChanged(prefix string, path string, yaml []byte) error {
 // prepareDeploymentForDebug alter controller deployments for working nicely with delve debugger;
 // most specifically, liveness and readiness probes are dropper and leader election turned off.
 func prepareDeploymentForDebug(prefix string, objs []unstructured.Unstructured) error {
-	for i := range objs {
-		obj := objs[i]
-		if obj.GetKind() != "Deployment" {
-			continue
-		}
-
-		// Convert Unstructured into a typed object
-		d := &appsv1.Deployment{}
-		if err := scheme.Scheme.Convert(&obj, d, nil); err != nil {
-			return errors.Wrapf(err, "[%s] failed to convert Deployment to typed object", prefix)
-		}
-
-		for j := range d.Spec.Template.Spec.Containers {
-			container := d.Spec.Template.Spec.Containers[j]
+	return updateDeployment(prefix, objs, func(d *appsv1.Deployment) {
+		for j, container := range d.Spec.Template.Spec.Containers {
+			if container.Name != "manager" {
+				// as defined in clusterctl Provider Contract "Controllers & Watching namespace"
+				continue
+			}
 
 			// Drop liveness and readiness probes.
 			container.LivenessProbe = nil
@@ -498,7 +496,63 @@ func prepareDeploymentForDebug(prefix string, objs []unstructured.Unstructured)
 
 			d.Spec.Template.Spec.Containers[j] = container
 		}
+	})
+}
+
+// prepareDeploymentForObservability alters controller deployments for working
+// nicely with prometheus metrics scraping. Specifically, the metrics endpoint is set to
+// listen on all interfaces instead of only localhost, and another port is added to the
+// container to expose the metrics endpoint.
+func prepareDeploymentForObservability(prefix string, objs []unstructured.Unstructured) error {
+	return updateDeployment(prefix, objs, func(d *appsv1.Deployment) {
+		for j, container := range d.Spec.Template.Spec.Containers {
+			if container.Name != "manager" {
+				// as defined in clusterctl Provider Contract "Controllers & Watching namespace"
+				continue
+			}
+
+			args := make([]string, 0, len(container.Args))
+			for _, a := range container.Args {
+				if strings.HasPrefix(a, "--metrics-bind-addr=") {
+					args = append(args, "--metrics-bind-addr=0.0.0.0:8080")
+					continue
+				}
+				args = append(args, a)
+			}
+			container.Args = args
+
+			container.Ports = append(container.Ports, corev1.ContainerPort{
+				Name:          "metrics",
+				ContainerPort: 8080,
+				Protocol:      "TCP",
+			})
+
+			d.Spec.Template.Spec.Containers[j] = container
+		}
+	})
+}
+
+type updateDeploymentFunction func(deployment *appsv1.Deployment)
+
+// updateDeployment passes all (typed) deployments to the updateDeploymentFunction
+// to modify as needed.
+func updateDeployment(prefix string, objs []unstructured.Unstructured, f updateDeploymentFunction) error {
+	for i := range objs {
+		obj := objs[i]
+		if obj.GetKind() != "Deployment" {
+			continue
+		}
+
+		// Convert Unstructured into a typed object
+		d := &appsv1.Deployment{}
+		if err := scheme.Scheme.Convert(&obj, d, nil); err != nil {
+			return errors.Wrapf(err, "[%s] failed to convert Deployment to typed object", prefix)
+		}
+
+		// Call updater function
+		f(d)
 
+		// Convert back to Unstructured
 		if err := scheme.Scheme.Convert(d, &obj, nil); err != nil {
 			return errors.Wrapf(err, "[%s] failed to convert Deployment to unstructured", prefix)
 		}