hack/observability: add tempo

Signed-off-by: Stefan Büringer [email protected]

Author: sbueringer

Tiltfile

@@ -432,6 +432,12 @@ def deploy_observability():
             ],
         )
 
+    if "tempo" in settings.get("deploy_observability", []):
+        k8s_yaml(read_file("./.tiltbuild/yaml/tempo.observability.yaml"), allow_duplicates = True)
+
+        # Port-forward the tracing port to localhost, so we can also send traces from local.
+        k8s_resource(workload = "tempo", port_forwards = "4317:4317", extra_pod_selectors = [{"app": "tempo"}], labels = ["observability"])
+
     if "grafana" in settings.get("deploy_observability", []):
         k8s_yaml(read_file("./.tiltbuild/yaml/grafana.observability.yaml"), allow_duplicates = True)
         k8s_resource(workload = "grafana", port_forwards = "3001:3000", extra_pod_selectors = [{"app": "grafana"}], labels = ["observability"], objects = ["grafana:serviceaccount"])

docs/book/src/developer/tilt.md

@@ -200,12 +200,13 @@ Important! This feature requires the `helm` command to be available in the user'
 
 Supported values are:
 
-  * `grafana`*: To create dashboards and query `loki` as well as `prometheus`.
+  * `grafana`*: To create dashboards and query `loki`, `prometheus` and `tempo`.
   * `kube-state-metrics`: For exposing metrics for Kubernetes and CAPI resources to `prometheus`.
   * `loki`: To receive and store logs.
   * `prometheus`*: For collecting metrics from Kubernetes.
   * `promtail`: For providing pod logs to `loki`.
   * `parca`*: For visualizing profiling data.
+  * `tempo`: To store traces.
   * `visualizer`*: Visualize Cluster API resources for each cluster, provide quick access to the specs and status of any resource.
 
 \*: Note: the UI will be accessible via a link in the tilt console

hack/observability/grafana/chart/values.yaml

@@ -13,24 +13,67 @@ grafana.ini:
     enabled: true
     org_name: Main Org.
     org_role: Admin
+  feature_toggles:
+    enable: "tempoSearch tempoBackendSearch"
 
 # Adds loki as a datasource.
 datasources:
   datasources.yaml:
     apiVersion: 1
     datasources:
+    # https://grafana.com/docs/grafana/latest/datasources/loki/
     - name: Loki
       type: loki
+      uid: loki
+      orgId: 1
       url: http://loki:3100
+      isDefault: true
+      editable: true
       # This header has to be set as we had to set an OrgID
       # in promtail to be able to push the logs to Loki.
       jsonData:
+        maxLines: 1000
+        derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: '"traceID":"(\w+)"'
+          name: TraceID
+          url: $${__value.raw}
+          urlDisplayLabel: Trace
         httpHeaderName1: 'X-Scope-OrgID'
       secureJsonData:
         httpHeaderValue1: '1'
     - name: Prometheus
       type: prometheus
+      uid: prometheus
       url: http://prometheus-server
+      editable: true
+      jsonData:
+        httpMethod: 'GET'
+        exemplarTraceIdDestinations:
+        - name: 'TraceID'
+          datasourceUid: 'tempo'
+    # https://grafana.com/docs/grafana/latest/datasources/tempo/
+    - name: Tempo
+      type: tempo
+      uid: tempo
+      url: http://tempo:3100
+      editable: true
+      jsonData:
+        tracesToLogs:
+          datasourceUid: 'loki'
+          tags: [ 'app' ]
+          spanEndTimeShift: 1h
+          spanStartTimeShift: -1h
+          filterByTraceID: true
+          filterBySpanID: false
+          lokiSearch: true
+
+        serviceMap:
+          datasourceUid: 'prometheus'
+        search:
+          hide: false
+        nodeGraph:
+          enabled: true
 
 dashboardProviders:
   dashboardproviders.yaml: