Merge pull request #8305 from systeminit/fnichol/tempo-tracing

feat(tempo): replace Jaeger with Grafana Tempo for distributed tracing

Author: fnichol

DEV_DOCS.md

@@ -215,7 +215,7 @@ These external services are deployed via [`docker compose`](https://docs.docker.
 Thus, it is worth checking if you are running these services to avoid conflicts when running SI.
 Potentially conflicting services include, but are not limited to, the following:
 
-* Jaeger
+* Tempo
 * NATS and NATS Jetstream
 * OpenTelemetry
 * PostgreSQL

README.md

@@ -87,7 +87,7 @@ We use [`buck2`](https://github.com/facebook/buck2) to run the stack, run and bu
 
 > [!WARNING]
 > Before continuing, you should stop any locally running services to avoid conflicting ports with the stack.
-> Some of the services that will run include, but are not limited to the following: PostgreSQL, NATS, Jaeger and OpenTelemetry._
+> Some of the services that will run include, but are not limited to the following: PostgreSQL, NATS, Tempo and OpenTelemetry._
 
 Check if you are ready to run the stack before continuing.
 

component/otelcol/Dockerfile

@@ -7,14 +7,16 @@ ENV SI_OTEL_COL__HONEYCOMB_API_KEY=""
 
 # We're going to base our image on a non-scratch image which should allow us to
 # `docker exec` into a running instance for debugging
+
 FROM alpine:3 AS final
+
 RUN apk add --no-cache \
     curl
-WORKDIR /app
+
 COPY --from=otelcol /otelcol-contrib /bin/otelcol
 COPY ./*config.yaml /etc/otelcol/
-COPY ./docker-entrypoint.sh /app/docker-entrypoint.sh
+COPY ./docker-entrypoint.sh /docker-entrypoint.sh
 
 HEALTHCHECK CMD curl --fail http://127.0.0.1:13133/ || exit 1
 
-ENTRYPOINT ["/app/docker-entrypoint.sh"]
+ENTRYPOINT ["/docker-entrypoint.sh"]

component/otelcol/config.yaml

@@ -1,3 +1,4 @@
+---
 receivers:
   otlp:
     protocols:
@@ -203,10 +204,12 @@ processors:
 exporters:
   debug:
     verbosity: normal
-  otlp:
-    endpoint: jaeger:4317
+  otlp/tempo:
+    endpoint: tempo:4317
     tls:
       insecure: true
+  otlphttp/loki:
+    endpoint: http://loki:3100/otlp
   prometheus:
     endpoint: otelcol:9090
 
@@ -219,15 +222,15 @@ service:
     logs:
       receivers: [otlp]
       processors: [batch]
-      exporters: [debug, otlp]
+      exporters: [debug, otlphttp/loki]
     traces:
       receivers: [otlp]
       processors:
         - memory_limiter
         - transform/spanevent
         - filter/spanevent
         - batch
-      exporters: [debug, otlp]
+      exporters: [debug, otlp/tempo]
     metrics:
       receivers: [otlp]
       processors: [batch]

component/otelcol/docker-entrypoint.sh

@@ -1,5 +1,10 @@
-#!/bin/sh
+#!/usr/bin/env sh
+set -eu
 
-sed -i "s/SI_OTEL_COL__HONEYCOMB_API_KEY/$SI_OTEL_COL__HONEYCOMB_API_KEY/g" ${SI_OTEL_COL__CONFIG_PATH-/etc/otelcol/config.yaml}
+config="${SI_OTEL_COL__CONFIG_PATH-/etc/otelcol/config.yaml}"
 
-/bin/otelcol --config ${SI_OTEL_COL__CONFIG_PATH-/etc/otelcol/config.yaml}
+sed \
+  -i "s/SI_OTEL_COL__HONEYCOMB_API_KEY/${SI_OTEL_COL__HONEYCOMB_API_KEY:-}/g" \
+  "$config"
+
+exec /bin/otelcol --config "$config"

component/otelcol/honeycomb-config.yaml

@@ -1,3 +1,4 @@
+---
 receivers:
   otlp:
     protocols:

component/tempo/BUCK

@@ -0,0 +1,19 @@
+load(
+    "@prelude-si//:macros.bzl",
+    "container_image",
+)
+
+container_image(
+    name = "tempo",
+    srcs = {
+        "tempo.yaml": ".",
+    },
+    build_args = {
+        "BASE_VERSION": "2.9.0",
+    },
+    run_docker_args = [
+        "--publish",
+        "3200:3200",
+    ],
+    default_target = True,
+)

component/tempo/Dockerfile

@@ -0,0 +1,3 @@
+ARG BASE_VERSION
+FROM grafana/tempo:${BASE_VERSION}
+COPY ./tempo.yaml /etc/tempo.yaml

component/tempo/tempo.yaml

@@ -0,0 +1,48 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+
+ingester:
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: 336h
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /tmp/tempo/wal
+    local:
+      path: /tmp/tempo/blocks
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: dev
+  storage:
+    path: /tmp/tempo/generator/wal
+  traces_storage:
+    path: /tmp/tempo/generator/traces
+  processor:
+    service_graphs:
+      dimensions: []
+    span_metrics:
+      dimensions: []

dev/Tiltfile

@@ -40,7 +40,7 @@ groups = {
     ],
     "telemetry": [
         "grafana",
-        "jaeger",
+        "tempo",
         "loki",
         "node_exporter",
         "prometheus",
@@ -165,11 +165,6 @@ compose_services = {
             link("http://localhost:3000", "grafana-ui"),
         ],
     },
-    "jaeger": {
-        "links": [
-            link("http://localhost:16686", "jaeger-ui"),
-        ],
-    },
     "localstack": {
         "links": [
             link("http://localhost:4566", "localstack-api"),
@@ -184,6 +179,11 @@ compose_services = {
     "prometheus": {},
     "promtail": {},
     "spicedb": {},
+    "tempo": {
+        "links": [
+            link("http://localhost:3200", "tempo-ui"),
+        ],
+    },
     "versitygw": {},
 }
 for service, kwargs in compose_services.items():