Add otel-collector, connect to Loki, Tempo, Prometheus to receive Beholder metrics (#1737)

Beholder metrics, otel-collector, debug tests

Author: skudasov

framework/.changeset/v0.6.3.md

@@ -0,0 +1,3 @@
+- Add otel-collector and connect it with Loki, Tempo and Prometheus
+- Tune Tempo, verify we can store traces for hours
+- Add basic tests to debug metrics, logs and traces endpoints

framework/cmd/observability/compose/conf/prometheus.yml

@@ -2,6 +2,10 @@ global:
   scrape_interval: 10s
 
 scrape_configs:
+  - job_name: 'otel-collector'
+    scrape_interval: 10s
+    static_configs:
+      - targets: [ 'otel-collector:8889' ]
   - job_name: 'ctf'
     metrics_path: /metrics
     docker_sd_configs:

framework/cmd/observability/compose/conf/provisioning/datasources/loki.yaml

@@ -1,6 +1,11 @@
 apiVersion: 1
 
 datasources:
+  - name: Tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200
+    isDefault: false
   - name: Loki
     type: loki
     isDefault: true

framework/cmd/observability/compose/docker-compose.yaml

@@ -1,4 +1,30 @@
 services:
+  tempo:
+    image: grafana/tempo:2.3.1
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml
+      - tempo_data:/tmp/tempo
+    ports:
+      - "3200:3200"   # tempo
+#      - "5317:4317"   # otlp grpc
+#      - "5318:4318"   # otlp http
+      - "9411:9411"   # zipkin
+      - "14268:14268" # jaeger ingest
+      - "14250:14250" # jaeger grpc
+      - "55680:55680" # otlp http legacy
+  otel-collector:
+    image: otel/opentelemetry-collector:latest
+    volumes:
+      - ./otel.yaml:/etc/otel/config.yaml
+    command:
+      - '--config=/etc/otel/config.yaml'
+    ports:
+      - "4317:4317" # grpc
+      - "4318:4318" # http
+      - "8889:8889"  # Prometheus scrape target
+    depends_on:
+      - loki
   cadvisor:
     image: gcr.io/cadvisor/cadvisor:v0.51.0
     container_name: cadvisor
@@ -23,7 +49,7 @@ services:
       - '9099:9090'
 
   loki:
-    image: grafana/loki:2.5.0
+    image: grafana/loki:3.4.1
     volumes:
       - ./loki-config.yaml:/etc/loki/mounted-config.yaml
       - ./conf/provisioning/rules/rules.yml:/etc/loki/rules/fake/rules.yml
@@ -47,6 +73,10 @@ services:
       - ./conf/provisioning/dashboards/cadvisor/cadvisor.json:/var/lib/grafana/dashboards/cadvisor/cadvisor.json
     ports:
       - '3000:3000'
+    depends_on:
+      - prometheus
+      - loki
+      - tempo
 
   pyroscope:
     image: 'pyroscope/pyroscope:latest'
@@ -106,3 +136,4 @@ volumes:
   grafana_home:
   grafana_logs:
   grafana_plugins:
+  tempo_data:

framework/cmd/observability/compose/loki-config.yaml

@@ -1,47 +1,40 @@
+
+# This is a complete configuration to deploy Loki backed by the filesystem.
+# The index will be shipped to the storage via tsdb-shipper.
+
 auth_enabled: false
 
+limits_config:
+  allow_structured_metadata: true
+  volume_enabled: true
+
 server:
   http_listen_port: 3100
-  grpc_listen_port: 9096
-
-limits_config:
-  unordered_writes: true
 
-ingester:
-  max_chunk_age: 1000h
 common:
-  path_prefix: /tmp/loki
-  storage:
-    filesystem:
-      chunks_directory: /tmp/loki/chunks
-      rules_directory: /tmp/loki/rules
-  replication_factor: 1
   ring:
-    instance_addr: 127.0.0.1
+    instance_addr: 0.0.0.0
     kvstore:
       store: inmemory
+  replication_factor: 1
+  path_prefix: /tmp/loki
 
 schema_config:
   configs:
-    - from: 2020-10-24
-      store: boltdb-shipper
+    - from: 2020-05-15
+      store: tsdb
       object_store: filesystem
-      schema: v11
+      schema: v13
       index:
         prefix: index_
         period: 24h
 
-ruler:
-  alertmanager_url: http://localhost:9093
-# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
-# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
-#
-# Statistics help us better understand how Loki is used, and they show us performance
-# levels for most users. This helps us prioritize features and documentation.
-# For more information on what's sent, look at
-# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go
-# Refer to the buildReport method to see what goes into a report.
-#
-# If you would like to disable reporting, uncomment the following lines:
-analytics:
-  reporting_enabled: false
+storage_config:
+  tsdb_shipper:
+    active_index_directory: /tmp/loki/index
+    cache_location: /tmp/loki/index_cache
+  filesystem:
+    directory: /tmp/loki/chunks
+
+pattern_ingester:
+  enabled: true

framework/cmd/observability/compose/otel.yaml

@@ -0,0 +1,41 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+
+exporters:
+  debug:
+    verbosity: detailed
+    sampling_initial: 5
+    sampling_thereafter: 200
+  otlphttp/logs:
+    endpoint: "http://loki:3100/otlp"
+    tls:
+      insecure: true
+  otlp:
+    endpoint: "http://tempo:4317"
+    tls:
+      insecure: true
+  prometheus:
+    endpoint: "0.0.0.0:8889"
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, otlp]
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, otlphttp/logs]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, prometheus]

framework/cmd/observability/compose/tempo.yaml

@@ -0,0 +1,25 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+        http:
+
+overrides:
+  max_traces_per_user: 50000
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /tmp/tempo/blocks
+    wal:
+      path: /tmp/tempo/wal
+
+compactor:
+  compaction:
+    max_block_bytes: 1073741824   # 1GB max block size
+    max_compaction_objects: 1000000