update observability stack

Author: ks6088ts

docs/references.md

@@ -47,10 +47,13 @@
 - [OpenTelemetry](https://opentelemetry.io/)
   - [Python / Getting Started](https://opentelemetry.io/docs/languages/python/getting-started/)
   - [Python / Cookbook](https://opentelemetry.io/docs/languages/python/cookbook/)
-  - [OpenTelemetry Collector](https://opentelemetry.io//docs/collector/)
+- [OpenTelemetry Collector](https://opentelemetry.io//docs/collector/)
+  - [OpenTelemetry Collector / Quick Start](https://opentelemetry.io/docs/collector/quick-start/)
+  - [zPages / Exposed zPages routes](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.132.0/extension/zpagesextension/README.md#exposed-zpages-routes)
 - [Jaeger](https://www.jaegertracing.io/)
   - [Jaeger / Minimal deployment example (Elasticsearch backend)](https://www.jaegertracing.io/docs/1.72/deployment/#minimal-deployment-example-elasticsearch-backend)
-- [OpenTelemetry Collector / Quick Start](https://opentelemetry.io/docs/collector/quick-start/)
+- [Prometheus](https://prometheus.io/)
+  - [Prometheus / Getting Started](https://prometheus.io/docs/prometheus/latest/getting_started/)
 
 ### n8n
 

observability.docker-compose.yml

@@ -0,0 +1,27 @@
+services:
+  prometheus:
+    image: prom/prometheus:v3.5.0
+    container_name: prometheus
+    ports:
+      - 9090:9090 # Prometheus UI
+    volumes:
+      - "./prometheus.yml:/etc/prometheus/prometheus.yml"
+      - "./assets/prometheus_data:/prometheus"
+  jaeger:
+    image: jaegertracing/all-in-one:1.72.0
+    container_name: jaeger
+    ports:
+      - 16686:16686 # Jaeger UI
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.133.0-nightly.6d46fb8
+    container_name: otel-collector
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
+    ports:
+      - 1888:1888 # pprof extension
+      - 8888:8888 # Prometheus metrics exposed by the Collector
+      - 8889:8889 # Prometheus exporter metrics
+      - 13133:13133 # health_check extension
+      - 4317:4317 # OTLP gRPC receiver
+      - 4318:4318 # OTLP http receiver
+      - 55679:55679 # zpages extension

otel-collector-config.yaml

@@ -45,24 +45,25 @@ processors:
 exporters:
   debug:
     verbosity: detailed
+  otlp/jaeger:
+    endpoint: jaeger:4317
+    tls:
+      insecure: true
+  prometheusremotewrite:
+    endpoint: prometheus:8889
 
 service:
-
   pipelines:
-
     traces:
       receivers: [otlp, jaeger, zipkin]
       processors: [batch]
-      exporters: [debug]
-
+      exporters: [debug, otlp/jaeger]
     metrics:
       receivers: [otlp, prometheus]
       processors: [batch]
-      exporters: [debug]
-
+      exporters: [debug, prometheusremotewrite]
     logs:
       receivers: [otlp]
       processors: [batch]
       exporters: [debug]
-
   extensions: [health_check, pprof, zpages]

prometheus.yml

@@ -0,0 +1,8 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['localhost:9090']

tools.docker-compose.yml

@@ -31,33 +31,3 @@ services:
       POSTGRES_PASSWORD: password
       POSTGRES_DB: db
     restart: always
-  jaeger:
-    image: jaegertracing/all-in-one:1.72.0
-    container_name: jaeger
-    environment:
-      - SPAN_STORAGE_TYPE=elasticsearch
-      - ES_SERVER_URLS=http://elasticsearch:9200
-      - LOG_LEVEL=debug
-    ports:
-      - "16686:16686" # to access the Jaeger UI
-      - "4317:4317"
-      - "4318:4318"
-      - "5778:5778"
-      - "9411:9411"
-    depends_on:
-      - elasticsearch
-    restart: unless-stopped
-  # FIXME: disable for now since its port conflicts with jaeger
-  # otel-collector:
-  #   image: otel/opentelemetry-collector-contrib:latest
-  #   container_name: otel-collector
-  #   volumes:
-  #     - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
-  #   ports:
-  #     - 1888:1888 # pprof extension
-  #     - 8888:8888 # Prometheus metrics exposed by the Collector
-  #     - 8889:8889 # Prometheus exporter metrics
-  #     - 13133:13133 # health_check extension
-  #     - 4317:4317 # OTLP gRPC receiver
-  #     - 4318:4318 # OTLP http receiver
-  #     - 55679:55679 # zpages extension