feat(prod): add grafana stack

Author: emptybutton

.github/workflows/cd.yml

@@ -30,3 +30,5 @@ jobs:
             NATS_TOKEN: ${{ secrets.NATS_TOKEN }}
 
             GEMINI_URL: ${{ secrets.GEMINI_URL }}
+
+            GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}

deploy/dev/docker-compose.yaml

@@ -54,7 +54,7 @@ services:
       POSTGRES_PASSWORD: root
     healthcheck:
       test: pg_isready -d root -U root
-      start_period: 1m
+      start_period: 2m
       start_interval: 1s
       interval: 5s
 

deploy/prod/.env.template

@@ -6,3 +6,5 @@ POSTGRES_REPLICA_PASSWORD=replica
 NATS_TOKEN=nats
 
 GEMINI_URL=https://my-gemini.vercel.app
+
+GRAFANA_ADMIN_PASSWORD=grafana

deploy/prod/alloy/config.alloy

@@ -0,0 +1,47 @@
+// This component is responsible for disovering new containers within the docker environment
+discovery.docker "getting_started" {
+    host = "unix:///var/run/docker.sock"
+    refresh_interval = "5s"
+}
+
+// This component is responsible for relabeling the discovered containers
+discovery.relabel "getting_started" {
+    targets = []
+
+    rule {
+        source_labels = ["__meta_docker_container_name"]
+        regex         = "/(.*)"
+        target_label  = "container"
+    }
+}
+
+// This component is responsible for collecting logs from the discovered containers
+loki.source.docker "getting_started" {
+    host             = "unix:///var/run/docker.sock"
+    targets          = discovery.docker.getting_started.targets
+    forward_to       = [loki.process.getting_started.receiver]
+    relabel_rules    = discovery.relabel.getting_started.rules
+    refresh_interval = "5s"
+}
+
+// This component is responsible for processing the logs (In this case adding static labels)
+loki.process "getting_started" {
+    stage.static_labels {
+    values = {
+      env = "production",
+    }
+}
+    forward_to = [loki.write.getting_started.receiver]
+}
+
+// This component is responsible for writing the logs to Loki
+loki.write "getting_started" {
+    endpoint {
+        url  = "http://loki:3100/loki/api/v1/push"
+    }
+}
+
+// Enables the ability to view logs in the Alloy UI in realtime
+livedebugging {
+  enabled = true
+}

deploy/prod/docker-compose.yaml

@@ -2,6 +2,7 @@ name: ttt
 
 services:
   ttt:
+    image: ttt:prod
     build:
       context: ../../
       dockerfile: deploy/prod/ttt/Dockerfile
@@ -86,7 +87,7 @@ services:
     mem_limit: 30mb
     healthcheck:
       test: pg_isready -d ttt -U ttt
-      start_period: 1m
+      start_period: 2m
       start_interval: 1s
       interval: 5s
 
@@ -116,7 +117,7 @@ services:
     mem_limit: 30mb
     healthcheck:
       test: pg_isready -d ttt -U ttt
-      start_period: 1m
+      start_period: 2m
       start_interval: 1s
       interval: 5s
 
@@ -154,6 +155,63 @@ services:
     entrypoint: [""]
     command: ["bash", "/mnt/add_streams.sh"]
 
+  alloy:
+    image: grafana/alloy:v1.10.0
+    container_name: ttt-alloy
+    # ports:
+    #   - 12345:12345
+    #   - 4317:4317
+    #   - 4318:4318
+    volumes:
+      - ./alloy/config.alloy:/etc/alloy/config.alloy
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - loki
+    command: run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
+    depends_on:
+      - loki
+
+  loki:
+    image: grafana/loki:3.4.5
+    container_name: ttt-loki
+    environment:
+      TZ: ${SYSTEM_TIMEZONE:-Europe/Moscow}
+    # ports:
+    #   - 3100:3100
+    volumes:
+      - loki-data:/loki
+      - ./loki/loki-config.yaml:/etc/loki/local-config.yaml
+    networks:
+      - loki
+    command: -config.file=/etc/loki/local-config.yaml
+
+  grafana:
+    image: grafana/grafana:12.0.2-ubuntu
+    container_name: ttt-grafana
+    environment:
+      - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
+      - GF_FEATURE_TOGGLES_ENABLE=grafanaManagedRecordingRules
+
+      - GF_AUTH_ANONYMOUS_ENABLED=false
+      - GF_AUTH_BASIC_ENABLED=true
+
+      - GF_SECURITY_FORCE_PASSWORD_CHANGE=true
+      - GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION=false
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD}
+      - GF_SECURITY_MIN_PASSWORD_LENGTH=4
+
+      - GF_USERS_ALLOW_SIGN_UP=false
+
+      - TZ=${SYSTEM_TIMEZONE:-Europe/Moscow}
+    ports:
+      - 3000:3000
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
+    networks:
+      - loki
+
 volumes:
   postgres-replica1-data:
     name: "ttt-prod-postgres-replica1-data"
@@ -163,11 +221,16 @@ volumes:
     name: "ttt-prod-redis-data"
   nats-data:
     name: "ttt-prod-nats-data"
+  loki-data:
+    name: "ttt-prod-loki-data"
+  grafana-data:
+    name: "ttt-prod-grafana-data"
 
 networks:
   postgres: null
   redis: null
   nats: null
+  loki: null
 
 secrets:
   secrets:

deploy/prod/grafana/provisioning/datasources/loki.yaml

@@ -0,0 +1,11 @@
+apiVersion: 1
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    orgId: 1 
+    url: http://loki:3100
+    basicAuth: false
+    isDefault: true
+    version: 1
+    editable: true

deploy/prod/loki/loki-config.yaml

@@ -0,0 +1,61 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+  grpc_server_max_concurrent_streams: 1000
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+limits_config:
+  metric_aggregation_enabled: true
+  allow_structured_metadata: true
+  volume_enabled: true
+  retention_period: 24h   # 24h
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: true
+  metric_aggregation:
+    loki_address: localhost:3100
+
+ruler:
+  enable_alertmanager_discovery: true
+  enable_api: true
+
+
+frontend:
+  encoding: protobuf
+
+
+compactor:
+  working_directory: /tmp/loki/retention
+  delete_request_store: filesystem
+  retention_enabled: true