Monitoring

README.md

@@ -10,6 +10,7 @@ This is an overview. You will find more detailed explanations for the components
 - **[Frontend](./frontend/)** A simple Svelte client application.
 - **[Tasks API](./backend/tasks/)** The API for interacting with tasks, written in Go, using the Fiber framework. Changes to tasks are published as events to Redis.
 - **[WebsocketServer](./backend/websocket-server/)** Listens for events in Redis and broadcasts them to all connected Websocket clients. The frontend connects to it by default.
+- **[Monitoring & Observability](./observability/)** The Prometheus/Grafana Stack monitors our services and let's us view some nice charts and graphs.
 
 ## Services
 
@@ -31,6 +32,7 @@ You can easily run this setup by checking out the repository and run the followi
 ```sh
 docker-compose up -d
 open http://localhost:8000
+open http://grafana.localhost:8000
 ```
 
 ```mermaid
@@ -47,6 +49,18 @@ graph LR;
         Redis
     end
 
+    subgraph Observability
+        Grafana
+        Prometheus
+        NodeExporter
+        MongodbExporter
+
+        Grafana-- :9090 --> Prometheus
+        Prometheus-- :9100 --> NodeExporter
+        Prometheus-- :9216 --> MongodbExporter
+        MongodbExporter --> MongoDB
+    end
+
     subgraph Backend
         TasksAPI-- :27017 --> MongoDB
         TasksAPI-- :6379 --> Redis
@@ -57,10 +71,13 @@ graph LR;
         Traeffik-- :5173 --> Frontend
         Traeffik-- :3000 --> TasksAPI
         Traeffik-- :8765 --> WebsocketServer
+        Traeffik-- :3000 --> Grafana
     end
 
 
-    User-- :8000 --> Traeffik
+    User-- localhost:8000 --> Traeffik
+
+    Prometheus-- /metrics --> Traeffik
 ```
 
 ### Kubernetes with plain YML files

backend/tasks/internal/store/mongo.go

@@ -11,6 +11,8 @@ import (
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
+const CollectionName = "tasks"
+
 type MongoStore struct {
 	database   *mongo.Database
 	collection mongo.Collection
@@ -30,7 +32,7 @@ func NewMongoDatabase(uri string, database string) (*mongo.Database, error) {
 }
 
 func NewMongoStore(database *mongo.Database) *MongoStore {
-	collection := database.Collection("tasks")
+	collection := database.Collection(CollectionName)
 	return &MongoStore{database: database, collection: *collection}
 }
 

compose.yaml

@@ -2,6 +2,7 @@ include:
   - apps.compose.yaml
   - db.compose.yaml
   - redis.compose.yaml
+  - monitoring.compose.yaml
 
 services:
   traefik:
@@ -12,9 +13,12 @@ services:
       - "--providers.docker.exposedbydefault=false"
       - "--entryPoints.web.address=:80"
       - "--ping.entrypoint=traefik"
+      - "--metrics.prometheus=true"
+      - "--metrics.addinternals"
     restart: always
     healthcheck:
-      test: ["CMD-SHELL", "/usr/bin/wget -O- --tries=1 --quiet 127.0.0.1:8080/ping"]
+      test:
+        ["CMD-SHELL", "/usr/bin/wget -O- --tries=1 --quiet 127.0.0.1:8080/ping"]
       start_period: 5s
       start_interval: 1s
     ports:

db.compose.yaml

@@ -1,11 +1,14 @@
 services:
   mongo:
-    image: mongo:8.0
+    image: mongodb/mongodb-community-server:latest
     restart: always
+    environment:
+      MONGO_INITDB_DATABASE: ict
     networks:
       - internal
     volumes:
       - mongo_data:/data/db
+      - ./db/mongodb:/docker-entrypoint-initdb.d
 
 volumes:
   mongo_data: {}

db/mongodb/init.js

@@ -0,0 +1,13 @@
+disableTelemetry()
+db.setProfilingLevel(2)
+
+db = db.getSiblingDB("admin");
+db.createUser({
+    user: "monitoring",
+    pwd: "monitoring",
+    roles: [
+        { role: "clusterMonitor", db: "admin" },
+        { role: "read", db: "local" },
+        { role: "read", db: "ict" }
+    ]
+});

monitoring.compose.yaml

@@ -0,0 +1,73 @@
+---
+services:
+  grafana:
+    image: grafana/grafana-oss
+    restart: unless-stopped
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.grafana.rule=Host(`grafana.localhost`)"
+      - "traefik.http.routers.grafana.entrypoints=web"
+      - "traefik.http.services.grafana.loadbalancer.server.port=3000"
+    volumes:
+      - grafana-storage:/var/lib/grafana
+      - ./observability/grafana:/etc/grafana/provisioning
+      - ./observability/grafana/dashboards:/var/lib/grafana/dashboards
+    networks:
+      - internal
+
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - "./observability/prometheus.yml:/etc/prometheus/prometheus.yml"
+      - prometheus-storage:/prometheus
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.prometheus.rule=Host(`prometheus.localhost`)"
+      - "traefik.http.routers.prometheus.entrypoints=web"
+      - "traefik.http.services.prometheus.loadbalancer.server.port=9090"
+    networks:
+      - internal
+
+  node-exporter:
+    image: quay.io/prometheus/node-exporter:latest
+    command:
+      - --path.procfs
+      - /host/proc
+      - --path.sysfs
+      - /host/sys
+      - --collector.filesystem.ignored-mount-points
+      - "^/(sys|proc|dev|host|etc)($|/)"
+    pid: host
+    restart: unless-stopped
+    volumes:
+      - "/proc:/host/proc:ro"
+      - "/sys:/host/sys:ro"
+      - "/:/rootfs:ro"
+    networks:
+      - internal
+
+  mongodb-exporter:
+    image: percona/mongodb_exporter:0.40
+    command:
+      - --mongodb.uri=mongodb://monitoring:monitoring@mongo:27017/admin
+      - --mongodb.collstats-colls=ict.tasks
+      - --mongodb.indexstats-colls=ict.tasks
+      - --collect-all
+    networks:
+      - internal
+
+  redis-exporter:
+    image: oliver006/redis_exporter
+    command:
+      - --redis.addr=redis://redis
+    networks:
+      - internal
+    ports:
+      - 9121:9121
+
+volumes:
+  grafana-storage:
+  prometheus-storage:

observability/README.md

@@ -0,0 +1,17 @@
+# Monitoring & Observability
+
+We are running a Grafana/Prometheus stack to get some observability into our setup. This stack also allows us to
+monitor our services.
+
+See the [main Readme](../README.md) on how to get it all set up.
+
+* **Prometheus**: Collects metrics from different sources and keeps them for a couple of days.
+* **Grafana**: Connects to Prometheus and displays its data in nice dashboards and charts.
+* **Node Exporter**: Collects metrics from our host. That's either our local machine or the Kubernetes host, depending on the setup.
+* **MongoDB Exporter**: Collects metrics from MongoDB.
+
+## Docker Compose Setup
+
+After everything is running, visit <http://grafana.localhost:8000> and log in using `admin` as username and as password.
+
+You can also directly access Prometheus' UI at <http://prometheus.localhost:8000>.

observability/grafana/dashboards/dashboards.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+providers:
+  - name: "Dashboard Files"
+    orgId: 1
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: false
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: true