Refactor ops-traefik

scripts/deployments/deploy_everything_locally.bash

@@ -243,9 +243,9 @@ if [ "$start_opsstack" -eq 0 ]; then
     call_make "." up-"$stack_target";
     popd
 
-    # -------------------------------- GRAYLOG -------------------------------
-    log_info "starting graylog..."
-    service_dir="${repo_basedir}"/services/graylog
+    # -------------------------------- LOGGING -------------------------------
+    log_info "starting logging..."
+    service_dir="${repo_basedir}"/services/logging
     pushd "${service_dir}"
     call_make "." up-"$stack_target"
     sleep 1

services/graylog/docker-compose.aws.yml → services/logging/docker-compose.aws.yml

@@ -1,4 +1,4 @@
-version: '3.7'
+version: '3.8'
 services:
   mongodb:
     deploy:

services/graylog/docker-compose.dalco.yml → services/logging/docker-compose.dalco.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 services:
   mongodb:
     deploy:

services/graylog/docker-compose.letsencrypt.dns.yml → services/logging/docker-compose.letsencrypt.dns.yml

@@ -1,4 +1,4 @@
-version: '3.7'
+version: '3.8'
 services:
   graylog:
     deploy:

services/graylog/docker-compose.letsencrypt.http.yml → services/logging/docker-compose.letsencrypt.http.yml

@@ -1,4 +1,4 @@
-version: '3.7'
+version: '3.8'
 services:
   graylog:
     deploy:

services/graylog/docker-compose.local.yml → services/logging/docker-compose.local.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 services:
   mongodb:
     deploy:

services/graylog/docker-compose.master.yml → services/logging/docker-compose.master.yml

@@ -1,4 +1,4 @@
-version: "3.7"
+version: "3.8"
 services:
   mongodb:
     deploy:

services/logging/docker-compose.yml.j2

@@ -0,0 +1,198 @@
+version: "3.8"
+services:
+  # MongoDB: https://hub.docker.com/_/mongo/
+  mongodb:
+    image: mongo:6.0.6
+    init: true
+    volumes:
+      # data persistency
+      - mongo_data:/data/db
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+      resources:
+        limits:
+          memory: 1.2G
+          cpus: "1"
+        reservations:
+          memory: 300M
+          cpus: "0.1"
+    networks:
+      graylog:
+        aliases:
+          - mongo # needed because of graylog configuration
+
+  # Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/docker.html
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
+    init: true
+    volumes:
+      # data persistency
+      - elasticsearch_data:/usr/share/elasticsearch/data
+    environment:
+      - http.host=0.0.0.0
+      - transport.host=localhost
+      - network.host=0.0.0.0
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+      resources:
+        limits:
+          memory: 2G
+          cpus: "2"
+        reservations:
+          memory: 1G
+          cpus: "0.1"
+    networks:
+      graylog:
+  # Graylog: https://hub.docker.com/r/graylog/graylog/
+  graylog:
+    image: graylog/graylog:6.0.5
+    init: true
+    # user: "1000:1001"
+    configs:
+    - source: graylog_config
+      target: /files/osparc-custom-content-pack-v2.json
+    volumes:
+      # Mount local configuration directory into Docker container
+      # - graylog_config:/usr/share/graylog/data/config
+      # data persistency
+      - graylog_journal:/usr/share/graylog/data/journal
+    env_file:
+      - .env
+    environment:
+      # CHANGE ME (must be at least 16 characters)!
+      - GRAYLOG_PASSWORD_SECRET=${GRAYLOG_PASSWORD_SECRET}
+      # Username: admin
+      - GRAYLOG_ROOT_PASSWORD_SHA2=${GRAYLOG_ROOT_PASSWORD_SHA2}
+      - GRAYLOG_HTTP_EXTERNAL_URI=${GRAYLOG_HTTP_EXTERNAL_URI}
+      - GRAYLOG_ELASTICSEARCH_HOSTS=http://elasticsearch:9200,
+    networks:
+      public:
+      monitoring:
+      graylog:
+        aliases:
+          - graylog
+    ports:
+    - 12201:12201/udp
+    - 12202:12202/udp
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+      resources:
+        limits:
+          cpus: "2.00"
+          memory: 5G
+        reservations:
+          cpus: "0.1"
+          memory: 1G
+      labels:
+        - traefik.enable=true
+        - traefik.docker.network=${PUBLIC_NETWORK}
+        # direct access through port
+        - traefik.http.services.graylog.loadbalancer.server.port=9000
+        - traefik.http.routers.graylog.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/graylog`)
+        - traefik.http.routers.graylog.entrypoints=https
+        - traefik.http.routers.graylog.tls=true
+        - traefik.http.middlewares.graylog_replace_regex.replacepathregex.regex=^/graylog/?(.*)$$
+        - traefik.http.middlewares.graylog_replace_regex.replacepathregex.replacement=/$${1}
+        - traefik.http.routers.graylog.middlewares=ops_whitelist_ips@swarm, ops_gzip@swarm, graylog_replace_regex
+  fluentd:
+    image: itisfoundation/fluentd:v1.16.8-1.0
+    configs:
+      - source: fluentd_config
+        target: /fluentd/etc/fluent.conf
+    environment:
+      - GRAYLOG_HOST=graylog
+      - GRAYLOG_PORT=12201
+      - LOKI_URL=http://loki:3100
+      - FLUENTD_HOSTNAME={% raw %}{{.Node.Hostname}}{% endraw %}
+    ports:
+      - "24224:24224/tcp"
+    deploy:
+      #mode: global  # Run on all nodes
+      restart_policy:
+        condition: on-failure
+      resources:
+        limits:
+          cpus: '1.0'
+          memory: 1G
+        reservations:
+          cpus: '0.5'
+          memory: 512M
+      update_config:
+        parallelism: 1
+        delay: 10s
+        order: start-first
+    networks:
+      - monitoring
+      - graylog
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://0.0.0.0:24220/api/plugins"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+
+  loki:
+    image: grafana/loki:3.5.0
+    configs:
+      - source: loki_config
+        target: /etc/loki/loki.yaml
+    command: -config.file=/etc/loki/loki.yaml
+    deploy:
+      placement:
+        constraints: []
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: 5s
+      resources:
+        limits:
+          cpus: '1.0'
+          memory: 2G
+        reservations:
+          cpus: '0.5'
+          memory: 1G
+      update_config:
+        parallelism: 1
+        delay: 10s
+        order: start-first
+    networks:
+      - monitoring
+    healthcheck:
+      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://0.0.0.0:3100/ready"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+
+
+volumes:
+  loki-data:
+  mongo_data:
+  elasticsearch_data:
+  graylog_journal:
+
+networks:
+  graylog:
+  public:
+    external: true
+    name: ${PUBLIC_NETWORK}
+  monitoring:
+    external: true
+    name: ${MONITORED_NETWORK}
+configs:
+  graylog_config:
+    name: ${STACK_NAME}_graylog_config_{{ "./data/contentpacks/osparc-custom-content-pack-v2.json" | sha256file | substring(0,10) }}
+    file: ./data/contentpacks/osparc-custom-content-pack-v2.json
+  fluentd_config:
+    name: ${STACK_NAME}_fluentd_config_{{ "./fluentd/fluent.conf" | sha256file | substring(0,10) }}
+    file: ./fluentd/fluent.conf
+  loki_config:
+    name: ${STACK_NAME}_loki_config_{{ "./loki.yaml" | sha256file | substring(0,10) }}
+    file: ./loki.yaml

services/logging/fluentd/Dockerfile

@@ -0,0 +1,26 @@
+FROM fluent/fluentd:v1.16.8-1.0
+
+USER root
+
+# Install dependencies and plugins
+RUN apk add --no-cache --update --virtual .build-deps \
+    sudo build-base ruby-dev curl \
+ && sudo gem install fluent-plugin-grafana-loki \
+ && sudo gem install fluent-plugin-gelf-best \
+ && sudo gem install fluent-plugin-prometheus \
+ && apk del .build-deps \
+ && apk add --no-cache curl jq \
+ && rm -rf /var/cache/apk/* \
+ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
+
+# Create directories with appropriate permissions
+RUN mkdir -p /fluentd/buffer /fluentd/log \
+    && chown -R fluent:fluent /fluentd/buffer /fluentd/log
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=30s --retries=3 \
+  CMD curl -s http://localhost:24220/api/plugins | jq -e '.plugins | length > 0' || exit 1
+
+USER fluent
+
+ENTRYPOINT ["fluentd", "-c", "/fluentd/etc/fluent.conf"]

services/logging/fluentd/fluent.conf

@@ -0,0 +1,105 @@
+# Monitoring
+<source>
+  @type monitor_agent
+  bind 0.0.0.0
+  port 24220
+</source>
+
+# Prometheus metrics
+<source>
+  @type prometheus
+  bind 0.0.0.0
+  port 24231
+  metrics_path metrics
+</source>
+
+#<system>
+#  log_level debug
+#</system>
+
+<source>
+  @type prometheus_output_monitor
+  interval 10
+  <labels>
+    hostname ${hostname}
+  </labels>
+</source>
+
+# Input: Receive logs from Docker containers
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+# Add additional metadata
+<filter docker.**>
+  @type record_transformer
+  <record>
+    hostname "#{Socket.gethostname}"
+    fluentd_hostname "#{ENV['FLUENTD_HOSTNAME']}"
+    tag ${tag}
+  </record>
+</filter>
+
+# Output to both Graylog (GELF) and Loki
+<match docker.**>
+  @type copy
+
+  # Output to Graylog using GELF
+  <store>
+    @type gelf
+    host graylog
+    port 12201
+    protocol udp
+    add_msec_time true
+    flush_interval 5s
+    <buffer>
+      @type file
+      path /fluentd/buffer/graylog
+      flush_thread_count 8
+      flush_interval 5s
+      retry_forever true
+      retry_max_interval 30
+      chunk_limit_size 8M
+      total_limit_size 1G
+    </buffer>
+    <secondary>
+      @type file
+      path /fluentd/log/graylog-error
+      append true
+      <format>
+        @type json
+      </format>
+    </secondary>
+  </store>
+
+  # Output to Loki
+  <store>
+    @type loki
+    url "#{ENV['LOKI_URL']}"
+    extra_labels {"job": "docker"}
+    line_format json
+    username ""
+    password ""
+    flush_interval 5s
+    <buffer>
+      @type file
+      path /fluentd/buffer/loki
+      flush_thread_count 8
+      flush_interval 5s
+      retry_forever true
+      retry_max_interval 30
+      chunk_limit_size 8M
+      total_limit_size 1G
+    </buffer>
+    <secondary>
+      @type file
+      path /fluentd/log/loki-error
+      append true
+      <format>
+        @type json
+      </format>
+    </secondary>
+  </store>
+</match>