Add loki logpush via promtail

This aims to recreate the functionality from
shutter-network/shutter-keyper-deployment#10
usable with the DAppnodePackage.

Because we can not install the docker loki plugin, and everything is running
in one container through supervisord, we need to resort to promtail and
some extra log redirections to keep the docker logging working.

Author: konradkonrad

setup-wizard.yml

@@ -31,6 +31,18 @@ fields:
       service: shutter
     required: true
 
+  - id: enable_push_logs
+    title: Enable Push Logs
+    description: |
+      Enable the push logs feature to send logs to an external server controlled by Shutter.
+    target:
+      type: environment
+      name: SHUTTER_PUSH_LOGS_ENABLED
+      service: [shutter, metrics]
+    enum:
+      - "true"
+      - "false"
+
   - id: enable_push_metrics
     title: Enable Push Metrics
     description: |
@@ -46,7 +58,7 @@ fields:
   - id: pushgateway_url
     title: Pushgateway URL
     description: |
-      The URL of the Pushgateway server to send metrics to.
+      The URL of the Pushgateway server to send metrics/logs to.
     target:
       type: environment
       name: PUSHGATEWAY_URL

shutter/Dockerfile

@@ -1,7 +1,7 @@
 ARG ASSETS_VERSION
 ARG UPSTREAM_VERSION
 
-FROM ghcr.io/shutter-network/assets:${ASSETS_VERSION} as assets
+FROM ghcr.io/shutter-network/assets:${ASSETS_VERSION} AS assets
 
 RUN rsync -aq --delete /assets-source/ /assets/
 
@@ -40,6 +40,7 @@ ADD ${STAKER_SCRIPTS_URL}/dvt_lsd_tools.sh /etc/profile.d/
 
 COPY go-shutter-settings ${SHUTTER_SETTINGS_SRC_DIR}
 COPY supervisord.conf /etc/supervisord.conf
+COPY promtail_config.yaml /etc/promtail_config.yaml
 
 RUN go build -C ${SHUTTER_SETTINGS_SRC_DIR} -o /usr/local/bin/go_shutter_settings
 
@@ -49,9 +50,18 @@ RUN mkdir -p ${KEYPER_CONFIG_DIR} ${SHUTTER_CHAIN_DIR} ${ASSETS_DIR} /opt/superv
 COPY scripts /usr/local/bin/
 COPY --from=assets ${ASSETS_DIR}/ ${ASSETS_DIR}/
 
+# For pushing logs to loki
+RUN apt-get -y install wget gpg
+RUN mkdir -p /etc/apt/keyrings/
+RUN wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor > /etc/apt/keyrings/grafana.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | tee /etc/apt/sources.list.d/grafana.list
+# promtail & rotatelogs (from apache2)
+RUN apt-get update && apt-get -y install promtail apache2
+
+
 # Placed here to rebuild less layers
 ENV CHAIN_PORT=${CHAIN_PORT} \
     SHUTTER_P2P_LISTENADDRESSES="/ip4/0.0.0.0/tcp/${KEYPER_PORT},/ip4/0.0.0.0/udp/${KEYPER_PORT}/quic-v1" \
     NETWORK=${NETWORK}
 
-ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"]
+ENTRYPOINT ["supervisord", "-c", "/etc/supervisord.conf"]

shutter/promtail_config.yaml

@@ -0,0 +1,46 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+client:
+  url: https://logs.metrics.shutter.network/insert/loki/api/v1/push
+  basic_auth:
+    username: $(PUSHGATEWAY_USERNAME)
+    password: $(PUSHGATEWAY_PASSWORD)
+
+scrape_configs:
+ - job_name: configure
+   pipeline_stages:
+   - docker:
+   static_configs:
+   - targets:
+      - localhost
+     labels:
+      job: configure
+      host: ${HOSTNAME}
+      __path__: /tmp/configure.log
+
+ - job_name: keyper
+   pipeline_stages:
+   - docker:
+   static_configs:
+   - targets:
+      - localhost
+     labels:
+      job: keyper
+      host: ${HOSTNAME}
+      __path__: /tmp/keyper.log
+
+ - job_name: chain
+   pipeline_stages:
+   - docker:
+   static_configs:
+   - targets:
+      - localhost
+     labels:
+      job: keyper
+      host: ${HOSTNAME}
+      __path__: /tmp/chain.log

shutter/scripts/run_chain.sh

@@ -3,8 +3,12 @@
 run_chain() {
 
     echo "[INFO | chain] Starting chain..."
-
-    $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE"
+    if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]];
+    then
+        $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE" | rotatelogs -n 1 -e -c /tmp/chain.log 5M
+    else
+        $SHUTTER_BIN chain --config "$SHUTTER_CHAIN_CONFIG_FILE"
+    fi
 }
 
 run_chain

shutter/scripts/run_configure.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+
+if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]];
+then
+    ./configure.sh | rotatelogs -n 1 -e -c /tmp/configure.log 5M
+else
+    ./configure.sh
+fi

shutter/scripts/run_keyper.sh

@@ -18,7 +18,12 @@ perform_chain_healthcheck() {
 }
 
 run_keyper() {
-    $SHUTTER_BIN shutterservicekeyper --config "$KEYPER_CONFIG_FILE"
+    if [[ SHUTTER_PUSH_LOGS_ENABLED=true ]];
+    then
+        $SHUTTER_BIN shutterservicekeyper --config "$KEYPER_CONFIG_FILE" | rotatelogs -n 1 -e -c /logs/keyper.log 5M
+    else
+        $SHUTTER_BIN shutterservicekeyper --config "$KEYPER_CONFIG_FILE"
+    fi
 }
 
 perform_chain_healthcheck

shutter/scripts/run_promtail.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+run_promtail() {
+    if [[ $SHUTTER_PUSH_LOGS_ENABLED=true ]];
+    then
+        promtail -log-config-reverse-order -print-config-stderr -config.file /etc/promtail_config.yaml
+    else
+        tail -f /dev/null
+    fi
+}
+
+run_promtail

shutter/supervisord.conf

@@ -18,7 +18,7 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 serverurl=unix:///var/run/supervisor.sock
 
 [program:configure]
-command = configure.sh
+command = run_configure.sh
 priority = 1
 autostart = true
 autorestart = false
@@ -47,4 +47,15 @@ autorestart = true
 stdout_logfile = /dev/stdout
 stdout_logfile_maxbytes = 0
 stderr_logfile = /dev/stderr
-stderr_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+
+[program:promtail]
+command = run_promtail.sh
+priority = 4
+autostart = true
+startretries = 9999  ; A large number enough to cover node updates
+autorestart = true
+stdout_logfile = /dev/stdout
+stdout_logfile_maxbytes = 0
+stderr_logfile = /dev/stderr
+stderr_logfile_maxbytes = 0