[WIP] Add an elastic search forward (inc. example usage)

.dockerignore

@@ -0,0 +1 @@
+RADIO

changelog/68.bugfix.rst

@@ -0,0 +1 @@
+Fix regression which prevented pipelines without processors

changelog/68.improvement.rst

@@ -0,0 +1,5 @@
+Several improvements to the project
+
+* When loading plugin configs, catch `KeyError` on unavailable plugins and list all available.
+* Pass the event tag and stamp explicitly when calling `saf.utils.eventbus._construct_event()`
+* Add the `id` attribute to `BeaconCollectedEvent`, which is minion ID.

changelog/69.improvement.rst

@@ -0,0 +1 @@
+Add an Elastic Search forwarder(inc. example usage)

docker/elastic/.env

@@ -0,0 +1,30 @@
+# Password for the 'elastic' user (at least 6 characters)
+ELASTIC_PASSWORD=elastic
+
+# Password for the 'kibana_system' user (at least 6 characters)
+KIBANA_PASSWORD=kibana
+
+# Version of Elastic products
+#STACK_VERSION=8.8.2
+STACK_VERSION=7.17.11
+
+# Set the cluster name
+CLUSTER_NAME=docker-cluster
+
+# Set to 'basic' or 'trial' to automatically start the 30-day trial
+LICENSE=basic
+#LICENSE=trial
+
+# Port to expose Elasticsearch HTTP API to the host
+ES_PORT=9200
+#ES_PORT=127.0.0.1:9200
+
+# Port to expose Kibana to the host
+KIBANA_PORT=5601
+#KIBANA_PORT=80
+
+# Increase or decrease based on the available host memory (in bytes)
+MEM_LIMIT=2073741824
+
+# Project namespace (defaults to the current folder name if not set)
+#COMPOSE_PROJECT_NAME=myproject

docker/elastic/centosstream8.Dockerfile

@@ -0,0 +1,39 @@
+FROM ghcr.io/saltstack/salt-ci-containers/centos-stream:8 as base
+
+ENV LANG=C.UTF-8
+ENV LANGUAGE=C.UTF-8
+RUN ln -sf /etc/localtime /usr/share/zoneinfo/America/Denver
+
+RUN dnf update -y \
+    && dnf upgrade -y \
+    && dnf install -y sed vim tmux sudo tree net-tools bind-utils lsof nmap which binutils iputils epel-release procps \
+    && dnf install -y --allowerasing curl \
+    && dnf install -y multitail supervisor
+
+RUN mkdir -p /etc/supervisor/conf.d/
+ADD docker/elastic/conf/supervisord.conf /etc/supervisord.conf
+
+RUN rpm --import https://repo.saltproject.io/salt/py3/redhat/9/x86_64/SALT-PROJECT-GPG-PUBKEY-2023.pub \
+  && curl -fsSL https://repo.saltproject.io/salt/py3/redhat/9/x86_64/3006.repo | tee /etc/yum.repos.d/salt.repo \
+  && dnf install -y salt
+
+COPY ../../dist/salt*.whl /src/
+RUN ls -lah /src \
+  && /opt/saltstack/salt/salt-pip install /src/salt_analytics_framework*.whl \
+  && rm -f /src/*.whl
+
+COPY ../../examples/dist/salt*.whl /src/
+RUN ls -lah /src \
+  && /opt/saltstack/salt/salt-pip install --find-links /src/ salt-analytics.examples[elasticsearch] \
+  && rm -f /src/*.whl
+
+
+FROM base as minion-2
+
+RUN dnf install -y salt-minion
+ADD docker/elastic/conf/supervisord.minion.conf /etc/supervisor/conf.d/minion.conf
+ADD docker/elastic/conf/beacons.conf /etc/salt/minion.d/beacons.conf
+ADD docker/elastic/conf/analytics.minion.conf /etc/salt/minion.d/salt-analytics.conf
+ADD docker/elastic/conf/minion-2.conf /etc/salt/minion.d/minion-2.conf
+
+CMD ["/usr/bin/supervisord","-c","/etc/supervisord.conf"]

docker/elastic/centosstream9.Dockerfile

@@ -0,0 +1,52 @@
+FROM ghcr.io/saltstack/salt-ci-containers/centos-stream:9 as base
+
+ENV LANG=C.UTF-8
+ENV LANGUAGE=C.UTF-8
+RUN ln -sf /etc/localtime /usr/share/zoneinfo/America/Denver
+
+RUN dnf update -y \
+    && dnf upgrade -y \
+    && dnf install -y sed vim tmux sudo tree net-tools bind-utils lsof nmap which binutils iputils epel-release procps \
+    && dnf install -y --allowerasing curl \
+    && dnf install -y multitail supervisor
+
+RUN mkdir -p /etc/supervisor/conf.d/
+ADD docker/elastic/conf/supervisord.conf /etc/supervisord.conf
+
+RUN rpm --import https://repo.saltproject.io/salt/py3/redhat/9/x86_64/SALT-PROJECT-GPG-PUBKEY-2023.pub \
+  && curl -fsSL https://repo.saltproject.io/salt/py3/redhat/9/x86_64/3006.repo | tee /etc/yum.repos.d/salt.repo \
+  && dnf install -y salt
+
+COPY ../../dist/salt*.whl /src/
+RUN ls -lah /src \
+  && /opt/saltstack/salt/salt-pip install /src/salt_analytics_framework*.whl \
+  && rm -f /src/*.whl
+
+COPY ../../examples/dist/salt*.whl /src/
+RUN ls -lah /src \
+  && /opt/saltstack/salt/salt-pip install --find-links /src/ salt-analytics.examples[elasticsearch] \
+  && rm -f /src/*.whl
+
+
+FROM base as master-1
+
+RUN dnf install -y salt-master
+ADD docker/elastic/conf/supervisord.master.conf /etc/supervisor/conf.d/master.conf
+ADD docker/elastic/conf/supervisord.loop-jobs.conf /etc/supervisor/conf.d/loop-jobs.conf
+ADD docker/elastic/loop-jobs.sh /usr/bin/loop-jobs.sh
+ADD docker/elastic/conf/analytics.master.conf /etc/salt/master.d/salt-analytics.conf
+ADD docker/elastic/conf/master-1.conf /etc/salt/master.d/master-1.conf
+ADD docker/elastic/state-tree/*.sls /srv/salt/
+
+CMD ["/usr/bin/supervisord","-c","/etc/supervisord.conf"]
+
+
+FROM base as minion-1
+
+RUN dnf install -y salt-minion
+ADD docker/elastic/conf/supervisord.minion.conf /etc/supervisor/conf.d/minion.conf
+ADD docker/elastic/conf/beacons.conf /etc/salt/minion.d/beacons.conf
+ADD docker/elastic/conf/analytics.minion.conf /etc/salt/minion.d/salt-analytics.conf
+ADD docker/elastic/conf/minion-1.conf /etc/salt/minion.d/minion-1.conf
+
+CMD ["/usr/bin/supervisord","-c","/etc/supervisord.conf"]

docker/elastic/conf/analytics.master.conf

@@ -0,0 +1,57 @@
+engines:
+  - analytics
+
+analytics:
+
+  collectors:
+    jobs-collector:
+      plugin: event-bus
+      tags:
+        - "salt/job/*"
+
+    grains-collector:
+      plugin: grains
+      interval: 30
+      grains:
+        - "os"
+        - "id"
+        - "role"
+        - "datacenter"
+
+  processors:
+    job-aggregate:
+      plugin: job-aggregate
+    state-job-aggregate:
+      plugin: job-aggregate
+      jobs:
+        - "state.*"
+    cast-to-es:
+      plugin: jobs_to_es
+    state-cast-to-es:
+      plugin: state_jobs_to_es
+
+  forwarders:
+    elasticsearch-forwarder:
+      plugin: elasticsearch
+      hosts:
+        - http://node01:9200
+
+  pipelines:
+
+    jobs-pipeline:
+      collect:
+        - grains-collector
+        - jobs-collector
+      process:
+        - job-aggregate
+        - cast-to-es
+      forward: elasticsearch-forwarder
+
+    state-jobs-pipeline:
+      collect:
+        - grains-collector
+        - jobs-collector
+      process:
+        - state-job-aggregate
+        - state-cast-to-es
+      forward: elasticsearch-forwarder

docker/elastic/conf/analytics.minion.conf

@@ -0,0 +1,31 @@
+engines:
+  - analytics
+
+analytics:
+
+  collectors:
+    beacons-collector:
+      plugin: beacons
+      beacons:
+        - "*"
+
+  processors:
+    cast-to-es:
+      plugin: beacons_to_es
+
+  forwarders:
+    elasticsearch-forwarder:
+      plugin: elasticsearch
+      hosts:
+        - http://node01:9200
+      use_ssl: false
+      verify_ssl: false
+
+  pipelines:
+    elastic-pipeline:
+      collect:
+        - beacons-collector
+      process:
+        - cast-to-es
+      forward:
+        - elasticsearch-forwarder

docker/elastic/conf/beacons.conf

@@ -0,0 +1,17 @@
+beacons:
+#  diskusage:
+#    - '^\/(?!home).*$': 1%
+#    - '^[a-zA-Z]:\\$': 1%
+#    - interval: 5
+  memusage:
+    - percent: 1%
+    - interval: 5
+#  salt_monitor:
+#    - salt_fun:
+#      - test.ping
+#    - interval: 5
+##  status:
+##    - interval: 5
+##  swapusage:
+##    - percent: 1%
+##    - interval: 5

docker/elastic/conf/master-1.conf

@@ -0,0 +1,5 @@
+id: master-1
+open_mode: true
+log_level_logfile: debug
+grains:
+  datacenter: dt-a

docker/elastic/conf/master-2.conf

@@ -0,0 +1,5 @@
+id: master-2
+open_mode: true
+log_level_logfile: debug
+grains:
+  datacenter: dt-b

docker/elastic/conf/minion-1.conf

@@ -0,0 +1,6 @@
+id: minion-1
+master: master-1
+log_level_logfile: debug
+grains:
+  role: db
+  datacenter: dt-a

docker/elastic/conf/minion-2.conf

@@ -0,0 +1,6 @@
+id: minion-2
+master: master-1
+log_level_logfile: debug
+grains:
+  role: web
+  datacenter: dt-a

docker/elastic/conf/minion-3.conf

@@ -0,0 +1,6 @@
+id: minion-3
+master: master-2
+log_level_logfile: debug
+grains:
+  role: db
+  datacenter: dt-b

docker/elastic/conf/minion-4.conf

@@ -0,0 +1,6 @@
+id: minion-4
+master: master-2
+log_level_logfile: debug
+grains:
+  role: web
+  datacenter: dt-b

docker/elastic/conf/supervisord.conf

@@ -0,0 +1,29 @@
+[supervisord]
+user=root
+logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
+logfile_maxbytes=50MB       ; (max main logfile bytes b4 rotation;default 50MB)
+logfile_backups=10          ; (num of main logfile rotation backups;default 10)
+loglevel=info               ; (logging level;default info; others: debug,warn)
+pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
+nodaemon=true              ; (start in foreground if true;default false)
+minfds=1024                 ; (min. avail startup file descriptors;default 1024)
+minprocs=200                ; (min. avail process descriptors;default 200)
+
+[unix_http_server]
+file=/var/tmp/supervisor.sock
+username=setup
+password=setup
+
+[supervisorctl]
+serverurl=unix:///var/tmp/supervisor.sock ; use a unix:// URL  for a unix socket
+username=setup
+password=setup
+
+; the below section must remain in the config file for RPC
+; (supervisorctl/web interface) to work, additional interfaces may be
+; added by defining them in separate rpcinterface: sections
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[include]
+files = /etc/supervisor/conf.d/*.conf

docker/elastic/conf/supervisord.loop-jobs.conf

@@ -0,0 +1,2 @@
+[program:loop-jobs]
+command=/bin/sh /usr/bin/loop-jobs.sh

docker/elastic/conf/supervisord.master.conf

@@ -0,0 +1,2 @@
+[program:salt-master]
+command=/opt/saltstack/salt/salt-master -l debug

docker/elastic/conf/supervisord.minion.conf

@@ -0,0 +1,2 @@
+[program:salt-minion]
+command=/opt/saltstack/salt/salt-minion -l debug

docker/elastic/debian10.Dockerfile

@@ -0,0 +1,39 @@
+FROM ghcr.io/saltstack/salt-ci-containers/debian:10 as base
+
+ENV LANG=C.UTF-8
+ENV LANGUAGE=C.UTF-8
+RUN ln -sf /etc/localtime /usr/share/zoneinfo/America/Denver
+
+RUN apt update \
+    && apt upgrade -y \
+    && apt install -y curl sed vim tmux sudo tree net-tools bind9utils lsof nmap binutils multitail supervisor iputils-ping procps
+
+RUN mkdir -p /etc/supervisor/conf.d/
+ADD docker/elastic/conf/supervisord.conf /etc/supervisor/supervisord.conf
+
+RUN mkdir /etc/apt/keyrings \
+  && curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.gpg https://repo.saltproject.io/salt/py3/debian/11/amd64/SALT-PROJECT-GPG-PUBKEY-2023.gpg \
+  && echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.gpg arch=amd64] https://repo.saltproject.io/salt/py3/debian/11/amd64/3006 bullseye main" | tee /etc/apt/sources.list.d/salt.list \
+  && apt update \
+  && apt install -y salt-common
+
+COPY ../../dist/salt*.whl /src/
+RUN ls -lah /src \
+  && /opt/saltstack/salt/salt-pip install /src/salt_analytics_framework*.whl \
+  && rm -f /src/*.whl
+
+COPY ../../examples/dist/salt*.whl /src/
+RUN ls -lah /src \
+  && /opt/saltstack/salt/salt-pip install --find-links /src/ salt-analytics.examples[elasticsearch] \
+  && rm -f /src/*.whl
+
+
+FROM base as minion-4
+
+RUN apt install -y salt-minion
+ADD docker/elastic/conf/supervisord.minion.conf /etc/supervisor/conf.d/minion.conf
+ADD docker/elastic/conf/beacons.conf /etc/salt/minion.d/beacons.conf
+ADD docker/elastic/conf/analytics.minion.conf /etc/salt/minion.d/salt-analytics.conf
+ADD docker/elastic/conf/minion-4.conf /etc/salt/minion.d/minion-4.conf
+
+CMD ["/usr/bin/supervisord","-c","/etc/supervisor/supervisord.conf"]