Skip to content

Commit 46e5efd

Browse files
s0undt3chs0undt3ch
committed
Add an elastic search forwarder (inc. example usage)
Fixes #69 Signed-off-by: Pedro Algarvio <[email protected]>
1 parent 3fc1029 commit 46e5efd

20 files changed

+608
-0
lines changed

‎.dockerignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
RADIO

‎changelog/69.improvement.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add an Elastic Search forwarder(inc. example usage)

‎docker/elastic/.env‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Password for the 'elastic' user (at least 6 characters)
2+
ELASTIC_PASSWORD=elastic
3+
4+
# Password for the 'kibana_system' user (at least 6 characters)
5+
KIBANA_PASSWORD=kibana
6+
7+
# Version of Elastic products
8+
#STACK_VERSION=8.8.2
9+
STACK_VERSION=7.17.11
10+
11+
# Set the cluster name
12+
CLUSTER_NAME=docker-cluster
13+
14+
# Set to 'basic' or 'trial' to automatically start the 30-day trial
15+
LICENSE=basic
16+
#LICENSE=trial
17+
18+
# Port to expose Elasticsearch HTTP API to the host
19+
ES_PORT=9200
20+
#ES_PORT=127.0.0.1:9200
21+
22+
# Port to expose Kibana to the host
23+
KIBANA_PORT=5601
24+
#KIBANA_PORT=80
25+
26+
# Increase or decrease based on the available host memory (in bytes)
27+
MEM_LIMIT=2073741824
28+
29+
# Project namespace (defaults to the current folder name if not set)
30+
#COMPOSE_PROJECT_NAME=myproject

‎docker/elastic/centosstream8.Dockerfile‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
FROM ghcr.io/saltstack/salt-ci-containers/centos-stream:8 as base
2+
3+
ENV LANG=C.UTF-8
4+
ENV LANGUAGE=C.UTF-8
5+
RUN ln -sf /etc/localtime /usr/share/zoneinfo/America/Denver
6+
7+
RUN dnf update -y \
8+
&& dnf upgrade -y \
9+
&& dnf install -y sed vim tmux sudo tree net-tools bind-utils lsof nmap which binutils iputils epel-release procps \
10+
&& dnf install -y --allowerasing curl \
11+
&& dnf install -y multitail supervisor
12+
13+
RUN mkdir -p /etc/supervisor/conf.d/
14+
ADD docker/elastic/conf/supervisord.conf /etc/supervisor/supervisord.conf
15+
16+
RUN rpm --import https://repo.saltproject.io/salt/py3/redhat/9/x86_64/SALT-PROJECT-GPG-PUBKEY-2023.pub \
17+
&& curl -fsSL https://repo.saltproject.io/salt/py3/redhat/9/x86_64/3006.repo | tee /etc/yum.repos.d/salt.repo \
18+
&& dnf install -y salt
19+
20+
COPY ../../dist/salt*.whl /src/
21+
RUN ls -lah /src \
22+
&& /opt/saltstack/salt/salt-pip install /src/salt_analytics_framework*.whl \
23+
&& rm -f /src/*.whl
24+
25+
COPY ../../examples/dist/salt*.whl /src/
26+
RUN ls -lah /src \
27+
&& /opt/saltstack/salt/salt-pip install --find-links /src/ salt-analytics.examples[elasticsearch] \
28+
&& rm -f /src/*.whl
29+
30+
31+
FROM base as minion-2
32+
33+
ADD docker/elastic/conf/supervisord.minion.conf /etc/supervisor/conf.d/minion.conf
34+
ADD docker/elastic/conf/beacons.conf /etc/salt/minion.d/beacons.conf
35+
ADD docker/elastic/conf/salt-analytics.conf /etc/salt/minion.d/salt-analytics.conf
36+
RUN mkdir -p /etc/salt/minion.d \
37+
&& echo 'id: minion-2' > /etc/salt/minion.d/id.conf \
38+
&& echo 'master: master-1' > /etc/salt/minion.d/master.conf \
39+
&& dnf install -y salt-minion
40+
41+
CMD ["/usr/bin/supervisord","-c","/etc/supervisor/supervisord.conf"]

‎docker/elastic/centosstream9.Dockerfile‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
FROM ghcr.io/saltstack/salt-ci-containers/centos-stream:9 as base
2+
3+
ENV LANG=C.UTF-8
4+
ENV LANGUAGE=C.UTF-8
5+
RUN ln -sf /etc/localtime /usr/share/zoneinfo/America/Denver
6+
7+
RUN dnf update -y \
8+
&& dnf upgrade -y \
9+
&& dnf install -y sed vim tmux sudo tree net-tools bind-utils lsof nmap which binutils iputils epel-release procps \
10+
&& dnf install -y --allowerasing curl \
11+
&& dnf install -y multitail supervisor
12+
13+
RUN mkdir -p /etc/supervisor/conf.d/
14+
ADD docker/elastic/conf/supervisord.conf /etc/supervisor/supervisord.conf
15+
16+
RUN rpm --import https://repo.saltproject.io/salt/py3/redhat/9/x86_64/SALT-PROJECT-GPG-PUBKEY-2023.pub \
17+
&& curl -fsSL https://repo.saltproject.io/salt/py3/redhat/9/x86_64/3006.repo | tee /etc/yum.repos.d/salt.repo \
18+
&& dnf install -y salt
19+
20+
COPY ../../dist/salt*.whl /src/
21+
RUN ls -lah /src \
22+
&& /opt/saltstack/salt/salt-pip install /src/salt_analytics_framework*.whl \
23+
&& rm -f /src/*.whl
24+
25+
COPY ../../examples/dist/salt*.whl /src/
26+
RUN ls -lah /src \
27+
&& /opt/saltstack/salt/salt-pip install --find-links /src/ salt-analytics.examples[elasticsearch] \
28+
&& rm -f /src/*.whl
29+
30+
31+
FROM base as master-1
32+
33+
ADD docker/elastic/conf/supervisord.master.conf /etc/supervisor/conf.d/master.conf
34+
ADD docker/elastic/conf/beacons.conf /etc/salt/master.d/beacons.conf
35+
ADD docker/elastic/conf/salt-analytics.conf /etc/salt/master.d/salt-analytics.conf
36+
RUN mkdir -p /etc/salt/master.d \
37+
&& echo 'id: master-1' > /etc/salt/master.d/id.conf \
38+
&& echo 'open_mode: true' > /etc/salt/master.d/open-mode.conf \
39+
&& dnf install -y salt-master
40+
41+
CMD ["/usr/bin/supervisord","-c","/etc/supervisor/supervisord.conf"]
42+
43+
44+
FROM base as minion-1
45+
46+
ADD docker/elastic/conf/supervisord.minion.conf /etc/supervisor/conf.d/minion.conf
47+
ADD docker/elastic/conf/beacons.conf /etc/salt/minion.d/beacons.conf
48+
ADD docker/elastic/conf/salt-analytics.conf /etc/salt/minion.d/salt-analytics.conf
49+
RUN mkdir -p /etc/salt/minion.d \
50+
&& echo 'id: minion-1' > /etc/salt/minion.d/id.conf \
51+
&& echo 'master: master-1' > /etc/salt/minion.d/master.conf \
52+
&& dnf install -y salt-minion
53+
54+
CMD ["/usr/bin/supervisord","-c","/etc/supervisor/supervisord.conf"]

‎docker/elastic/conf/beacons.conf‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
beacons:
2+
# diskusage:
3+
# - '^\/(?!home).*$': 1%
4+
# - '^[a-zA-Z]:\\$': 1%
5+
# - interval: 5
6+
memusage:
7+
- percent: 1%
8+
- interval: 5
9+
# salt_monitor:
10+
# - salt_fun:
11+
# - test.ping
12+
# - interval: 5
13+
## status:
14+
## - interval: 5
15+
## swapusage:
16+
## - percent: 1%
17+
## - interval: 5

‎docker/elastic/conf/salt-analytics.conf‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
engines:
2+
- analytics
3+
4+
analytics:
5+
6+
collectors:
7+
beacons-collector:
8+
plugin: beacons
9+
beacons:
10+
- "*"
11+
12+
processors:
13+
cast-to-es:
14+
plugin: beacons_to_es
15+
16+
forwarders:
17+
elasticsearch-forwarder:
18+
plugin: elasticsearch
19+
hosts:
20+
- http://node01:9200
21+
use_ssl: false
22+
verify_ssl: false
23+
24+
pipelines:
25+
elastic-pipeline:
26+
collect:
27+
- beacons-collector
28+
process:
29+
- cast-to-es
30+
forward:
31+
- elasticsearch-forwarder

‎docker/elastic/conf/supervisord.conf‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
[supervisord]
2+
user=root
3+
logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
4+
logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB)
5+
logfile_backups=10 ; (num of main logfile rotation backups;default 10)
6+
loglevel=info ; (logging level;default info; others: debug,warn)
7+
pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
8+
nodaemon=true ; (start in foreground if true;default false)
9+
minfds=1024 ; (min. avail startup file descriptors;default 1024)
10+
minprocs=200 ; (min. avail process descriptors;default 200)
11+
12+
[unix_http_server]
13+
file=/var/tmp/supervisor.sock
14+
username=setup
15+
password=setup
16+
17+
[supervisorctl]
18+
serverurl=unix:///var/tmp/supervisor.sock ; use a unix:// URL for a unix socket
19+
username=setup
20+
password=setup
21+
22+
; the below section must remain in the config file for RPC
23+
; (supervisorctl/web interface) to work, additional interfaces may be
24+
; added by defining them in separate rpcinterface: sections
25+
[rpcinterface:supervisor]
26+
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
27+
28+
[include]
29+
files = /etc/supervisor/conf.d/*.conf

‎docker/elastic/conf/supervisord.master.conf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[program:salt-master]
2+
command=/opt/saltstack/salt/salt-master -l debug

‎docker/elastic/conf/supervisord.minion.conf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[program:salt-minion]
2+
command=/opt/saltstack/salt/salt-minion -l debug

0 commit comments

Comments
 (0)