Skip to content

Commit 1d31e02

Browse files
codeyourwebcodeyourweb
committed
fluentbit ingest evtx and auditd files
1 parent 1f18508 commit 1d31e02

14 files changed

+365
-52
lines changed

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@
77
/data/ftp_data/*
88
/data/grafana/*
99
/data/kibana/*
10-
/data/log_ingest_data/*
10+
/data/fluentbit_db/*
11+
/data/log_ingest_data/evtx/*
12+
/data/log_ingest_data/auditd/*
13+
/data/log_ingest_data/json/*
1114
/data/mysql_data/*
1215
/data/yara_triage_data/*
1316
/docs/graphs/*.bkp

‎clean-user-data.ps1‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,10 @@ Remove-Item ./data/caddy_logs/* -Recurse -Force
22
Remove-Item ./data/ftp_data/* -Recurse -Force
33
Remove-Item ./data/grafana/* -Recurse -Force
44
Remove-Item ./data/kibana/* -Recurse -Force
5+
Remove-Item ./data/log_ingest_data/auditd/* -Recurse -Force
6+
Remove-Item ./data/log_ingest_data/evtx/* -Recurse -Force
7+
Remove-Item ./data/log_ingest_data/json/* -Recurse -Force
8+
Remove-Item ./data/fluentbit_db/* -Recurse -Force
59
Remove-Item ./data/mysql_data/* -Recurse -Force
10+
Remove-Item ./data/yara_triage_data/* -Recurse -Force
611
docker compose down -v

‎clean-user-data.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,10 @@ rm -rf ./data/caddy_logs/*
22
rm -rf ./data/ftp_data/*
33
rm -rf ./data/grafana/*
44
rm -rf ./data/kibana/*
5+
rm -rf ./data/log_ingest_data/evtx/*
6+
rm -rf ./data/log_ingest_data/auditd/*
7+
rm -rf ./data/log_ingest_data/json/*
8+
rm -rf ./data/fluentbit_db/*
59
rm -rf ./data/mysql_data/*
10+
rm -rf ./data/yara_triage_data/*
611
docker compose down -v

‎config/docker-config/Dockerfile.fluentbit‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
FROM ubuntu:24.04
2+
3+
RUN mkdir -p fluent-bit /fluent-bit/etc /fluent-bit/scripts /fluent-bit/logs /fluent-bit/database
4+
WORKDIR /fluent-bit
5+
COPY config/docker-config/fluentbit-evtx-dump.sh /fluent-bit/scripts/fluentbit-evtx-dump.sh
6+
RUN chmod +x /fluent-bit/scripts/fluentbit-evtx-dump.sh
7+
8+
RUN apt update && \
9+
apt install -y curl ca-certificates dpkg gnupg dpkg-dev && \
10+
curl -L -o /tmp/fluentbit.key https://packages.fluentbit.io/fluentbit.key && \
11+
gpg --dearmor < /tmp/fluentbit.key > /usr/share/keyrings/fluentbit-keyring.gpg && \
12+
rm /tmp/fluentbit.key && \
13+
echo "deb [signed-by=/usr/share/keyrings/fluentbit-keyring.gpg] https://packages.fluentbit.io/ubuntu/noble noble main" > /etc/apt/sources.list.d/fluent-bit.list && \
14+
apt update && \
15+
apt install -y fluent-bit
16+
17+
RUN curl -L -o /tmp/fd-musl_10.3.0_amd64.deb https://github.com/sharkdp/fd/releases/download/v10.3.0/fd-musl_10.3.0_amd64.deb && \
18+
dpkg -i /tmp/fd-musl_10.3.0_amd64.deb && \
19+
rm /tmp/fd-musl_10.3.0_amd64.deb && \
20+
curl -L -o /usr/bin/evtx_dump https://github.com/omerbenamram/evtx/releases/download/v0.9.0/evtx_dump-v0.9.0-x86_64-unknown-linux-musl && \
21+
chmod +x /usr/bin/evtx_dump && \
22+
apt clean && rm -rf /var/lib/apt/lists/*
23+
24+
COPY config/fluentbit_server/fluent-bit.conf /fluent-bit/etc/fluent-bit.conf
25+
ENTRYPOINT ["/opt/fluent-bit/bin/fluent-bit"]
26+
CMD ["-c", "/fluent-bit/etc/fluent-bit.conf"]

‎config/docker-config/fluentbit-evtx-dump.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
EVTX_DIR="/fluent-bit/logs/evtx"
4+
PROCESSED_DIR="/fluent-bit/logs/evtx/processed"
5+
6+
mkdir -p "$PROCESSED_DIR"
7+
8+
find "$EVTX_DIR" -maxdepth 1 -name "*.evtx" -print0 | while IFS= read -r -d $'\0' evtx_file; do
9+
filename=$(basename "$evtx_file" .evtx)
10+
jsonl_file="${EVTX_DIR}/${filename}.jsonl"
11+
echo "Converting $evtx_file to $jsonl_file"
12+
13+
/usr/bin/evtx_dump -f "$jsonl_file" -o jsonl "$evtx_file"
14+
15+
if [ $? -eq 0 ]; then
16+
echo "Conversion successful. Moving original..."
17+
mv "$evtx_file" "$PROCESSED_DIR/"
18+
else
19+
echo "Error occurred during evtx_dump for $evtx_file." >&2
20+
fi
21+
done

‎config/fluentbit_server/add_timestamp.lua‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,31 @@
11
function add_timestamp(tag, timestamp, record)
2-
if record["@timestamp"] == nil then
3-
local seconds = timestamp[1]
4-
local formatted_time = os.date("!%Y-%m-%dT%H:%M:%S", seconds) .. string.format(".%03dZ", timestamp[2]/1000000)
5-
record["@timestamp"] = formatted_time
2+
3+
local event_timestamp = nil
4+
5+
if record["Event"] ~= nil and record["Event"]["System"] ~= nil and
6+
record["Event"]["System"]["TimeCreated"] ~= nil and
7+
record["Event"]["System"]["TimeCreated"]["#attributes"] ~= nil and
8+
record["Event"]["System"]["TimeCreated"]["#attributes"]["SystemTime"] ~= nil then
9+
10+
event_timestamp = record["Event"]["System"]["TimeCreated"]["#attributes"]["SystemTime"]
11+
end
12+
13+
14+
if record["@timestamp"] == nil and record["timestamp"] ~= nil then
15+
event_timestamp = record["timestamp"]
16+
record["timestamp"] = nil
17+
end
18+
19+
if event_timestamp ~= nil and event_timestamp ~= "" then
20+
record["@timestamp"] = event_timestamp
21+
else
22+
if record["@timestamp"] == nil then
23+
local seconds = math.floor(timestamp)
24+
local nanoseconds = (timestamp - seconds) * 1e9
25+
local milliseconds = math.floor(nanoseconds / 1e6)
26+
local formatted_time = os.date("!%Y-%m-%dT%H:%M:%S", seconds) .. string.format(".%03dZ", milliseconds)
27+
record["@timestamp"] = formatted_time
28+
end
629
end
730

831
return 1, timestamp, record

‎config/fluentbit_server/fluent-bit.conf‎

Lines changed: 6 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# Fluent Bit service configuration
12
[SERVICE]
23
Flush 1
34
Daemon off
@@ -7,46 +8,9 @@
78
HTTP_Listen 0.0.0.0
89
HTTP_Port 2020
910
Health_Check On
11+
HTTP_Buffer_Size 1048576
1012

11-
[INPUT]
12-
Name http
13-
Listen 0.0.0.0
14-
Port 24224
15-
Tag http.logs
16-
17-
[FILTER]
18-
Name parser
19-
Match http.logs
20-
Key_Name log
21-
Parser json_parser
22-
Preserve_Key On
23-
24-
[FILTER]
25-
Name lua
26-
Match http.logs
27-
Script /fluent-bit/etc/add_timestamp.lua
28-
Call add_timestamp
29-
30-
[FILTER]
31-
Name lua
32-
Match http.logs
33-
Script /fluent-bit/etc/set_target_index.lua
34-
Call set_target_index
35-
36-
[OUTPUT]
37-
Name es
38-
Match http.logs
39-
Host sentinel-kit-db-elasticsearch-es01
40-
Port 9200
41-
Logstash_Format On
42-
Logstash_Prefix_Key target_index
43-
Logstash_DateFormat %Y.%m.%d
44-
Type _doc
45-
Time_Key @timestamp
46-
Replace_Dots On
47-
Suppress_Type_Name On
48-
Retry_Limit False
49-
TLS On
50-
TLS.Verify Off
51-
HTTP_User elastic
52-
HTTP_Passwd ${ELASTIC_PASSWORD}
13+
@include /fluent-bit/etc/logs-evtx.conf
14+
@include /fluent-bit/etc/logs-auditd.conf
15+
@include /fluent-bit/etc/logs-json.conf
16+
@include /fluent-bit/etc/logs-http.conf

‎config/fluentbit_server/logs-auditd.conf‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
[INPUT]
2+
Name tail
3+
Path /fluent-bit/logs/auditd/*.log
4+
Tag auditd.logs
5+
Parser auditd_line
6+
DB /fluent-bit/database/auditd_db.db
7+
Read_from_Head true
8+
Buffer_Chunk_Size 1M
9+
Buffer_Max_Size 5M
10+
11+
[FILTER]
12+
Name lua
13+
Match auditd.logs
14+
script /fluent-bit/etc/parse_audit_message.lua
15+
call parse_auditd_message
16+
17+
[OUTPUT]
18+
Name es
19+
Match auditd.logs
20+
Host sentinel-kit-db-elasticsearch-es01
21+
Port 9200
22+
Buffer_Size 5M
23+
Logstash_Format On
24+
Logstash_Prefix ingest-auditd
25+
Logstash_DateFormat %Y.%m.%d
26+
Type _doc
27+
Time_Key @timestamp
28+
Replace_Dots On
29+
Suppress_Type_Name On
30+
Retry_Limit False
31+
TLS On
32+
TLS.Verify Off
33+
HTTP_User elastic
34+
HTTP_Passwd ${ELASTIC_PASSWORD}

‎config/fluentbit_server/logs-evtx.conf‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
[INPUT]
2+
Name exec
3+
Tag evtx.converter.run
4+
Interval_Sec 30
5+
Interval_NSec 0
6+
Buf_Size 128mb
7+
Command /fluent-bit/scripts/fluentbit-evtx-dump.sh
8+
Oneshot false
9+
Threaded true
10+
11+
12+
[INPUT]
13+
Name tail
14+
Path /fluent-bit/logs/evtx/*.jsonl
15+
Tag json_evtx.logs
16+
DB /fluent-bit/database/evtx_jsonl_db.db
17+
Parser json_parser
18+
Read_from_Head true
19+
Buffer_Chunk_Size 1M
20+
Buffer_Max_Size 5M
21+
22+
[FILTER]
23+
Name lua
24+
Match json_evtx.logs
25+
Script /fluent-bit/etc/add_timestamp.lua
26+
Call add_timestamp
27+
28+
[OUTPUT]
29+
Name es
30+
Match json_evtx.logs
31+
Host sentinel-kit-db-elasticsearch-es01
32+
Port 9200
33+
Buffer_Size 5M
34+
Logstash_Format On
35+
Logstash_Prefix ingest-evtx
36+
Logstash_DateFormat %Y.%m.%d
37+
Type _doc
38+
Time_Key @timestamp
39+
Replace_Dots On
40+
Suppress_Type_Name On
41+
Retry_Limit False
42+
TLS On
43+
TLS.Verify Off
44+
HTTP_User elastic
45+
HTTP_Passwd ${ELASTIC_PASSWORD}

‎config/fluentbit_server/logs-http.conf‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
[INPUT]
2+
Name http
3+
Listen 0.0.0.0
4+
Port 24224
5+
Tag http.logs
6+
7+
[FILTER]
8+
Name parser
9+
Match http.logs
10+
Key_Name log
11+
Parser json_parser
12+
Preserve_Key On
13+
14+
[FILTER]
15+
Name lua
16+
Match http.logs
17+
Script /fluent-bit/etc/add_timestamp.lua
18+
Call add_timestamp
19+
20+
[FILTER]
21+
Name lua
22+
Match http.logs
23+
Script /fluent-bit/etc/set_target_index.lua
24+
Call set_target_index
25+
26+
[OUTPUT]
27+
Name es
28+
Match http.logs
29+
Host sentinel-kit-db-elasticsearch-es01
30+
Port 9200
31+
Buffer_Size 480K
32+
Logstash_Format On
33+
Logstash_Prefix_Key target_index
34+
Logstash_DateFormat %Y.%m.%d
35+
Type _doc
36+
Time_Key @timestamp
37+
Replace_Dots On
38+
Suppress_Type_Name On
39+
Retry_Limit False
40+
TLS On
41+
TLS.Verify Off
42+
HTTP_User elastic
43+
HTTP_Passwd ${ELASTIC_PASSWORD}

0 commit comments

Comments
 (0)