add logstash files

Author: c-neto

logstash/db-enrichment/Makefile

@@ -0,0 +1,5 @@
+up:
+	docker-compose up --build
+
+down:
+	docker-compose down

logstash/db-enrichment/docker-compose.yaml

@@ -0,0 +1,37 @@
+services:
+  postgres:
+    image: postgres:15
+    container_name: postgres
+    restart: always
+    environment:
+      POSTGRES_USER: user
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: main
+    ports:
+      - 5432:5432
+    volumes:
+      - ./postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
+
+  logstash:
+    image: docker.elastic.co/logstash/logstash:8.11.2
+    container_name: logstash
+    restart: always
+    build:
+      context: logstash
+    environment:
+      DB_USER: user
+      DB_PASSWORD: password
+      DB_JDBC_CONNECTION_STRING: jdbc:postgresql://postgres:5432/users_logs
+    volumes:
+      - ./logstash/pipeline:/usr/share/logstash/pipeline
+      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml
+      - ./logstash/config/pipelines.yml:/usr/share/logstash/config/pipelines.yml
+
+  adminer:
+    image: adminer
+    container_name: adminer
+    restart: always
+    ports:
+      - "8080:8080"
+    environment:
+      ADMINER_DEFAULT_SERVER: postgres

logstash/db-enrichment/logstash/Dockerfile

@@ -0,0 +1,5 @@
+FROM docker.elastic.co/logstash/logstash:8.11.2
+
+WORKDIR /usr/share/logstash
+
+RUN curl -o /usr/share/logstash/postgresql-42.5.0.jar https://jdbc.postgresql.org/download/postgresql-42.5.0.jar

logstash/db-enrichment/logstash/config/logstash.yml

@@ -0,0 +1,4 @@
+---
+http.host: "0.0.0.0"
+config.reload.automatic: true
+log.level: info

logstash/db-enrichment/logstash/config/pipelines.yml

@@ -0,0 +1,5 @@
+- pipeline.id: app
+  path.config: pipeline/pipeline.cfg
+  pipeline.workers: 1
+  config.reload.automatic: true
+  config.reload.interval: 10s

logstash/db-enrichment/logstash/pipeline/pipeline.cfg

@@ -0,0 +1,42 @@
+input {
+  generator {
+    lines => [
+      '{"event": "login", "user_id": 1}',
+      '{"event": "logout", "user_id": 2}',
+      '{"event": "purchase", "user_id": 3}'
+    ]
+    count => 3
+  }
+}
+
+filter {
+  json {
+    source => "message"
+  }
+
+  mutate {
+    remove_field => ["message"]
+  }
+
+  jdbc_streaming {
+    jdbc_connection_string => "${DB_JDBC_CONNECTION_STRING}"
+    jdbc_user => "${DB_USER}"
+    jdbc_password => "${DB_PASSWORD}"
+    jdbc_driver_library => "/usr/share/logstash/postgresql-42.5.0.jar"
+    jdbc_driver_class => "org.postgresql.Driver"
+    statement => "
+      SELECT user_name, user_email, user_group
+      FROM db_users
+      WHERE id = :input_parameter
+    "
+    parameters => {
+      "input_parameter" => "user_id"
+    }
+    target => "sql"
+  }
+
+}
+
+output {
+  stdout { codec => rubydebug }
+}

logstash/db-enrichment/postgres/init.sql

@@ -0,0 +1,16 @@
+CREATE DATABASE users_logs;
+
+\c users_logs;
+
+CREATE TABLE db_users (
+    id SERIAL PRIMARY KEY,
+    user_name VARCHAR(100) NOT NULL,
+    user_email VARCHAR(100) NOT NULL UNIQUE,
+    user_group VARCHAR(50) NOT NULL
+);
+
+INSERT INTO db_users (user_name, user_email, user_group) VALUES
+('Alice Silva', '[email protected]', 'admin'),
+('Bruno Costa', '[email protected]', 'user'),
+('Carlos Mendes', '[email protected]', 'user'),
+('Diana Lopes', '[email protected]', 'moderator');