add logstash files

Author: c-neto

logstash/db-enrichment/Makefile

@@ -1,5 +1,8 @@
 up:
-	docker-compose up --build
+	docker-compose up --build -d
 
 down:
 	docker-compose down
+
+test:
+	./test.sh

logstash/db-enrichment/docker-compose.yaml

@@ -22,6 +22,9 @@ services:
       DB_USER: user
       DB_PASSWORD: password
       DB_JDBC_CONNECTION_STRING: jdbc:postgresql://postgres:5432/users_logs
+      # DB_JDBC_CONNECTION_STRING: jdbc:postgresql://postgres:5432/users_logs?ApplicationName=logstash&loginTimeout=10&socketTimeout=10&options=-c%20statement_timeout=5000%20-c%20lock_timeout=1000
+    ports:
+      - 8090:8090
     volumes:
       - ./logstash/pipeline:/usr/share/logstash/pipeline
       - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml

logstash/db-enrichment/logstash/Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.elastic.co/logstash/logstash:8.11.2
+FROM docker.elastic.co/logstash/logstash:8.9.2
 
 WORKDIR /usr/share/logstash
 

logstash/db-enrichment/logstash/pipeline/pipeline.cfg

@@ -1,36 +1,43 @@
 input {
-  generator {
-    lines => [
-      '{"event": "login", "user_id": 1}',
-      '{"event": "logout", "user_id": 2}',
-      '{"event": "purchase", "user_id": 3}'
-    ]
-    count => 3
+  http {
+    port => 8090
+    ecs_compatibility => "disabled"
+    additional_codecs => {}
+    codec => json {
+      target => "[document]"
+    }
   }
 }
 
 filter {
-  json {
-    source => "message"
-  }
-
   mutate {
-    remove_field => ["message"]
+    remove_field => ["headers"]
   }
 
   jdbc_streaming {
     jdbc_connection_string => "${DB_JDBC_CONNECTION_STRING}"
     jdbc_user => "${DB_USER}"
     jdbc_password => "${DB_PASSWORD}"
+    jdbc_validate_connection => true
+    jdbc_validation_timeout => 1
     jdbc_driver_library => "/usr/share/logstash/postgresql-42.5.0.jar"
     jdbc_driver_class => "org.postgresql.Driver"
+    # https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc#top
+    # sequel_opts => {
+    #   max_connections => 4
+    #   pool_timeout => 5
+    # }
+    # pg_sleep(10)::text,
     statement => "
-      SELECT user_name, user_email, user_group
+      SELECT
+        user_name,
+        user_email,
+        user_group
       FROM db_users
       WHERE id = :input_parameter
     "
     parameters => {
-      "input_parameter" => "user_id"
+      "input_parameter" => "[document][user_id]"
     }
     target => "sql"
   }

logstash/db-enrichment/test.sh

@@ -0,0 +1,3 @@
+curl -X POST http://127.0.0.1:8090/ \
+  -H "Content-Type: application/json" \
+  -d '{"event": "purchase", "user_id": 3}'