Merge pull request #1 from NETWAYS/initial

Initial import of code

Author: widhalmt

.github/workflows/logstash.yml

@@ -0,0 +1,31 @@
+---
+name: Logstash Syntax
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install gpg
+          wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+          echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-7.x.list
+          sudo apt-get update
+          sudo apt-get install logstash
+          mkdir -p /tmp/logstash/data /tmp/logstash/logs
+
+      - name: Test with Logstash
+        run: |
+          /usr/share/logstash/bin/logstash  --path.settings /etc/logstash/ --path.config '*conf' --path.data /tmp/logstash/data --path.logs /tmp/logstash/logs --config.test_and_exit

.gitignore

@@ -0,0 +1,2 @@
+input.conf
+output.conf

README.md

@@ -1,2 +1,38 @@
 # ansible-logstash-pipeline
 Logstash pipeline for processing Ansible logs
+
+[![CI](https://github.com/netways/ansible-logstash-pipeline/workflows/Logstash%20Syntax/badge.svg?event=push)](https://github.com/netways/ansible-logstash-pipeline/actions?query=workflow%3A%22Logstash+Syntax%22)
+
+Minimalist pipeline to parse Ansible logs on managed hosts
+
+Note, that Ansible uses it's module as part of the process name. So make the condition to route into this pipeline a regex check: `[process][name] =~ "ansible"`.
+
+## Inputs and Outputs ##
+
+If you use files called `input.conf` and `output.conf` they will not collide with this rules, even when you want to pull new versions.
+
+### Examples ###
+
+Here's an example for an `input.conf`
+
+```
+input {
+  redis {
+    host => "localhost"
+    data_type => "list"
+    key => "netways-ansible-input"
+  }
+}
+```
+
+and one for `output.conf`.
+
+```
+output {
+  redis {
+    host => "localhost"
+    data_type => "list"
+    key => "netways-ansible-output"
+  }
+}
+```

filter-10-module.conf

@@ -0,0 +1,7 @@
+filter {
+  grok {
+    match => ["[process][name]","ansible-%{GREEDYDATA:[ansible][module]:string}"]
+    id => ansible_module
+    tag_on_failure => ["_grokparsefailure","ansible_module_failed"]
+  }
+}

filter-50-invoked.conf

@@ -0,0 +1,18 @@
+filter {
+  # remove if if really *every* message starts like this
+  if [message] =~ /Invoked with/ {
+
+    grok {
+      match => ["message", "Invoked with %{GREEDYDATA:[@metadata][ansiblekv]}"]
+      id => "ansible_invoked"
+      tag_on_failure => ["_grokparsefailure","ansible_invoked_failed"]
+    }
+
+    kv {
+      source => "[@metadata][ansiblekv]"
+      target => "ansible"
+      id => "ansible_kv"
+      # doesn't take array as tag_on_failure
+    }
+  }
+}