[Platform Observability] Create initial PO package for ingesting kibana ECS formatted logs

packages/platform-observability/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1

packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs-config.yml

@@ -0,0 +1,11 @@
+multiline:
+  first_line_pattern: "^{"
+fields:
+  ecs:
+    version: "8.0.0"
+  event:
+    dataset: ecs_router
+  data_stream:
+    type: logs
+    dataset: ecs_router
+    namespace: default

packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log

@@ -0,0 +1,3 @@
+{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"localhost","path":"/internal/security/session","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"1c8c5808-d2d6-41fc-8cb7-998aa8996be9"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:03.742+00:00","message":"User is requesting [/internal/security/session] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"f8863d86567119e6"}}
+{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/internal/bsearch","port":5601,"query":"compress=true","scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"abc8b4ad-5d96-42cf-9653-08aaeac0034e"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:08.178+00:00","message":"User is requesting [/internal/bsearch] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"3098796995e24283"}}
+{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/api/log_entries/summary","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"f4181218-b2d3-480e-b9da-78aef88683ff"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:08.187+00:00","message":"User is requesting [/api/log_entries/summary] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"c1480039d6e6e321"}}

packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json

@@ -0,0 +1,167 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2022-06-29T12:05:03.742+00:00",
+            "data_stream": {
+                "dataset": "kibana-audit-log",
+                "namespace": "platform-observability",
+                "type": "logs"
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "action": "http_request",
+                "category": [
+                    "web"
+                ],
+                "dataset": "kibana-audit-log",
+                "outcome": "unknown"
+            },
+            "http": {
+                "request": {
+                    "method": "get"
+                }
+            },
+            "kibana": {
+                "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=",
+                "space_id": "default"
+            },
+            "log": {
+                "level": "INFO",
+                "logger": "plugins.security.audit.ecs"
+            },
+            "message": "User is requesting [/internal/security/session] endpoint",
+            "process": {
+                "pid": 7
+            },
+            "trace": {
+                "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9"
+            },
+            "transaction": {
+                "id": "f8863d86567119e6"
+            },
+            "url": {
+                "domain": "localhost",
+                "path": "/internal/security/session",
+                "port": 5601,
+                "scheme": "http"
+            },
+            "user": {
+                "name": "elastic",
+                "roles": [
+                    "superuser"
+                ]
+            }
+        },
+        {
+            "@timestamp": "2022-06-29T12:05:08.178+00:00",
+            "data_stream": {
+                "dataset": "kibana-audit-log",
+                "namespace": "platform-observability",
+                "type": "logs"
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "action": "http_request",
+                "category": [
+                    "web"
+                ],
+                "dataset": "kibana-audit-log",
+                "outcome": "unknown"
+            },
+            "http": {
+                "request": {
+                    "method": "post"
+                }
+            },
+            "kibana": {
+                "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=",
+                "space_id": "default"
+            },
+            "log": {
+                "level": "INFO",
+                "logger": "plugins.security.audit.ecs"
+            },
+            "message": "User is requesting [/internal/bsearch] endpoint",
+            "process": {
+                "pid": 7
+            },
+            "trace": {
+                "id": "abc8b4ad-5d96-42cf-9653-08aaeac0034e"
+            },
+            "transaction": {
+                "id": "3098796995e24283"
+            },
+            "url": {
+                "domain": "localhost",
+                "path": "/internal/bsearch",
+                "port": 5601,
+                "query": "compress=true",
+                "scheme": "http"
+            },
+            "user": {
+                "name": "elastic",
+                "roles": [
+                    "superuser"
+                ]
+            }
+        },
+        {
+            "@timestamp": "2022-06-29T12:05:08.187+00:00",
+            "data_stream": {
+                "dataset": "kibana-audit-log",
+                "namespace": "platform-observability",
+                "type": "logs"
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "action": "http_request",
+                "category": [
+                    "web"
+                ],
+                "dataset": "kibana-audit-log",
+                "outcome": "unknown"
+            },
+            "http": {
+                "request": {
+                    "method": "post"
+                }
+            },
+            "kibana": {
+                "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=",
+                "space_id": "default"
+            },
+            "log": {
+                "level": "INFO",
+                "logger": "plugins.security.audit.ecs"
+            },
+            "message": "User is requesting [/api/log_entries/summary] endpoint",
+            "process": {
+                "pid": 7
+            },
+            "trace": {
+                "id": "f4181218-b2d3-480e-b9da-78aef88683ff"
+            },
+            "transaction": {
+                "id": "c1480039d6e6e321"
+            },
+            "url": {
+                "domain": "localhost",
+                "path": "/api/log_entries/summary",
+                "port": 5601,
+                "scheme": "http"
+            },
+            "user": {
+                "name": "elastic",
+                "roles": [
+                    "superuser"
+                ]
+            }
+        }
+    ]
+}

packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs

@@ -0,0 +1,7 @@
+paths:
+{{#each paths}}
+  - {{this}}
+{{/each}}
+{{#if processors}}
+{{processors}}  
+{{/if}}

packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,47 @@
+---
+description:
+  This data stream is meant for routing only, we want to avoid that data is written to it.
+  We'll use the dataset that is specified in the ECS JSON log message, or use 'generic' as the default.
+processors:
+  - remove:
+      field: data_stream.dataset
+      ignore_missing: true
+  - remove:
+      field: event.dataset
+      ignore_missing: true
+  - pipeline:
+      name: '{{ IngestPipeline "kibana-audit-logs-ecs" }}'
+      if: |-
+        def message = ctx.message;
+        return message != null
+          && message.startsWith('{')
+          && message.endsWith('}')
+          && message.contains('"@timestamp"')
+  - set:
+      description: Uses event.dataset as a default for data_stream.dataset if the latter is not set.
+      field: data_stream.dataset
+      copy_from: event.dataset
+      if: ctx.event?.dataset instanceof String && ctx.event.dataset.length() > 1
+      override: false
+  - script:
+      source: |
+        ctx.data_stream.dataset = /[\/*?"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_')
+      if: ctx.data_stream?.dataset != null
+  - script:
+      source: |
+        ctx.data_stream.namespace = /[\/*?"<>|, #:]/.matcher(ctx.data_stream.namespace).replaceAll('_')
+      if: ctx.data_stream?.namespace != null
+  - set:
+      field: data_stream.type
+      value: logs
+  - set:
+      field: data_stream.dataset
+      value: kibana-audit-log
+      override: false
+  - set:
+      field: data_stream.namespace
+      value: platform-observability
+      override: false
+  - set:
+      field: event.dataset
+      copy_from: data_stream.dataset

packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/kibana-audit-logs-ecs.yml

@@ -0,0 +1,32 @@
+---
+description: Pipeline for Kibana Audit Logs
+processors:
+  - rename:
+      field: message
+      target_field: _ecs_json_message
+      ignore_missing: true
+  - json:
+      field: _ecs_json_message
+      add_to_root: true
+      add_to_root_conflict_strategy: merge
+      allow_duplicate_keys: true
+      if: ctx.containsKey('_ecs_json_message')
+      on_failure:
+        - rename:
+            field: _ecs_json_message
+            target_field: message
+            ignore_missing: true
+        - set:
+            field: error.message
+            value: Error while parsing JSON
+            override: false
+  - remove:
+      field: _ecs_json_message
+      ignore_missing: true
+  - dot_expander:
+      field: "*"
+      override: true
+  - join:
+      field: error.stack_trace
+      separator: "\n"
+      if: ctx.error?.stack_trace instanceof Collection

packages/platform-observability/data_stream/audit/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/platform-observability/data_stream/audit/fields/ecs.yml

@@ -0,0 +1,34 @@
+- name: ecs.version
+  external: ecs
+- name: http.request.method
+  external: ecs
+- name: kibana.session_id
+  external: ecs
+- name: kibana.space_id
+  external: ecs
+- name: log.level
+  external: ecs
+- name: log.logger
+  external: ecs
+- name: message
+  external: ecs
+- name: process.pid
+  external: ecs
+- name: trace.id
+  external: ecs
+- name: transaction.id
+  external: ecs
+- name: url.domain
+  external: ecs
+- name: url.path
+  external: ecs
+- name: url.port
+  external: ecs
+- name: url.query
+  external: ecs
+- name: url.scheme
+  external: ecs
+- name: user.name
+  external: ecs
+- name: user.roles
+  external: ecs

packages/platform-observability/data_stream/audit/manifest.yml

@@ -0,0 +1,17 @@
+type: logs
+title: Platform Observability Kibana Audit Logs
+release: experimental
+streams:
+  - input: logfile
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        required: true
+        show_user: true
+        default:
+          - /tmp/service_logs/audit*.log
+    template_path: log.yml.hbs
+    title: Kibana Audit Logs
+    description: Collect Kibana Audit Logs

packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs-config.yml

@@ -0,0 +1,11 @@
+multiline:
+  first_line_pattern: "^{"
+fields:
+  ecs:
+    version: "8.0.0"
+  event:
+    dataset: ecs_router
+  data_stream:
+    type: logs
+    dataset: ecs_router
+    namespace: default

packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log

@@ -0,0 +1,4 @@
+{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Trying to authenticate user request to /login.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}}
+{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Authorization header is not presented.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}}
+{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Could not handle authentication attempt","log":{"level":"DEBUG","logger":"plugins.security.authentication"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}}
+{"client":{"ip":"127.0.0.1"},"http":{"request":{"method":"GET","mime_type":null,"referrer":"","headers":{"host":"127.0.0.1:5601","user-agent":"curl/7.68.0","accept":"*/*"}},"response":{"body":{"bytes":84749},"status_code":200,"headers":{"content-security-policy":"script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'","x-content-type-options":"nosniff","referrer-policy":"no-referrer-when-downgrade","kbn-name":"kibana","kbn-license-sig":"fc1dbd9f3decaa38ee166c07aa16570255c36c01fd35842394ebec327e175722","content-type":"text/html; charset=utf-8","cache-control":"private, no-cache, no-store, must-revalidate","content-length":84749,"vary":"accept-encoding","accept-ranges":"bytes"},"responseTime":8}},"url":{"path":"/login","query":""},"user_agent":{"original":"curl/7.68.0"},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.905+00:00","message":"GET /login 200 8ms - 82.8KB","log":{"level":"DEBUG","logger":"http.server.response"},"process":{"pid":7},"transaction":{"id":"3be6994d7f6d5465"}}