[Platform Observability] Create initial PO package for ingesting kibana ECS formatted logs (#3622)

* create package for ingesting kibana ECS formatted logs

* changelog fix

* update test expected result

* adds a filebeat processor alternative

* clean up poc code

* fix json identation

* Align ingest pipeline approach and improve README.md

* Improve README.md; Fix package folder name; Document ECS fields

* Fix supported kibana version on readme

* Remove experimental attribute from manifest; Fix changelog PR; Fix min version

* Support current and old model  for license

* Fix manifest

* Add event.ingested field; Improve Kibana audit fields documentation

* Fix pipeline test

* Extract ECS processor into a new file; Fix kibana logo; Add event.* fields

* Simplify ingest pipeline and clean-ups

* Fix json format

* Align default log file name with kibana docs

Author: crespocarlos

.github/CODEOWNERS

@@ -178,3 +178,4 @@
 /packages/zscaler @elastic/security-external-integrations
 /packages/zscaler_zia @elastic/security-external-integrations
 /packages/zscaler_zpa @elastic/security-external-integrations
+/packages/platform_observability @elastic/infra-monitoring-ui

packages/platform_observability/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]

packages/platform_observability/_dev/build/docs/README.md

@@ -0,0 +1,27 @@
+# Platform Observability
+
+## Compatibility
+
+This package works with Kibana 8.3.0 and later.
+
+## Kibana logs
+
+The Kibana integration collects logs from [Kibana](https://www.elastic.co/guide/en/kibana/current/introduction.html) instance.
+
+### Logs
+
+#### Audit
+
+Audit logs collects the [Kibana audit logs](https://www.elastic.co/guide/en/kibana/current/security-settings-kb.html).
+
+{{event "kibana_audit"}}
+
+{{fields "kibana_audit"}}
+
+#### Log
+
+Log collects the [Kibana logs](https://www.elastic.co/guide/en/kibana/current/logging-configuration.html).
+
+{{event "kibana_log"}}
+
+{{fields "kibana_log"}}

packages/platform_observability/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3622

packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,2 @@
+dynamic_fields:
+  event.ingested: ".*"

packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log

@@ -0,0 +1 @@
+{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"localhost","path":"/internal/security/session","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"1c8c5808-d2d6-41fc-8cb7-998aa8996be9"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:03.742+00:00","message":"User is requesting [/internal/security/session] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"f8863d86567119e6"}}

packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json

@@ -0,0 +1,60 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2022-06-29T12:05:03.742+00:00",
+            "data_stream": {
+                "dataset": "kibana-audit-log",
+                "namespace": "platform-observability",
+                "type": "logs"
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "action": "http_request",
+                "category": [
+                    "web"
+                ],
+                "dataset": "kibana-audit-log",
+                "ingested": "2022-07-20T08:36:57.202942842Z",
+                "kind": "event",
+                "outcome": "unknown"
+            },
+            "http": {
+                "request": {
+                    "method": "get"
+                }
+            },
+            "kibana": {
+                "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=",
+                "space_id": "default"
+            },
+            "log": {
+                "level": "INFO",
+                "logger": "plugins.security.audit.ecs"
+            },
+            "message": "User is requesting [/internal/security/session] endpoint",
+            "process": {
+                "pid": 7
+            },
+            "trace": {
+                "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9"
+            },
+            "transaction": {
+                "id": "f8863d86567119e6"
+            },
+            "url": {
+                "domain": "localhost",
+                "path": "/internal/security/session",
+                "port": 5601,
+                "scheme": "http"
+            },
+            "user": {
+                "name": "elastic",
+                "roles": [
+                    "superuser"
+                ]
+            }
+        }
+    ]
+}

packages/platform_observability/data_stream/kibana_audit/agent/stream/log.yml.hbs

@@ -0,0 +1,8 @@
+paths:
+{{#each paths}}
+  - {{this}}
+{{/each}}
+{{#if processors}}
+processors:
+{{processors}}  
+{{/if}}

packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,33 @@
+---
+description: Pipeline for parsing Kibana Audit ECS formatted logs
+processors:
+  - pipeline:
+      name: '{{ IngestPipeline "ecs-logs-pipeline" }}'
+      if: |-
+        def message = ctx.message;
+        return message != null
+          && message.startsWith('{')
+          && message.endsWith('}')
+          && message.contains('"@timestamp"')
+  - set:
+      field: data_stream.type
+      value: logs
+  - set:
+      field: data_stream.dataset
+      value: kibana-audit-log
+  - set:
+      field: data_stream.namespace
+      value: platform-observability
+  - set:
+      field: event.dataset
+      copy_from: data_stream.dataset
+  - set:
+      field: event.ingested
+      value: "{{_ingest.timestamp}}"
+  - set:
+      field: event.kind
+      value: event
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"

packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/ecs-logs-pipeline.yml

@@ -0,0 +1,31 @@
+---
+processors:
+  - rename:
+      field: message
+      target_field: _ecs_json_message
+      ignore_missing: true
+  - json:
+      field: _ecs_json_message
+      add_to_root: true
+      add_to_root_conflict_strategy: merge
+      allow_duplicate_keys: true
+      if: ctx.containsKey('_ecs_json_message')
+      on_failure:
+        - rename:
+            field: _ecs_json_message
+            target_field: message
+            ignore_missing: true
+        - set:
+            field: error.message
+            value: Error while parsing JSON
+            override: false
+  - remove:
+      field: _ecs_json_message
+      ignore_missing: true
+  - dot_expander:
+      field: "*"
+      override: true
+  - join:
+      field: error.stack_trace
+      separator: "\n"
+      if: ctx.error?.stack_trace instanceof Collection