From ba764cb0beb666dbf4ee6bc0c35fa103397284b6 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 29 Jun 2022 14:28:13 +0200 Subject: [PATCH 01/18] create package for ingesting kibana ECS formatted logs --- packages/platform-observability/changelog.yml | 26 +++ .../test-kibana-audit-logs-config.yml | 11 ++ .../test/pipeline/test-kibana-audit-logs.log | 3 + .../test-kibana-audit-logs.log-expected.json | 167 ++++++++++++++++++ .../audit/agent/stream/log.yml.hbs | 7 + .../elasticsearch/ingest_pipeline/default.yml | 47 +++++ .../ingest_pipeline/kibana-audit-logs-ecs.yml | 32 ++++ .../data_stream/audit/fields/base-fields.yml | 12 ++ .../data_stream/audit/fields/ecs.yml | 34 ++++ .../data_stream/audit/manifest.yml | 17 ++ .../test/pipeline/test-kibana-logs-config.yml | 11 ++ .../_dev/test/pipeline/test-kibana-logs.log | 4 + .../test-kibana-logs.log-expected.json | 157 ++++++++++++++++ .../data_stream/log/agent/stream/log.yml.hbs | 7 + .../elasticsearch/ingest_pipeline/default.yml | 47 +++++ .../ingest_pipeline/kibana-logs-ecs.yml | 32 ++++ .../data_stream/log/fields/base-fields.yml | 12 ++ .../data_stream/log/fields/ecs.yml | 62 +++++++ .../data_stream/log/manifest.yml | 17 ++ .../platform-observability/docs/README.md | 7 + .../img/logo_kibana.svg | 7 + packages/platform-observability/manifest.yml | 26 +++ 22 files changed, 745 insertions(+) create mode 100644 packages/platform-observability/changelog.yml create mode 100644 packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs-config.yml create mode 100644 packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log create mode 100644 packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json create mode 100644 packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs create mode 100644 packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/kibana-audit-logs-ecs.yml create mode 100644 packages/platform-observability/data_stream/audit/fields/base-fields.yml create mode 100644 packages/platform-observability/data_stream/audit/fields/ecs.yml create mode 100644 packages/platform-observability/data_stream/audit/manifest.yml create mode 100644 packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs-config.yml create mode 100644 packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log create mode 100644 packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json create mode 100644 packages/platform-observability/data_stream/log/agent/stream/log.yml.hbs create mode 100644 packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/kibana-logs-ecs.yml create mode 100644 packages/platform-observability/data_stream/log/fields/base-fields.yml create mode 100644 packages/platform-observability/data_stream/log/fields/ecs.yml create mode 100644 packages/platform-observability/data_stream/log/manifest.yml create mode 100644 packages/platform-observability/docs/README.md create mode 100644 packages/platform-observability/img/logo_kibana.svg create mode 100644 packages/platform-observability/manifest.yml diff --git a/packages/platform-observability/changelog.yml b/packages/platform-observability/changelog.yml new file mode 100644 index 00000000000..0b18f4f6f4a --- /dev/null +++ b/packages/platform-observability/changelog.yml @@ -0,0 +1,26 @@ +# newer versions go on top +- version: "0.0.5" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 +- version: "0.0.4" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 +- version: "0.0.3" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 +- version: "0.0.2" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 diff --git a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs-config.yml b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs-config.yml new file mode 100644 index 00000000000..2794ccb00fe --- /dev/null +++ b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs-config.yml @@ -0,0 +1,11 @@ +multiline: + first_line_pattern: "^{" +fields: + ecs: + version: "8.0.0" + event: + dataset: ecs_router + data_stream: + type: logs + dataset: ecs_router + namespace: default diff --git a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log new file mode 100644 index 00000000000..95080c52454 --- /dev/null +++ b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log @@ -0,0 +1,3 @@ +{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"localhost","path":"/internal/security/session","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"1c8c5808-d2d6-41fc-8cb7-998aa8996be9"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:03.742+00:00","message":"User is requesting [/internal/security/session] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"f8863d86567119e6"}} +{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/internal/bsearch","port":5601,"query":"compress=true","scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"abc8b4ad-5d96-42cf-9653-08aaeac0034e"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:08.178+00:00","message":"User is requesting [/internal/bsearch] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"3098796995e24283"}} +{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/api/log_entries/summary","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"f4181218-b2d3-480e-b9da-78aef88683ff"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:08.187+00:00","message":"User is requesting [/api/log_entries/summary] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"c1480039d6e6e321"}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json new file mode 100644 index 00000000000..ccfb81549e3 --- /dev/null +++ b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json @@ -0,0 +1,167 @@ +{ + "expected": [ + { + "@timestamp": "2022-06-29T12:05:03.742+00:00", + "data_stream": { + "dataset": "kibana-audit-log", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "action": "http_request", + "category": [ + "web" + ], + "dataset": "kibana-audit-log", + "outcome": "unknown" + }, + "http": { + "request": { + "method": "get" + } + }, + "kibana": { + "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", + "space_id": "default" + }, + "log": { + "level": "INFO", + "logger": "plugins.security.audit.ecs" + }, + "message": "User is requesting [/internal/security/session] endpoint", + "process": { + "pid": 7 + }, + "trace": { + "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9" + }, + "transaction": { + "id": "f8863d86567119e6" + }, + "url": { + "domain": "localhost", + "path": "/internal/security/session", + "port": 5601, + "scheme": "http" + }, + "user": { + "name": "elastic", + "roles": [ + "superuser" + ] + } + }, + { + "@timestamp": "2022-06-29T12:05:08.178+00:00", + "data_stream": { + "dataset": "kibana-audit-log", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "action": "http_request", + "category": [ + "web" + ], + "dataset": "kibana-audit-log", + "outcome": "unknown" + }, + "http": { + "request": { + "method": "post" + } + }, + "kibana": { + "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", + "space_id": "default" + }, + "log": { + "level": "INFO", + "logger": "plugins.security.audit.ecs" + }, + "message": "User is requesting [/internal/bsearch] endpoint", + "process": { + "pid": 7 + }, + "trace": { + "id": "abc8b4ad-5d96-42cf-9653-08aaeac0034e" + }, + "transaction": { + "id": "3098796995e24283" + }, + "url": { + "domain": "localhost", + "path": "/internal/bsearch", + "port": 5601, + "query": "compress=true", + "scheme": "http" + }, + "user": { + "name": "elastic", + "roles": [ + "superuser" + ] + } + }, + { + "@timestamp": "2022-06-29T12:05:08.187+00:00", + "data_stream": { + "dataset": "kibana-audit-log", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "action": "http_request", + "category": [ + "web" + ], + "dataset": "kibana-audit-log", + "outcome": "unknown" + }, + "http": { + "request": { + "method": "post" + } + }, + "kibana": { + "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", + "space_id": "default" + }, + "log": { + "level": "INFO", + "logger": "plugins.security.audit.ecs" + }, + "message": "User is requesting [/api/log_entries/summary] endpoint", + "process": { + "pid": 7 + }, + "trace": { + "id": "f4181218-b2d3-480e-b9da-78aef88683ff" + }, + "transaction": { + "id": "c1480039d6e6e321" + }, + "url": { + "domain": "localhost", + "path": "/api/log_entries/summary", + "port": 5601, + "scheme": "http" + }, + "user": { + "name": "elastic", + "roles": [ + "superuser" + ] + } + } + ] +} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs b/packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs new file mode 100644 index 00000000000..94643239588 --- /dev/null +++ b/packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} +{{#if processors}} +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..ab58b655a50 --- /dev/null +++ b/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,47 @@ +--- +description: + This data stream is meant for routing only, we want to avoid that data is written to it. + We'll use the dataset that is specified in the ECS JSON log message, or use 'generic' as the default. +processors: + - remove: + field: data_stream.dataset + ignore_missing: true + - remove: + field: event.dataset + ignore_missing: true + - pipeline: + name: '{{ IngestPipeline "kibana-audit-logs-ecs" }}' + if: |- + def message = ctx.message; + return message != null + && message.startsWith('{') + && message.endsWith('}') + && message.contains('"@timestamp"') + - set: + description: Uses event.dataset as a default for data_stream.dataset if the latter is not set. + field: data_stream.dataset + copy_from: event.dataset + if: ctx.event?.dataset instanceof String && ctx.event.dataset.length() > 1 + override: false + - script: + source: | + ctx.data_stream.dataset = /[\/*?"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_') + if: ctx.data_stream?.dataset != null + - script: + source: | + ctx.data_stream.namespace = /[\/*?"<>|, #:]/.matcher(ctx.data_stream.namespace).replaceAll('_') + if: ctx.data_stream?.namespace != null + - set: + field: data_stream.type + value: logs + - set: + field: data_stream.dataset + value: kibana-audit-log + override: false + - set: + field: data_stream.namespace + value: platform-observability + override: false + - set: + field: event.dataset + copy_from: data_stream.dataset diff --git a/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/kibana-audit-logs-ecs.yml b/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/kibana-audit-logs-ecs.yml new file mode 100644 index 00000000000..83e8479dbce --- /dev/null +++ b/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/kibana-audit-logs-ecs.yml @@ -0,0 +1,32 @@ +--- +description: Pipeline for Kibana Audit Logs +processors: + - rename: + field: message + target_field: _ecs_json_message + ignore_missing: true + - json: + field: _ecs_json_message + add_to_root: true + add_to_root_conflict_strategy: merge + allow_duplicate_keys: true + if: ctx.containsKey('_ecs_json_message') + on_failure: + - rename: + field: _ecs_json_message + target_field: message + ignore_missing: true + - set: + field: error.message + value: Error while parsing JSON + override: false + - remove: + field: _ecs_json_message + ignore_missing: true + - dot_expander: + field: "*" + override: true + - join: + field: error.stack_trace + separator: "\n" + if: ctx.error?.stack_trace instanceof Collection diff --git a/packages/platform-observability/data_stream/audit/fields/base-fields.yml b/packages/platform-observability/data_stream/audit/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/platform-observability/data_stream/audit/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/platform-observability/data_stream/audit/fields/ecs.yml b/packages/platform-observability/data_stream/audit/fields/ecs.yml new file mode 100644 index 00000000000..63fdfbd725d --- /dev/null +++ b/packages/platform-observability/data_stream/audit/fields/ecs.yml @@ -0,0 +1,34 @@ +- name: ecs.version + external: ecs +- name: http.request.method + external: ecs +- name: kibana.session_id + external: ecs +- name: kibana.space_id + external: ecs +- name: log.level + external: ecs +- name: log.logger + external: ecs +- name: message + external: ecs +- name: process.pid + external: ecs +- name: trace.id + external: ecs +- name: transaction.id + external: ecs +- name: url.domain + external: ecs +- name: url.path + external: ecs +- name: url.port + external: ecs +- name: url.query + external: ecs +- name: url.scheme + external: ecs +- name: user.name + external: ecs +- name: user.roles + external: ecs diff --git a/packages/platform-observability/data_stream/audit/manifest.yml b/packages/platform-observability/data_stream/audit/manifest.yml new file mode 100644 index 00000000000..03092971721 --- /dev/null +++ b/packages/platform-observability/data_stream/audit/manifest.yml @@ -0,0 +1,17 @@ +type: logs +title: Platform Observability Kibana Audit Logs +release: experimental +streams: + - input: logfile + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /tmp/service_logs/audit*.log + template_path: log.yml.hbs + title: Kibana Audit Logs + description: Collect Kibana Audit Logs diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs-config.yml b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs-config.yml new file mode 100644 index 00000000000..2794ccb00fe --- /dev/null +++ b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs-config.yml @@ -0,0 +1,11 @@ +multiline: + first_line_pattern: "^{" +fields: + ecs: + version: "8.0.0" + event: + dataset: ecs_router + data_stream: + type: logs + dataset: ecs_router + namespace: default diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log new file mode 100644 index 00000000000..d50d1e4f8d0 --- /dev/null +++ b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log @@ -0,0 +1,4 @@ +{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Trying to authenticate user request to /login.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} +{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Authorization header is not presented.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} +{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Could not handle authentication attempt","log":{"level":"DEBUG","logger":"plugins.security.authentication"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} +{"client":{"ip":"127.0.0.1"},"http":{"request":{"method":"GET","mime_type":null,"referrer":"","headers":{"host":"127.0.0.1:5601","user-agent":"curl/7.68.0","accept":"*/*"}},"response":{"body":{"bytes":84749},"status_code":200,"headers":{"content-security-policy":"script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'","x-content-type-options":"nosniff","referrer-policy":"no-referrer-when-downgrade","kbn-name":"kibana","kbn-license-sig":"fc1dbd9f3decaa38ee166c07aa16570255c36c01fd35842394ebec327e175722","content-type":"text/html; charset=utf-8","cache-control":"private, no-cache, no-store, must-revalidate","content-length":84749,"vary":"accept-encoding","accept-ranges":"bytes"},"responseTime":8}},"url":{"path":"/login","query":""},"user_agent":{"original":"curl/7.68.0"},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.905+00:00","message":"GET /login 200 8ms - 82.8KB","log":{"level":"DEBUG","logger":"http.server.response"},"process":{"pid":7},"transaction":{"id":"3be6994d7f6d5465"}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json new file mode 100644 index 00000000000..0009b135cc9 --- /dev/null +++ b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json @@ -0,0 +1,157 @@ +{ + "expected": [ + { + "@timestamp": "2022-06-29T11:24:17.898+00:00", + "data_stream": { + "dataset": "kibana", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana" + }, + "log": { + "level": "DEBUG", + "logger": "plugins.security.http" + }, + "message": "Trying to authenticate user request to /login.", + "process": { + "pid": 7 + }, + "trace": { + "id": "e6e1c25936546ec690b11a3b78b2a8db" + }, + "transaction": { + "id": "3be6994d7f6d5465" + } + }, + { + "@timestamp": "2022-06-29T11:24:17.898+00:00", + "data_stream": { + "dataset": "kibana", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana" + }, + "log": { + "level": "DEBUG", + "logger": "plugins.security.http" + }, + "message": "Authorization header is not presented.", + "process": { + "pid": 7 + }, + "trace": { + "id": "e6e1c25936546ec690b11a3b78b2a8db" + }, + "transaction": { + "id": "3be6994d7f6d5465" + } + }, + { + "@timestamp": "2022-06-29T11:24:17.898+00:00", + "data_stream": { + "dataset": "kibana", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana" + }, + "log": { + "level": "DEBUG", + "logger": "plugins.security.authentication" + }, + "message": "Could not handle authentication attempt", + "process": { + "pid": 7 + }, + "trace": { + "id": "e6e1c25936546ec690b11a3b78b2a8db" + }, + "transaction": { + "id": "3be6994d7f6d5465" + } + }, + { + "@timestamp": "2022-06-29T11:24:17.905+00:00", + "client": { + "ip": "127.0.0.1" + }, + "data_stream": { + "dataset": "kibana", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana" + }, + "http": { + "request": { + "headers": { + "accept": "*/*", + "host": "127.0.0.1:5601", + "user-agent": "curl/7.68.0" + }, + "method": "GET", + "mime_type": null, + "referrer": "" + }, + "response": { + "body": { + "bytes": 84749 + }, + "headers": { + "accept-ranges": "bytes", + "cache-control": "private, no-cache, no-store, must-revalidate", + "content-length": 84749, + "content-security-policy": "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", + "content-type": "text/html; charset=utf-8", + "kbn-license-sig": "fc1dbd9f3decaa38ee166c07aa16570255c36c01fd35842394ebec327e175722", + "kbn-name": "kibana", + "referrer-policy": "no-referrer-when-downgrade", + "vary": "accept-encoding", + "x-content-type-options": "nosniff" + }, + "responseTime": 8, + "status_code": 200 + } + }, + "log": { + "level": "DEBUG", + "logger": "http.server.response" + }, + "message": "GET /login 200 8ms - 82.8KB", + "process": { + "pid": 7 + }, + "trace": { + "id": "e6e1c25936546ec690b11a3b78b2a8db" + }, + "transaction": { + "id": "3be6994d7f6d5465" + }, + "url": { + "path": "/login", + "query": "" + }, + "user_agent": { + "original": "curl/7.68.0" + } + } + ] +} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/log/agent/stream/log.yml.hbs b/packages/platform-observability/data_stream/log/agent/stream/log.yml.hbs new file mode 100644 index 00000000000..94643239588 --- /dev/null +++ b/packages/platform-observability/data_stream/log/agent/stream/log.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} +{{#if processors}} +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..f6bbe5e0e62 --- /dev/null +++ b/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,47 @@ +--- +description: + This data stream is meant for routing only, we want to avoid that data is written to it. + We'll use the dataset that is specified in the ECS JSON log message, or use 'generic' as the default. +processors: + - remove: + field: data_stream.dataset + ignore_missing: true + - remove: + field: event.dataset + ignore_missing: true + - pipeline: + name: '{{ IngestPipeline "kibana-logs-ecs" }}' + if: |- + def message = ctx.message; + return message != null + && message.startsWith('{') + && message.endsWith('}') + && message.contains('"@timestamp"') + - set: + description: Uses event.dataset as a default for data_stream.dataset if the latter is not set. + field: data_stream.dataset + copy_from: event.dataset + if: ctx.event?.dataset instanceof String && ctx.event.dataset.length() > 1 + override: false + - script: + source: | + ctx.data_stream.dataset = /[\/*?"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_') + if: ctx.data_stream?.dataset != null + - script: + source: | + ctx.data_stream.namespace = /[\/*?"<>|, #:]/.matcher(ctx.data_stream.namespace).replaceAll('_') + if: ctx.data_stream?.namespace != null + - set: + field: data_stream.type + value: logs + - set: + field: data_stream.dataset + value: kibana-logs + override: false + - set: + field: data_stream.namespace + value: platform-observability + override: false + - set: + field: event.dataset + copy_from: data_stream.dataset diff --git a/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/kibana-logs-ecs.yml b/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/kibana-logs-ecs.yml new file mode 100644 index 00000000000..9a57ead28d0 --- /dev/null +++ b/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/kibana-logs-ecs.yml @@ -0,0 +1,32 @@ +--- +description: Pipeline for Kibana Logs +processors: + - rename: + field: message + target_field: _ecs_json_message + ignore_missing: true + - json: + field: _ecs_json_message + add_to_root: true + add_to_root_conflict_strategy: merge + allow_duplicate_keys: true + if: ctx.containsKey('_ecs_json_message') + on_failure: + - rename: + field: _ecs_json_message + target_field: message + ignore_missing: true + - set: + field: error.message + value: Error while parsing JSON + override: false + - remove: + field: _ecs_json_message + ignore_missing: true + - dot_expander: + field: "*" + override: true + - join: + field: error.stack_trace + separator: "\n" + if: ctx.error?.stack_trace instanceof Collection diff --git a/packages/platform-observability/data_stream/log/fields/base-fields.yml b/packages/platform-observability/data_stream/log/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/platform-observability/data_stream/log/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/platform-observability/data_stream/log/fields/ecs.yml b/packages/platform-observability/data_stream/log/fields/ecs.yml new file mode 100644 index 00000000000..ed12d6ec32c --- /dev/null +++ b/packages/platform-observability/data_stream/log/fields/ecs.yml @@ -0,0 +1,62 @@ +- name: client.ip + external: ecs +- name: log.level + external: ecs +- name: log.logger + external: ecs +- name: ecs.version + external: ecs +- name: http.request.headers.accept + external: ecs +- name: http.request.headers.host + external: ecs +- name: http.request.headers.user-agent + external: ecs +- name: http.request.mime_type + external: ecs +- name: http.request.method + external: ecs +- name: http.request.referrer + external: ecs +- name: http.response.status_code + external: ecs +- name: http.response.body.bytes + external: ecs +- name: http.response.headers.accept-ranges + external: ecs +- name: http.response.headers.cache-control + external: ecs +- name: http.response.headers.content-length + external: ecs +- name: http.response.headers.content-security-policy + external: ecs +- name: http.response.headers.content-type + external: ecs +- name: http.response.headers.kbn-license-sig + external: ecs +- name: http.response.headers.kbn-name + external: ecs +- name: http.response.headers.referrer-policy + external: ecs +- name: http.response.headers.vary + external: ecs +- name: http.response.headers.x-content-type-options + external: ecs +- name: http.response.responseTime + external: ecs +- name: http.response.headers.referrer-policy + external: ecs +- name: message + external: ecs +- name: process.pid + external: ecs +- name: trace.id + external: ecs +- name: transaction.id + external: ecs +- name: user_agent.original + external: ecs +- name: url.path + external: ecs +- name: url.query + external: ecs diff --git a/packages/platform-observability/data_stream/log/manifest.yml b/packages/platform-observability/data_stream/log/manifest.yml new file mode 100644 index 00000000000..38b2117adf5 --- /dev/null +++ b/packages/platform-observability/data_stream/log/manifest.yml @@ -0,0 +1,17 @@ +type: logs +title: Platform Observability Kibana Logs +release: experimental +streams: + - input: logfile + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /tmp/service_logs/kibana.log + template_path: log.yml.hbs + title: Kibana logs + description: Collect Kibana Logs diff --git a/packages/platform-observability/docs/README.md b/packages/platform-observability/docs/README.md new file mode 100644 index 00000000000..af8ebc4599e --- /dev/null +++ b/packages/platform-observability/docs/README.md @@ -0,0 +1,7 @@ +# Platform Observability Logs + +V3 + +This is a new integration created using the [elastic-package](https://github.com/elastic/elastic-package) tool. + +Consider using the README template file `_dev/build/docs/README.md`to generate a list of exported fields or include a sample event. diff --git a/packages/platform-observability/img/logo_kibana.svg b/packages/platform-observability/img/logo_kibana.svg new file mode 100644 index 00000000000..bafebd9368c --- /dev/null +++ b/packages/platform-observability/img/logo_kibana.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/packages/platform-observability/manifest.yml b/packages/platform-observability/manifest.yml new file mode 100644 index 00000000000..ad42e2c56dd --- /dev/null +++ b/packages/platform-observability/manifest.yml @@ -0,0 +1,26 @@ +format_version: 1.0.0 +name: platform_observability +title: "Platform Observability" +version: 0.0.5 +license: basic +release: experimental +description: "This is a PoC for ingesting ECS formatted logs of one stack component" +type: integration +icons: + - src: /img/logo_kibana.svg + title: logo kibana + size: 32x32 + type: image/svg+xml +categories: ["elastic_stack"] +conditions: + kibana.version: "^8.2.0" +policy_templates: + - name: stack-monitoring-kibana-logs + title: Kibana logs and metrics + description: Collect logs and metrics from Kibana instances + inputs: + - type: logfile + title: "Collect Kibana logs" + description: "Collecting audit and application logs from Kibana instances" +owner: + github: elastic/infra-monitoring-ui From f91647a8bfe5f627d41c66842a30496ea33feda5 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 29 Jun 2022 14:31:39 +0200 Subject: [PATCH 02/18] changelog fix --- packages/platform-observability/changelog.yml | 20 ------------------- packages/platform-observability/manifest.yml | 2 +- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/packages/platform-observability/changelog.yml b/packages/platform-observability/changelog.yml index 0b18f4f6f4a..e00f8813359 100644 --- a/packages/platform-observability/changelog.yml +++ b/packages/platform-observability/changelog.yml @@ -1,24 +1,4 @@ # newer versions go on top -- version: "0.0.5" - changes: - - description: Initial draft of the package - type: enhancement - link: https://github.com/elastic/integrations/pull/1 -- version: "0.0.4" - changes: - - description: Initial draft of the package - type: enhancement - link: https://github.com/elastic/integrations/pull/1 -- version: "0.0.3" - changes: - - description: Initial draft of the package - type: enhancement - link: https://github.com/elastic/integrations/pull/1 -- version: "0.0.2" - changes: - - description: Initial draft of the package - type: enhancement - link: https://github.com/elastic/integrations/pull/1 - version: "0.0.1" changes: - description: Initial draft of the package diff --git a/packages/platform-observability/manifest.yml b/packages/platform-observability/manifest.yml index ad42e2c56dd..3aaa178a52f 100644 --- a/packages/platform-observability/manifest.yml +++ b/packages/platform-observability/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: platform_observability title: "Platform Observability" -version: 0.0.5 +version: 0.0.1 license: basic release: experimental description: "This is a PoC for ingesting ECS formatted logs of one stack component" From 2da545be068f48b362154b47f19662f4eb2a4b9c Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Thu, 30 Jun 2022 10:35:31 +0200 Subject: [PATCH 03/18] update test expected result --- .../pipeline/test-kibana-logs.log-expected.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json index 0009b135cc9..a644a3dd1ba 100644 --- a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json +++ b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json @@ -3,7 +3,7 @@ { "@timestamp": "2022-06-29T11:24:17.898+00:00", "data_stream": { - "dataset": "kibana", + "dataset": "kibana-logs", "namespace": "platform-observability", "type": "logs" }, @@ -11,7 +11,7 @@ "version": "8.0.0" }, "event": { - "dataset": "kibana" + "dataset": "kibana-logs" }, "log": { "level": "DEBUG", @@ -31,7 +31,7 @@ { "@timestamp": "2022-06-29T11:24:17.898+00:00", "data_stream": { - "dataset": "kibana", + "dataset": "kibana-logs", "namespace": "platform-observability", "type": "logs" }, @@ -39,7 +39,7 @@ "version": "8.0.0" }, "event": { - "dataset": "kibana" + "dataset": "kibana-logs" }, "log": { "level": "DEBUG", @@ -59,7 +59,7 @@ { "@timestamp": "2022-06-29T11:24:17.898+00:00", "data_stream": { - "dataset": "kibana", + "dataset": "kibana-logs", "namespace": "platform-observability", "type": "logs" }, @@ -67,7 +67,7 @@ "version": "8.0.0" }, "event": { - "dataset": "kibana" + "dataset": "kibana-logs" }, "log": { "level": "DEBUG", @@ -90,7 +90,7 @@ "ip": "127.0.0.1" }, "data_stream": { - "dataset": "kibana", + "dataset": "kibana-logs", "namespace": "platform-observability", "type": "logs" }, @@ -98,7 +98,7 @@ "version": "8.0.0" }, "event": { - "dataset": "kibana" + "dataset": "kibana-logs" }, "http": { "request": { From a8e4920b0189678a188635e42857639f208f0bb1 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 4 Jul 2022 15:00:37 +0200 Subject: [PATCH 04/18] adds a filebeat processor alternative --- .../test-kibana-audit-logs-config.yml | 11 -- .../test/pipeline/test-kibana-audit-logs.log | 3 +- .../test-kibana-audit-logs.log-expected.json | 149 ++---------------- .../audit/agent/stream/log.yml.hbs | 10 ++ .../elasticsearch/ingest_pipeline/default.yml | 8 - .../ingest_pipeline/kibana-audit-logs-ecs.yml | 32 ---- .../data_stream/audit/fields/base-fields.yml | 2 +- .../data_stream/audit/fields/ecs.yml | 32 ---- .../test/pipeline/test-kibana-logs-config.yml | 11 -- .../_dev/test/pipeline/test-kibana-logs.log | 3 +- .../test-kibana-logs.log-expected.json | 28 ++++ .../data_stream/log/agent/stream/log.yml.hbs | 1 + .../elasticsearch/ingest_pipeline/default.yml | 42 ++++- .../ingest_pipeline/kibana-logs-ecs.yml | 32 ---- .../data_stream/log/fields/base-fields.yml | 2 +- 15 files changed, 94 insertions(+), 272 deletions(-) delete mode 100644 packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs-config.yml delete mode 100644 packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/kibana-audit-logs-ecs.yml delete mode 100644 packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs-config.yml delete mode 100644 packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/kibana-logs-ecs.yml diff --git a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs-config.yml b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs-config.yml deleted file mode 100644 index 2794ccb00fe..00000000000 --- a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs-config.yml +++ /dev/null @@ -1,11 +0,0 @@ -multiline: - first_line_pattern: "^{" -fields: - ecs: - version: "8.0.0" - event: - dataset: ecs_router - data_stream: - type: logs - dataset: ecs_router - namespace: default diff --git a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log index 95080c52454..605259fae86 100644 --- a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log +++ b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log @@ -1,3 +1,4 @@ {"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"localhost","path":"/internal/security/session","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"1c8c5808-d2d6-41fc-8cb7-998aa8996be9"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:03.742+00:00","message":"User is requesting [/internal/security/session] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"f8863d86567119e6"}} {"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/internal/bsearch","port":5601,"query":"compress=true","scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"abc8b4ad-5d96-42cf-9653-08aaeac0034e"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:08.178+00:00","message":"User is requesting [/internal/bsearch] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"3098796995e24283"}} -{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/api/log_entries/summary","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"f4181218-b2d3-480e-b9da-78aef88683ff"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:08.187+00:00","message":"User is requesting [/api/log_entries/summary] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"c1480039d6e6e321"}} \ No newline at end of file +{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/api/log_entries/summary","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"f4181218-b2d3-480e-b9da-78aef88683ff"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:08.187+00:00","message":"User is requesting [/api/log_entries/summary] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"c1480039d6e6e321"}} +{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"localhost","path":"/internal/security/session","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"83d80454-6b8a-4727-91ba-22e6ab27e476"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T13:14:30.142+00:00","message":"User is requesting [/internal/security/session] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"c1f684ff5fcf7eaf"}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json index ccfb81549e3..7f19c39b4c1 100644 --- a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json +++ b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json @@ -1,167 +1,48 @@ { "expected": [ { - "@timestamp": "2022-06-29T12:05:03.742+00:00", "data_stream": { "dataset": "kibana-audit-log", "namespace": "platform-observability", "type": "logs" }, - "ecs": { - "version": "8.0.0" - }, "event": { - "action": "http_request", - "category": [ - "web" - ], - "dataset": "kibana-audit-log", - "outcome": "unknown" - }, - "http": { - "request": { - "method": "get" - } - }, - "kibana": { - "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", - "space_id": "default" - }, - "log": { - "level": "INFO", - "logger": "plugins.security.audit.ecs" - }, - "message": "User is requesting [/internal/security/session] endpoint", - "process": { - "pid": 7 - }, - "trace": { - "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9" - }, - "transaction": { - "id": "f8863d86567119e6" + "dataset": "kibana-audit-log" }, - "url": { - "domain": "localhost", - "path": "/internal/security/session", - "port": 5601, - "scheme": "http" - }, - "user": { - "name": "elastic", - "roles": [ - "superuser" - ] - } + "message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"get\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/internal/security/session\",\"port\":5601,\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"1c8c5808-d2d6-41fc-8cb7-998aa8996be9\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T12:05:03.742+00:00\",\"message\":\"User is requesting [/internal/security/session] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"f8863d86567119e6\"}}" }, { - "@timestamp": "2022-06-29T12:05:08.178+00:00", "data_stream": { "dataset": "kibana-audit-log", "namespace": "platform-observability", "type": "logs" }, - "ecs": { - "version": "8.0.0" - }, "event": { - "action": "http_request", - "category": [ - "web" - ], - "dataset": "kibana-audit-log", - "outcome": "unknown" - }, - "http": { - "request": { - "method": "post" - } - }, - "kibana": { - "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", - "space_id": "default" - }, - "log": { - "level": "INFO", - "logger": "plugins.security.audit.ecs" - }, - "message": "User is requesting [/internal/bsearch] endpoint", - "process": { - "pid": 7 - }, - "trace": { - "id": "abc8b4ad-5d96-42cf-9653-08aaeac0034e" + "dataset": "kibana-audit-log" }, - "transaction": { - "id": "3098796995e24283" - }, - "url": { - "domain": "localhost", - "path": "/internal/bsearch", - "port": 5601, - "query": "compress=true", - "scheme": "http" - }, - "user": { - "name": "elastic", - "roles": [ - "superuser" - ] - } + "message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"post\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/internal/bsearch\",\"port\":5601,\"query\":\"compress=true\",\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"abc8b4ad-5d96-42cf-9653-08aaeac0034e\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T12:05:08.178+00:00\",\"message\":\"User is requesting [/internal/bsearch] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"3098796995e24283\"}}" }, { - "@timestamp": "2022-06-29T12:05:08.187+00:00", "data_stream": { "dataset": "kibana-audit-log", "namespace": "platform-observability", "type": "logs" }, - "ecs": { - "version": "8.0.0" - }, "event": { - "action": "http_request", - "category": [ - "web" - ], - "dataset": "kibana-audit-log", - "outcome": "unknown" - }, - "http": { - "request": { - "method": "post" - } - }, - "kibana": { - "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", - "space_id": "default" - }, - "log": { - "level": "INFO", - "logger": "plugins.security.audit.ecs" - }, - "message": "User is requesting [/api/log_entries/summary] endpoint", - "process": { - "pid": 7 + "dataset": "kibana-audit-log" }, - "trace": { - "id": "f4181218-b2d3-480e-b9da-78aef88683ff" - }, - "transaction": { - "id": "c1480039d6e6e321" + "message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"post\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/api/log_entries/summary\",\"port\":5601,\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"f4181218-b2d3-480e-b9da-78aef88683ff\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T12:05:08.187+00:00\",\"message\":\"User is requesting [/api/log_entries/summary] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"c1480039d6e6e321\"}}" + }, + { + "data_stream": { + "dataset": "kibana-audit-log", + "namespace": "platform-observability", + "type": "logs" }, - "url": { - "domain": "localhost", - "path": "/api/log_entries/summary", - "port": 5601, - "scheme": "http" + "event": { + "dataset": "kibana-audit-log" }, - "user": { - "name": "elastic", - "roles": [ - "superuser" - ] - } + "message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"get\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/internal/security/session\",\"port\":5601,\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"83d80454-6b8a-4727-91ba-22e6ab27e476\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T13:14:30.142+00:00\",\"message\":\"User is requesting [/internal/security/session] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"c1f684ff5fcf7eaf\"}}" } ] } \ No newline at end of file diff --git a/packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs b/packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs index 94643239588..3ff81290f0d 100644 --- a/packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs +++ b/packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs @@ -2,6 +2,16 @@ paths: {{#each paths}} - {{this}} {{/each}} +processors: +- rename: + fields: + - from: "message" + to: "event.original" +- decode_json_fields: + fields: [event.original] + target: "" + add_error_key: true + overwrite_keys: true {{#if processors}} {{processors}} {{/if}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index ab58b655a50..acea456d5c8 100644 --- a/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -9,14 +9,6 @@ processors: - remove: field: event.dataset ignore_missing: true - - pipeline: - name: '{{ IngestPipeline "kibana-audit-logs-ecs" }}' - if: |- - def message = ctx.message; - return message != null - && message.startsWith('{') - && message.endsWith('}') - && message.contains('"@timestamp"') - set: description: Uses event.dataset as a default for data_stream.dataset if the latter is not set. field: data_stream.dataset diff --git a/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/kibana-audit-logs-ecs.yml b/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/kibana-audit-logs-ecs.yml deleted file mode 100644 index 83e8479dbce..00000000000 --- a/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/kibana-audit-logs-ecs.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -description: Pipeline for Kibana Audit Logs -processors: - - rename: - field: message - target_field: _ecs_json_message - ignore_missing: true - - json: - field: _ecs_json_message - add_to_root: true - add_to_root_conflict_strategy: merge - allow_duplicate_keys: true - if: ctx.containsKey('_ecs_json_message') - on_failure: - - rename: - field: _ecs_json_message - target_field: message - ignore_missing: true - - set: - field: error.message - value: Error while parsing JSON - override: false - - remove: - field: _ecs_json_message - ignore_missing: true - - dot_expander: - field: "*" - override: true - - join: - field: error.stack_trace - separator: "\n" - if: ctx.error?.stack_trace instanceof Collection diff --git a/packages/platform-observability/data_stream/audit/fields/base-fields.yml b/packages/platform-observability/data_stream/audit/fields/base-fields.yml index 7c798f4534c..0d1791ffed6 100644 --- a/packages/platform-observability/data_stream/audit/fields/base-fields.yml +++ b/packages/platform-observability/data_stream/audit/fields/base-fields.yml @@ -7,6 +7,6 @@ - name: data_stream.namespace type: constant_keyword description: Data stream namespace. -- name: '@timestamp' +- name: "@timestamp" type: date description: Event timestamp. diff --git a/packages/platform-observability/data_stream/audit/fields/ecs.yml b/packages/platform-observability/data_stream/audit/fields/ecs.yml index 63fdfbd725d..9bbe4c84e14 100644 --- a/packages/platform-observability/data_stream/audit/fields/ecs.yml +++ b/packages/platform-observability/data_stream/audit/fields/ecs.yml @@ -1,34 +1,2 @@ -- name: ecs.version - external: ecs -- name: http.request.method - external: ecs -- name: kibana.session_id - external: ecs -- name: kibana.space_id - external: ecs -- name: log.level - external: ecs -- name: log.logger - external: ecs - name: message external: ecs -- name: process.pid - external: ecs -- name: trace.id - external: ecs -- name: transaction.id - external: ecs -- name: url.domain - external: ecs -- name: url.path - external: ecs -- name: url.port - external: ecs -- name: url.query - external: ecs -- name: url.scheme - external: ecs -- name: user.name - external: ecs -- name: user.roles - external: ecs diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs-config.yml b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs-config.yml deleted file mode 100644 index 2794ccb00fe..00000000000 --- a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs-config.yml +++ /dev/null @@ -1,11 +0,0 @@ -multiline: - first_line_pattern: "^{" -fields: - ecs: - version: "8.0.0" - event: - dataset: ecs_router - data_stream: - type: logs - dataset: ecs_router - namespace: default diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log index d50d1e4f8d0..3a5573f97a6 100644 --- a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log +++ b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log @@ -1,4 +1,5 @@ {"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Trying to authenticate user request to /login.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} {"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Authorization header is not presented.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} {"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Could not handle authentication attempt","log":{"level":"DEBUG","logger":"plugins.security.authentication"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} -{"client":{"ip":"127.0.0.1"},"http":{"request":{"method":"GET","mime_type":null,"referrer":"","headers":{"host":"127.0.0.1:5601","user-agent":"curl/7.68.0","accept":"*/*"}},"response":{"body":{"bytes":84749},"status_code":200,"headers":{"content-security-policy":"script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'","x-content-type-options":"nosniff","referrer-policy":"no-referrer-when-downgrade","kbn-name":"kibana","kbn-license-sig":"fc1dbd9f3decaa38ee166c07aa16570255c36c01fd35842394ebec327e175722","content-type":"text/html; charset=utf-8","cache-control":"private, no-cache, no-store, must-revalidate","content-length":84749,"vary":"accept-encoding","accept-ranges":"bytes"},"responseTime":8}},"url":{"path":"/login","query":""},"user_agent":{"original":"curl/7.68.0"},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.905+00:00","message":"GET /login 200 8ms - 82.8KB","log":{"level":"DEBUG","logger":"http.server.response"},"process":{"pid":7},"transaction":{"id":"3be6994d7f6d5465"}} \ No newline at end of file +{"client":{"ip":"127.0.0.1"},"http":{"request":{"method":"GET","mime_type":null,"referrer":"","headers":{"host":"127.0.0.1:5601","user-agent":"curl/7.68.0","accept":"*/*"}},"response":{"body":{"bytes":84749},"status_code":200,"headers":{"content-security-policy":"script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'","x-content-type-options":"nosniff","referrer-policy":"no-referrer-when-downgrade","kbn-name":"kibana","kbn-license-sig":"fc1dbd9f3decaa38ee166c07aa16570255c36c01fd35842394ebec327e175722","content-type":"text/html; charset=utf-8","cache-control":"private, no-cache, no-store, must-revalidate","content-length":84749,"vary":"accept-encoding","accept-ranges":"bytes"},"responseTime":8}},"url":{"path":"/login","query":""},"user_agent":{"original":"curl/7.68.0"},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.905+00:00","message":"GET /login 200 8ms - 82.8KB","log":{"level":"DEBUG","logger":"http.server.response"},"process":{"pid":7},"transaction":{"id":"3be6994d7f6d5465"}} +{"_tag":"Right","right":"update_aliases_succeeded","ecs":{"version":"8.0.0"},"@timestamp":"2022-07-04T09:17:38.611+00:00","message":"[.kibana] MARK_VERSION_INDEX_READY RESPONSE","log":{"level":"DEBUG","logger":"savedobjects-service"},"process":{"pid":7},"trace":{"id":"a167d1124764379d4121b357e20baee2"},"transaction":{"id":"14717ae6e3b30d5a"}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json index a644a3dd1ba..78386649624 100644 --- a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json +++ b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json @@ -152,6 +152,34 @@ "user_agent": { "original": "curl/7.68.0" } + }, + { + "@timestamp": "2022-07-04T09:17:38.611+00:00", + "data_stream": { + "dataset": "kibana-logs", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana-logs" + }, + "log": { + "level": "DEBUG", + "logger": "savedobjects-service" + }, + "message": "[.kibana] MARK_VERSION_INDEX_READY RESPONSE", + "process": { + "pid": 7 + }, + "trace": { + "id": "a167d1124764379d4121b357e20baee2" + }, + "transaction": { + "id": "14717ae6e3b30d5a" + } } ] } \ No newline at end of file diff --git a/packages/platform-observability/data_stream/log/agent/stream/log.yml.hbs b/packages/platform-observability/data_stream/log/agent/stream/log.yml.hbs index 94643239588..5042f840763 100644 --- a/packages/platform-observability/data_stream/log/agent/stream/log.yml.hbs +++ b/packages/platform-observability/data_stream/log/agent/stream/log.yml.hbs @@ -3,5 +3,6 @@ paths: - {{this}} {{/each}} {{#if processors}} +processors: {{processors}} {{/if}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml index f6bbe5e0e62..06abb2408e8 100644 --- a/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -9,14 +9,6 @@ processors: - remove: field: event.dataset ignore_missing: true - - pipeline: - name: '{{ IngestPipeline "kibana-logs-ecs" }}' - if: |- - def message = ctx.message; - return message != null - && message.startsWith('{') - && message.endsWith('}') - && message.contains('"@timestamp"') - set: description: Uses event.dataset as a default for data_stream.dataset if the latter is not set. field: data_stream.dataset @@ -45,3 +37,37 @@ processors: - set: field: event.dataset copy_from: data_stream.dataset + - rename: + field: message + target_field: _ecs_json_message + if: |- + def message = ctx.message; + return message != null + && message.startsWith('{') + && message.endsWith('}') + && message.contains('"@timestamp"') + ignore_missing: true + - json: + field: _ecs_json_message + add_to_root: true + add_to_root_conflict_strategy: merge + allow_duplicate_keys: true + if: ctx.containsKey('_ecs_json_message') + on_failure: + - rename: + field: _ecs_json_message + target_field: message + ignore_missing: true + - set: + field: error.message + value: Error while parsing JSON + override: false + - remove: + field: _ecs_json_message + ignore_missing: true + - remove: + field: _tag + ignore_missing: true + - remove: + field: right + ignore_missing: true diff --git a/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/kibana-logs-ecs.yml b/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/kibana-logs-ecs.yml deleted file mode 100644 index 9a57ead28d0..00000000000 --- a/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/kibana-logs-ecs.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -description: Pipeline for Kibana Logs -processors: - - rename: - field: message - target_field: _ecs_json_message - ignore_missing: true - - json: - field: _ecs_json_message - add_to_root: true - add_to_root_conflict_strategy: merge - allow_duplicate_keys: true - if: ctx.containsKey('_ecs_json_message') - on_failure: - - rename: - field: _ecs_json_message - target_field: message - ignore_missing: true - - set: - field: error.message - value: Error while parsing JSON - override: false - - remove: - field: _ecs_json_message - ignore_missing: true - - dot_expander: - field: "*" - override: true - - join: - field: error.stack_trace - separator: "\n" - if: ctx.error?.stack_trace instanceof Collection diff --git a/packages/platform-observability/data_stream/log/fields/base-fields.yml b/packages/platform-observability/data_stream/log/fields/base-fields.yml index 7c798f4534c..0d1791ffed6 100644 --- a/packages/platform-observability/data_stream/log/fields/base-fields.yml +++ b/packages/platform-observability/data_stream/log/fields/base-fields.yml @@ -7,6 +7,6 @@ - name: data_stream.namespace type: constant_keyword description: Data stream namespace. -- name: '@timestamp' +- name: "@timestamp" type: date description: Event timestamp. From b171f936ce70d8b57c4afce4a068bdfc479e2863 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 6 Jul 2022 10:20:56 +0200 Subject: [PATCH 05/18] clean up poc code --- .github/CODEOWNERS | 1 + .../elasticsearch/ingest_pipeline/default.yml | 4 +- .../data_stream/audit/fields/ecs.yml | 1 + .../_dev/test/pipeline/test-kibana-logs.log | 1 - .../test-kibana-logs.log-expected.json | 299 +++++++----------- .../elasticsearch/ingest_pipeline/default.yml | 4 +- .../data_stream/log/fields/ecs.yml | 51 +-- 7 files changed, 121 insertions(+), 240 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b934f13eb62..7a28c44ff16 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -168,3 +168,4 @@ /packages/zscaler @elastic/security-external-integrations /packages/zscaler_zia @elastic/security-external-integrations /packages/zscaler_zpa @elastic/security-external-integrations +/packages/platform-observability @elastic/obs-service-integrations @elastic/infra-monitoring-ui diff --git a/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index acea456d5c8..5251f11f89f 100644 --- a/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -1,7 +1,5 @@ --- -description: - This data stream is meant for routing only, we want to avoid that data is written to it. - We'll use the dataset that is specified in the ECS JSON log message, or use 'generic' as the default. +description: Pipeline for parsing ECS logs processors: - remove: field: data_stream.dataset diff --git a/packages/platform-observability/data_stream/audit/fields/ecs.yml b/packages/platform-observability/data_stream/audit/fields/ecs.yml index 9bbe4c84e14..392fcb5f2e1 100644 --- a/packages/platform-observability/data_stream/audit/fields/ecs.yml +++ b/packages/platform-observability/data_stream/audit/fields/ecs.yml @@ -1,2 +1,3 @@ +# only used for tests - name: message external: ecs diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log index 3a5573f97a6..aa08e9c6569 100644 --- a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log +++ b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log @@ -1,5 +1,4 @@ {"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Trying to authenticate user request to /login.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} {"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Authorization header is not presented.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} {"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Could not handle authentication attempt","log":{"level":"DEBUG","logger":"plugins.security.authentication"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} -{"client":{"ip":"127.0.0.1"},"http":{"request":{"method":"GET","mime_type":null,"referrer":"","headers":{"host":"127.0.0.1:5601","user-agent":"curl/7.68.0","accept":"*/*"}},"response":{"body":{"bytes":84749},"status_code":200,"headers":{"content-security-policy":"script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'","x-content-type-options":"nosniff","referrer-policy":"no-referrer-when-downgrade","kbn-name":"kibana","kbn-license-sig":"fc1dbd9f3decaa38ee166c07aa16570255c36c01fd35842394ebec327e175722","content-type":"text/html; charset=utf-8","cache-control":"private, no-cache, no-store, must-revalidate","content-length":84749,"vary":"accept-encoding","accept-ranges":"bytes"},"responseTime":8}},"url":{"path":"/login","query":""},"user_agent":{"original":"curl/7.68.0"},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.905+00:00","message":"GET /login 200 8ms - 82.8KB","log":{"level":"DEBUG","logger":"http.server.response"},"process":{"pid":7},"transaction":{"id":"3be6994d7f6d5465"}} {"_tag":"Right","right":"update_aliases_succeeded","ecs":{"version":"8.0.0"},"@timestamp":"2022-07-04T09:17:38.611+00:00","message":"[.kibana] MARK_VERSION_INDEX_READY RESPONSE","log":{"level":"DEBUG","logger":"savedobjects-service"},"process":{"pid":7},"trace":{"id":"a167d1124764379d4121b357e20baee2"},"transaction":{"id":"14717ae6e3b30d5a"}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json index 78386649624..bd6d794fc56 100644 --- a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json +++ b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json @@ -1,185 +1,116 @@ { - "expected": [ - { - "@timestamp": "2022-06-29T11:24:17.898+00:00", - "data_stream": { - "dataset": "kibana-logs", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "dataset": "kibana-logs" - }, - "log": { - "level": "DEBUG", - "logger": "plugins.security.http" - }, - "message": "Trying to authenticate user request to /login.", - "process": { - "pid": 7 - }, - "trace": { - "id": "e6e1c25936546ec690b11a3b78b2a8db" - }, - "transaction": { - "id": "3be6994d7f6d5465" - } - }, - { - "@timestamp": "2022-06-29T11:24:17.898+00:00", - "data_stream": { - "dataset": "kibana-logs", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "dataset": "kibana-logs" - }, - "log": { - "level": "DEBUG", - "logger": "plugins.security.http" - }, - "message": "Authorization header is not presented.", - "process": { - "pid": 7 - }, - "trace": { - "id": "e6e1c25936546ec690b11a3b78b2a8db" - }, - "transaction": { - "id": "3be6994d7f6d5465" - } - }, - { - "@timestamp": "2022-06-29T11:24:17.898+00:00", - "data_stream": { - "dataset": "kibana-logs", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "dataset": "kibana-logs" - }, - "log": { - "level": "DEBUG", - "logger": "plugins.security.authentication" - }, - "message": "Could not handle authentication attempt", - "process": { - "pid": 7 - }, - "trace": { - "id": "e6e1c25936546ec690b11a3b78b2a8db" - }, - "transaction": { - "id": "3be6994d7f6d5465" - } - }, - { - "@timestamp": "2022-06-29T11:24:17.905+00:00", - "client": { - "ip": "127.0.0.1" - }, - "data_stream": { - "dataset": "kibana-logs", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "dataset": "kibana-logs" - }, - "http": { - "request": { - "headers": { - "accept": "*/*", - "host": "127.0.0.1:5601", - "user-agent": "curl/7.68.0" - }, - "method": "GET", - "mime_type": null, - "referrer": "" - }, - "response": { - "body": { - "bytes": 84749 - }, - "headers": { - "accept-ranges": "bytes", - "cache-control": "private, no-cache, no-store, must-revalidate", - "content-length": 84749, - "content-security-policy": "script-src 'unsafe-eval' 'self'; worker-src blob: 'self'; style-src 'unsafe-inline' 'self'", - "content-type": "text/html; charset=utf-8", - "kbn-license-sig": "fc1dbd9f3decaa38ee166c07aa16570255c36c01fd35842394ebec327e175722", - "kbn-name": "kibana", - "referrer-policy": "no-referrer-when-downgrade", - "vary": "accept-encoding", - "x-content-type-options": "nosniff" - }, - "responseTime": 8, - "status_code": 200 - } - }, - "log": { - "level": "DEBUG", - "logger": "http.server.response" - }, - "message": "GET /login 200 8ms - 82.8KB", - "process": { - "pid": 7 - }, - "trace": { - "id": "e6e1c25936546ec690b11a3b78b2a8db" - }, - "transaction": { - "id": "3be6994d7f6d5465" - }, - "url": { - "path": "/login", - "query": "" - }, - "user_agent": { - "original": "curl/7.68.0" - } - }, - { - "@timestamp": "2022-07-04T09:17:38.611+00:00", - "data_stream": { - "dataset": "kibana-logs", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "dataset": "kibana-logs" - }, - "log": { - "level": "DEBUG", - "logger": "savedobjects-service" - }, - "message": "[.kibana] MARK_VERSION_INDEX_READY RESPONSE", - "process": { - "pid": 7 - }, - "trace": { - "id": "a167d1124764379d4121b357e20baee2" - }, - "transaction": { - "id": "14717ae6e3b30d5a" - } - } - ] -} \ No newline at end of file + "expected": [ + { + "@timestamp": "2022-06-29T11:24:17.898+00:00", + "data_stream": { + "dataset": "kibana-logs", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana-logs" + }, + "log": { + "level": "DEBUG", + "logger": "plugins.security.http" + }, + "message": "Trying to authenticate user request to /login.", + "process": { + "pid": 7 + }, + "trace": { + "id": "e6e1c25936546ec690b11a3b78b2a8db" + }, + "transaction": { + "id": "3be6994d7f6d5465" + } + }, + { + "@timestamp": "2022-06-29T11:24:17.898+00:00", + "data_stream": { + "dataset": "kibana-logs", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana-logs" + }, + "log": { + "level": "DEBUG", + "logger": "plugins.security.http" + }, + "message": "Authorization header is not presented.", + "process": { + "pid": 7 + }, + "trace": { + "id": "e6e1c25936546ec690b11a3b78b2a8db" + }, + "transaction": { + "id": "3be6994d7f6d5465" + } + }, + { + "@timestamp": "2022-06-29T11:24:17.898+00:00", + "data_stream": { + "dataset": "kibana-logs", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana-logs" + }, + "log": { + "level": "DEBUG", + "logger": "plugins.security.authentication" + }, + "message": "Could not handle authentication attempt", + "process": { + "pid": 7 + }, + "trace": { + "id": "e6e1c25936546ec690b11a3b78b2a8db" + }, + "transaction": { + "id": "3be6994d7f6d5465" + } + }, + { + "@timestamp": "2022-07-04T09:17:38.611+00:00", + "data_stream": { + "dataset": "kibana-logs", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana-logs" + }, + "log": { + "level": "DEBUG", + "logger": "savedobjects-service" + }, + "message": "[.kibana] MARK_VERSION_INDEX_READY RESPONSE", + "process": { + "pid": 7 + }, + "trace": { + "id": "a167d1124764379d4121b357e20baee2" + }, + "transaction": { + "id": "14717ae6e3b30d5a" + } + } + ] +} diff --git a/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 06abb2408e8..5ed5392b5ef 100644 --- a/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -1,7 +1,5 @@ --- -description: - This data stream is meant for routing only, we want to avoid that data is written to it. - We'll use the dataset that is specified in the ECS JSON log message, or use 'generic' as the default. +description: Pipeline for parsing ECS logs processors: - remove: field: data_stream.dataset diff --git a/packages/platform-observability/data_stream/log/fields/ecs.yml b/packages/platform-observability/data_stream/log/fields/ecs.yml index ed12d6ec32c..6df80b6f4f7 100644 --- a/packages/platform-observability/data_stream/log/fields/ecs.yml +++ b/packages/platform-observability/data_stream/log/fields/ecs.yml @@ -1,51 +1,10 @@ -- name: client.ip +# only used for tests +- name: ecs.version external: ecs - name: log.level external: ecs - name: log.logger external: ecs -- name: ecs.version - external: ecs -- name: http.request.headers.accept - external: ecs -- name: http.request.headers.host - external: ecs -- name: http.request.headers.user-agent - external: ecs -- name: http.request.mime_type - external: ecs -- name: http.request.method - external: ecs -- name: http.request.referrer - external: ecs -- name: http.response.status_code - external: ecs -- name: http.response.body.bytes - external: ecs -- name: http.response.headers.accept-ranges - external: ecs -- name: http.response.headers.cache-control - external: ecs -- name: http.response.headers.content-length - external: ecs -- name: http.response.headers.content-security-policy - external: ecs -- name: http.response.headers.content-type - external: ecs -- name: http.response.headers.kbn-license-sig - external: ecs -- name: http.response.headers.kbn-name - external: ecs -- name: http.response.headers.referrer-policy - external: ecs -- name: http.response.headers.vary - external: ecs -- name: http.response.headers.x-content-type-options - external: ecs -- name: http.response.responseTime - external: ecs -- name: http.response.headers.referrer-policy - external: ecs - name: message external: ecs - name: process.pid @@ -54,9 +13,3 @@ external: ecs - name: transaction.id external: ecs -- name: user_agent.original - external: ecs -- name: url.path - external: ecs -- name: url.query - external: ecs From d6cacfd624b957574026ef04389f8aa26df5646d Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 6 Jul 2022 12:26:55 +0200 Subject: [PATCH 06/18] fix json identation --- .../test-kibana-logs.log-expected.json | 230 +++++++++--------- 1 file changed, 115 insertions(+), 115 deletions(-) diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json index bd6d794fc56..4d20c4546da 100644 --- a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json +++ b/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json @@ -1,116 +1,116 @@ { - "expected": [ - { - "@timestamp": "2022-06-29T11:24:17.898+00:00", - "data_stream": { - "dataset": "kibana-logs", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "dataset": "kibana-logs" - }, - "log": { - "level": "DEBUG", - "logger": "plugins.security.http" - }, - "message": "Trying to authenticate user request to /login.", - "process": { - "pid": 7 - }, - "trace": { - "id": "e6e1c25936546ec690b11a3b78b2a8db" - }, - "transaction": { - "id": "3be6994d7f6d5465" - } - }, - { - "@timestamp": "2022-06-29T11:24:17.898+00:00", - "data_stream": { - "dataset": "kibana-logs", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "dataset": "kibana-logs" - }, - "log": { - "level": "DEBUG", - "logger": "plugins.security.http" - }, - "message": "Authorization header is not presented.", - "process": { - "pid": 7 - }, - "trace": { - "id": "e6e1c25936546ec690b11a3b78b2a8db" - }, - "transaction": { - "id": "3be6994d7f6d5465" - } - }, - { - "@timestamp": "2022-06-29T11:24:17.898+00:00", - "data_stream": { - "dataset": "kibana-logs", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "dataset": "kibana-logs" - }, - "log": { - "level": "DEBUG", - "logger": "plugins.security.authentication" - }, - "message": "Could not handle authentication attempt", - "process": { - "pid": 7 - }, - "trace": { - "id": "e6e1c25936546ec690b11a3b78b2a8db" - }, - "transaction": { - "id": "3be6994d7f6d5465" - } - }, - { - "@timestamp": "2022-07-04T09:17:38.611+00:00", - "data_stream": { - "dataset": "kibana-logs", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "dataset": "kibana-logs" - }, - "log": { - "level": "DEBUG", - "logger": "savedobjects-service" - }, - "message": "[.kibana] MARK_VERSION_INDEX_READY RESPONSE", - "process": { - "pid": 7 - }, - "trace": { - "id": "a167d1124764379d4121b357e20baee2" - }, - "transaction": { - "id": "14717ae6e3b30d5a" - } - } - ] -} + "expected": [ + { + "@timestamp": "2022-06-29T11:24:17.898+00:00", + "data_stream": { + "dataset": "kibana-logs", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana-logs" + }, + "log": { + "level": "DEBUG", + "logger": "plugins.security.http" + }, + "message": "Trying to authenticate user request to /login.", + "process": { + "pid": 7 + }, + "trace": { + "id": "e6e1c25936546ec690b11a3b78b2a8db" + }, + "transaction": { + "id": "3be6994d7f6d5465" + } + }, + { + "@timestamp": "2022-06-29T11:24:17.898+00:00", + "data_stream": { + "dataset": "kibana-logs", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana-logs" + }, + "log": { + "level": "DEBUG", + "logger": "plugins.security.http" + }, + "message": "Authorization header is not presented.", + "process": { + "pid": 7 + }, + "trace": { + "id": "e6e1c25936546ec690b11a3b78b2a8db" + }, + "transaction": { + "id": "3be6994d7f6d5465" + } + }, + { + "@timestamp": "2022-06-29T11:24:17.898+00:00", + "data_stream": { + "dataset": "kibana-logs", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana-logs" + }, + "log": { + "level": "DEBUG", + "logger": "plugins.security.authentication" + }, + "message": "Could not handle authentication attempt", + "process": { + "pid": 7 + }, + "trace": { + "id": "e6e1c25936546ec690b11a3b78b2a8db" + }, + "transaction": { + "id": "3be6994d7f6d5465" + } + }, + { + "@timestamp": "2022-07-04T09:17:38.611+00:00", + "data_stream": { + "dataset": "kibana-logs", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "dataset": "kibana-logs" + }, + "log": { + "level": "DEBUG", + "logger": "savedobjects-service" + }, + "message": "[.kibana] MARK_VERSION_INDEX_READY RESPONSE", + "process": { + "pid": 7 + }, + "trace": { + "id": "a167d1124764379d4121b357e20baee2" + }, + "transaction": { + "id": "14717ae6e3b30d5a" + } + } + ] +} \ No newline at end of file From a3bd0c0ae460445fa730ca87811666e88ce189be Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 12 Jul 2022 13:18:59 +0200 Subject: [PATCH 07/18] Align ingest pipeline approach and improve README.md --- .github/CODEOWNERS | 2 +- .../test-kibana-audit-logs.log-expected.json | 48 ---- .../audit/agent/stream/log.yml.hbs | 17 -- .../data_stream/audit/fields/ecs.yml | 3 - .../test/pipeline/test-kibana-audit-logs.log | 0 .../test-kibana-audit-logs.log-expected.json | 221 ++++++++++++++++++ .../agent/stream/log.yml.hbs | 0 .../elasticsearch/ingest_pipeline/default.yml | 34 ++- .../fields/base-fields.yml | 0 .../data_stream/kibana_audit/fields/ecs.yml | 35 +++ .../{audit => kibana_audit}/manifest.yml | 4 +- .../_dev/test/pipeline/test-kibana-logs.log | 0 .../test-kibana-logs.log-expected.json | 0 .../kibana_log/agent/stream/log.yml.hbs | 8 + .../elasticsearch/ingest_pipeline/default.yml | 6 +- .../fields/base-fields.yml | 0 .../{log => kibana_log}/fields/ecs.yml | 0 .../{log => kibana_log}/manifest.yml | 4 +- .../platform-observability/docs/README.md | 27 ++- packages/platform-observability/manifest.yml | 10 +- 20 files changed, 335 insertions(+), 84 deletions(-) delete mode 100644 packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json delete mode 100644 packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs delete mode 100644 packages/platform-observability/data_stream/audit/fields/ecs.yml rename packages/platform-observability/data_stream/{audit => kibana_audit}/_dev/test/pipeline/test-kibana-audit-logs.log (100%) create mode 100644 packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json rename packages/platform-observability/data_stream/{log => kibana_audit}/agent/stream/log.yml.hbs (100%) rename packages/platform-observability/data_stream/{audit => kibana_audit}/elasticsearch/ingest_pipeline/default.yml (52%) rename packages/platform-observability/data_stream/{audit => kibana_audit}/fields/base-fields.yml (100%) create mode 100644 packages/platform-observability/data_stream/kibana_audit/fields/ecs.yml rename packages/platform-observability/data_stream/{audit => kibana_audit}/manifest.yml (89%) rename packages/platform-observability/data_stream/{log => kibana_log}/_dev/test/pipeline/test-kibana-logs.log (100%) rename packages/platform-observability/data_stream/{log => kibana_log}/_dev/test/pipeline/test-kibana-logs.log-expected.json (100%) create mode 100644 packages/platform-observability/data_stream/kibana_log/agent/stream/log.yml.hbs rename packages/platform-observability/data_stream/{log => kibana_log}/elasticsearch/ingest_pipeline/default.yml (92%) rename packages/platform-observability/data_stream/{log => kibana_log}/fields/base-fields.yml (100%) rename packages/platform-observability/data_stream/{log => kibana_log}/fields/ecs.yml (100%) rename packages/platform-observability/data_stream/{log => kibana_log}/manifest.yml (89%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7a28c44ff16..6a14d64cd70 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -168,4 +168,4 @@ /packages/zscaler @elastic/security-external-integrations /packages/zscaler_zia @elastic/security-external-integrations /packages/zscaler_zpa @elastic/security-external-integrations -/packages/platform-observability @elastic/obs-service-integrations @elastic/infra-monitoring-ui +/packages/platform-observability @elastic/infra-monitoring-ui diff --git a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json b/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json deleted file mode 100644 index 7f19c39b4c1..00000000000 --- a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "expected": [ - { - "data_stream": { - "dataset": "kibana-audit-log", - "namespace": "platform-observability", - "type": "logs" - }, - "event": { - "dataset": "kibana-audit-log" - }, - "message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"get\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/internal/security/session\",\"port\":5601,\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"1c8c5808-d2d6-41fc-8cb7-998aa8996be9\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T12:05:03.742+00:00\",\"message\":\"User is requesting [/internal/security/session] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"f8863d86567119e6\"}}" - }, - { - "data_stream": { - "dataset": "kibana-audit-log", - "namespace": "platform-observability", - "type": "logs" - }, - "event": { - "dataset": "kibana-audit-log" - }, - "message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"post\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/internal/bsearch\",\"port\":5601,\"query\":\"compress=true\",\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"abc8b4ad-5d96-42cf-9653-08aaeac0034e\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T12:05:08.178+00:00\",\"message\":\"User is requesting [/internal/bsearch] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"3098796995e24283\"}}" - }, - { - "data_stream": { - "dataset": "kibana-audit-log", - "namespace": "platform-observability", - "type": "logs" - }, - "event": { - "dataset": "kibana-audit-log" - }, - "message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"post\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/api/log_entries/summary\",\"port\":5601,\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"f4181218-b2d3-480e-b9da-78aef88683ff\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T12:05:08.187+00:00\",\"message\":\"User is requesting [/api/log_entries/summary] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"c1480039d6e6e321\"}}" - }, - { - "data_stream": { - "dataset": "kibana-audit-log", - "namespace": "platform-observability", - "type": "logs" - }, - "event": { - "dataset": "kibana-audit-log" - }, - "message": "{\"event\":{\"action\":\"http_request\",\"category\":[\"web\"],\"outcome\":\"unknown\"},\"http\":{\"request\":{\"method\":\"get\"}},\"url\":{\"domain\":\"localhost\",\"path\":\"/internal/security/session\",\"port\":5601,\"scheme\":\"http\"},\"user\":{\"name\":\"elastic\",\"roles\":[\"superuser\"]},\"kibana\":{\"space_id\":\"default\",\"session_id\":\"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=\"},\"trace\":{\"id\":\"83d80454-6b8a-4727-91ba-22e6ab27e476\"},\"ecs\":{\"version\":\"8.0.0\"},\"@timestamp\":\"2022-06-29T13:14:30.142+00:00\",\"message\":\"User is requesting [/internal/security/session] endpoint\",\"log\":{\"level\":\"INFO\",\"logger\":\"plugins.security.audit.ecs\"},\"process\":{\"pid\":7},\"transaction\":{\"id\":\"c1f684ff5fcf7eaf\"}}" - } - ] -} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs b/packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs deleted file mode 100644 index 3ff81290f0d..00000000000 --- a/packages/platform-observability/data_stream/audit/agent/stream/log.yml.hbs +++ /dev/null @@ -1,17 +0,0 @@ -paths: -{{#each paths}} - - {{this}} -{{/each}} -processors: -- rename: - fields: - - from: "message" - to: "event.original" -- decode_json_fields: - fields: [event.original] - target: "" - add_error_key: true - overwrite_keys: true -{{#if processors}} -{{processors}} -{{/if}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/audit/fields/ecs.yml b/packages/platform-observability/data_stream/audit/fields/ecs.yml deleted file mode 100644 index 392fcb5f2e1..00000000000 --- a/packages/platform-observability/data_stream/audit/fields/ecs.yml +++ /dev/null @@ -1,3 +0,0 @@ -# only used for tests -- name: message - external: ecs diff --git a/packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log b/packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log similarity index 100% rename from packages/platform-observability/data_stream/audit/_dev/test/pipeline/test-kibana-audit-logs.log rename to packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log diff --git a/packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json b/packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json new file mode 100644 index 00000000000..86c1fd4ac49 --- /dev/null +++ b/packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json @@ -0,0 +1,221 @@ +{ + "expected": [ + { + "@timestamp": "2022-06-29T12:05:03.742+00:00", + "data_stream": { + "dataset": "kibana-audit-log", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "action": "http_request", + "category": [ + "web" + ], + "dataset": "kibana-audit-log", + "outcome": "unknown" + }, + "http": { + "request": { + "method": "get" + } + }, + "kibana": { + "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", + "space_id": "default" + }, + "log": { + "level": "INFO", + "logger": "plugins.security.audit.ecs" + }, + "message": "User is requesting [/internal/security/session] endpoint", + "process": { + "pid": 7 + }, + "trace": { + "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9" + }, + "transaction": { + "id": "f8863d86567119e6" + }, + "url": { + "domain": "localhost", + "path": "/internal/security/session", + "port": 5601, + "scheme": "http" + }, + "user": { + "name": "elastic", + "roles": [ + "superuser" + ] + } + }, + { + "@timestamp": "2022-06-29T12:05:08.178+00:00", + "data_stream": { + "dataset": "kibana-audit-log", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "action": "http_request", + "category": [ + "web" + ], + "dataset": "kibana-audit-log", + "outcome": "unknown" + }, + "http": { + "request": { + "method": "post" + } + }, + "kibana": { + "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", + "space_id": "default" + }, + "log": { + "level": "INFO", + "logger": "plugins.security.audit.ecs" + }, + "message": "User is requesting [/internal/bsearch] endpoint", + "process": { + "pid": 7 + }, + "trace": { + "id": "abc8b4ad-5d96-42cf-9653-08aaeac0034e" + }, + "transaction": { + "id": "3098796995e24283" + }, + "url": { + "domain": "localhost", + "path": "/internal/bsearch", + "port": 5601, + "query": "compress=true", + "scheme": "http" + }, + "user": { + "name": "elastic", + "roles": [ + "superuser" + ] + } + }, + { + "@timestamp": "2022-06-29T12:05:08.187+00:00", + "data_stream": { + "dataset": "kibana-audit-log", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "action": "http_request", + "category": [ + "web" + ], + "dataset": "kibana-audit-log", + "outcome": "unknown" + }, + "http": { + "request": { + "method": "post" + } + }, + "kibana": { + "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", + "space_id": "default" + }, + "log": { + "level": "INFO", + "logger": "plugins.security.audit.ecs" + }, + "message": "User is requesting [/api/log_entries/summary] endpoint", + "process": { + "pid": 7 + }, + "trace": { + "id": "f4181218-b2d3-480e-b9da-78aef88683ff" + }, + "transaction": { + "id": "c1480039d6e6e321" + }, + "url": { + "domain": "localhost", + "path": "/api/log_entries/summary", + "port": 5601, + "scheme": "http" + }, + "user": { + "name": "elastic", + "roles": [ + "superuser" + ] + } + }, + { + "@timestamp": "2022-06-29T13:14:30.142+00:00", + "data_stream": { + "dataset": "kibana-audit-log", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "action": "http_request", + "category": [ + "web" + ], + "dataset": "kibana-audit-log", + "outcome": "unknown" + }, + "http": { + "request": { + "method": "get" + } + }, + "kibana": { + "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", + "space_id": "default" + }, + "log": { + "level": "INFO", + "logger": "plugins.security.audit.ecs" + }, + "message": "User is requesting [/internal/security/session] endpoint", + "process": { + "pid": 7 + }, + "trace": { + "id": "83d80454-6b8a-4727-91ba-22e6ab27e476" + }, + "transaction": { + "id": "c1f684ff5fcf7eaf" + }, + "url": { + "domain": "localhost", + "path": "/internal/security/session", + "port": 5601, + "scheme": "http" + }, + "user": { + "name": "elastic", + "roles": [ + "superuser" + ] + } + } + ] +} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/log/agent/stream/log.yml.hbs b/packages/platform-observability/data_stream/kibana_audit/agent/stream/log.yml.hbs similarity index 100% rename from packages/platform-observability/data_stream/log/agent/stream/log.yml.hbs rename to packages/platform-observability/data_stream/kibana_audit/agent/stream/log.yml.hbs diff --git a/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/platform-observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml similarity index 52% rename from packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml rename to packages/platform-observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml index 5251f11f89f..870ef8fbae9 100644 --- a/packages/platform-observability/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform-observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml @@ -1,5 +1,5 @@ --- -description: Pipeline for parsing ECS logs +description: Pipeline for parsing Kibana Audit ECS formatted logs processors: - remove: field: data_stream.dataset @@ -35,3 +35,35 @@ processors: - set: field: event.dataset copy_from: data_stream.dataset + - rename: + field: message + target_field: _ecs_json_message + if: |- + def message = ctx.message; + return message != null + && message.startsWith('{') + && message.endsWith('}') + && message.contains('"@timestamp"') + ignore_missing: true + - json: + field: _ecs_json_message + add_to_root: true + add_to_root_conflict_strategy: merge + allow_duplicate_keys: true + if: ctx.containsKey('_ecs_json_message') + on_failure: + - rename: + field: _ecs_json_message + target_field: message + ignore_missing: true + - set: + field: error.message + value: Error while parsing JSON + override: false + - remove: + field: _ecs_json_message + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/platform-observability/data_stream/audit/fields/base-fields.yml b/packages/platform-observability/data_stream/kibana_audit/fields/base-fields.yml similarity index 100% rename from packages/platform-observability/data_stream/audit/fields/base-fields.yml rename to packages/platform-observability/data_stream/kibana_audit/fields/base-fields.yml diff --git a/packages/platform-observability/data_stream/kibana_audit/fields/ecs.yml b/packages/platform-observability/data_stream/kibana_audit/fields/ecs.yml new file mode 100644 index 00000000000..4b0086ccb71 --- /dev/null +++ b/packages/platform-observability/data_stream/kibana_audit/fields/ecs.yml @@ -0,0 +1,35 @@ +# only used for tests +- name: ecs.version + external: ecs +- name: http.request.method + external: ecs +- name: kibana.session_id + external: ecs +- name: kibana.space_id + external: ecs +- name: log.level + external: ecs +- name: log.logger + external: ecs +- name: message + external: ecs +- name: process.pid + external: ecs +- name: trace.id + external: ecs +- name: transaction.id + external: ecs +- name: url.domain + external: ecs +- name: url.path + external: ecs +- name: url.port + external: ecs +- name: url.query + external: ecs +- name: url.scheme + external: ecs +- name: user.name + external: ecs +- name: user.roles + external: ecs diff --git a/packages/platform-observability/data_stream/audit/manifest.yml b/packages/platform-observability/data_stream/kibana_audit/manifest.yml similarity index 89% rename from packages/platform-observability/data_stream/audit/manifest.yml rename to packages/platform-observability/data_stream/kibana_audit/manifest.yml index 03092971721..2d2f5614562 100644 --- a/packages/platform-observability/data_stream/audit/manifest.yml +++ b/packages/platform-observability/data_stream/kibana_audit/manifest.yml @@ -4,9 +4,9 @@ release: experimental streams: - input: logfile vars: - - name: paths + - name: path type: text - title: Paths + title: Path multi: true required: true show_user: true diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log b/packages/platform-observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log similarity index 100% rename from packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log rename to packages/platform-observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log diff --git a/packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json b/packages/platform-observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json similarity index 100% rename from packages/platform-observability/data_stream/log/_dev/test/pipeline/test-kibana-logs.log-expected.json rename to packages/platform-observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json diff --git a/packages/platform-observability/data_stream/kibana_log/agent/stream/log.yml.hbs b/packages/platform-observability/data_stream/kibana_log/agent/stream/log.yml.hbs new file mode 100644 index 00000000000..5042f840763 --- /dev/null +++ b/packages/platform-observability/data_stream/kibana_log/agent/stream/log.yml.hbs @@ -0,0 +1,8 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/platform-observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml similarity index 92% rename from packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml rename to packages/platform-observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml index 5ed5392b5ef..f201a199440 100644 --- a/packages/platform-observability/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform-observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml @@ -1,5 +1,5 @@ --- -description: Pipeline for parsing ECS logs +description: Pipeline for parsing Kibana ECS formatted logs processors: - remove: field: data_stream.dataset @@ -69,3 +69,7 @@ processors: - remove: field: right ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/platform-observability/data_stream/log/fields/base-fields.yml b/packages/platform-observability/data_stream/kibana_log/fields/base-fields.yml similarity index 100% rename from packages/platform-observability/data_stream/log/fields/base-fields.yml rename to packages/platform-observability/data_stream/kibana_log/fields/base-fields.yml diff --git a/packages/platform-observability/data_stream/log/fields/ecs.yml b/packages/platform-observability/data_stream/kibana_log/fields/ecs.yml similarity index 100% rename from packages/platform-observability/data_stream/log/fields/ecs.yml rename to packages/platform-observability/data_stream/kibana_log/fields/ecs.yml diff --git a/packages/platform-observability/data_stream/log/manifest.yml b/packages/platform-observability/data_stream/kibana_log/manifest.yml similarity index 89% rename from packages/platform-observability/data_stream/log/manifest.yml rename to packages/platform-observability/data_stream/kibana_log/manifest.yml index 38b2117adf5..7d9f61ced34 100644 --- a/packages/platform-observability/data_stream/log/manifest.yml +++ b/packages/platform-observability/data_stream/kibana_log/manifest.yml @@ -4,9 +4,9 @@ release: experimental streams: - input: logfile vars: - - name: paths + - name: path type: text - title: Paths + title: Path multi: true required: true show_user: true diff --git a/packages/platform-observability/docs/README.md b/packages/platform-observability/docs/README.md index af8ebc4599e..b296edb5f56 100644 --- a/packages/platform-observability/docs/README.md +++ b/packages/platform-observability/docs/README.md @@ -1,7 +1,26 @@ -# Platform Observability Logs +# Platform Observability -V3 +## Compatibility -This is a new integration created using the [elastic-package](https://github.com/elastic/elastic-package) tool. +This package works with Kibana 8.2.0 and later. -Consider using the README template file `_dev/build/docs/README.md`to generate a list of exported fields or include a sample event. +## Kibana logs + +The Kibana integration collects logs from [Kibana](https://www.elastic.co/guide/en/kibana/current/introduction.html) instance. + +### Logs + +#### Audit + +Configure `Path` pointing to the location where audit logs will be created, based on the [Kibana Audit logging settings](https://www.elastic.co/guide/en/kibana/current/security-settings-kb.html#audit-logging-settings) in `kibana.yml` + +**Exported fields** +TODO: define whether all ECS fields will be exported + +#### Log + + +Configure `Path` pointing to the location where the logs will be created, based on the [Kibana logging settings](https://www.elastic.co/guide/en/kibana/current/logging-configuration.html#logging-appenders) in `kibana.yml` + +**Exported fields** +TODO: define whether all ECS fields will be exported diff --git a/packages/platform-observability/manifest.yml b/packages/platform-observability/manifest.yml index 3aaa178a52f..8c5d75956d4 100644 --- a/packages/platform-observability/manifest.yml +++ b/packages/platform-observability/manifest.yml @@ -4,7 +4,7 @@ title: "Platform Observability" version: 0.0.1 license: basic release: experimental -description: "This is a PoC for ingesting ECS formatted logs of one stack component" +description: "Collect Stack Monitoring logs with Elastic Agent" type: integration icons: - src: /img/logo_kibana.svg @@ -13,11 +13,11 @@ icons: type: image/svg+xml categories: ["elastic_stack"] conditions: - kibana.version: "^8.2.0" + kibana.version: "^8.0.0" policy_templates: - - name: stack-monitoring-kibana-logs - title: Kibana logs and metrics - description: Collect logs and metrics from Kibana instances + - name: platform-observability-kibana-logs + title: Kibana logs + description: Collect logs from Kibana inputs: - type: logfile title: "Collect Kibana logs" From 6e781a3471a7a7b89611b4d150e64f6fbbec6acf Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Thu, 14 Jul 2022 15:47:33 +0200 Subject: [PATCH 08/18] Improve README.md; Fix package folder name; Document ECS fields --- .github/CODEOWNERS | 2 +- .../test/pipeline/test-kibana-audit-logs.log | 4 - .../test-kibana-audit-logs.log-expected.json | 221 ------------------ .../_dev/build/build.yml | 3 + .../_dev/build}/docs/README.md | 7 +- .../changelog.yml | 0 .../test/pipeline/test-kibana-audit-logs.log | 1 + .../test-kibana-audit-logs.log-expected.json | 58 +++++ .../kibana_audit/agent/stream/log.yml.hbs | 0 .../elasticsearch/ingest_pipeline/default.yml | 0 .../kibana_audit/fields/base-fields.yml | 0 .../data_stream/kibana_audit/fields/beats.yml | 6 + .../data_stream/kibana_audit/fields/ecs.yml | 4 - .../data_stream/kibana_audit/manifest.yml | 12 +- .../_dev/test/pipeline/test-kibana-logs.log | 3 +- .../test-kibana-logs.log-expected.json | 52 ++--- .../kibana_log/agent/stream/log.yml.hbs | 0 .../elasticsearch/ingest_pipeline/default.yml | 0 .../kibana_log/fields/base-fields.yml | 0 .../data_stream/kibana_log/fields/ecs.yml | 12 + .../data_stream/kibana_log/manifest.yml | 10 +- .../platform_observability/docs/README.md | 70 ++++++ .../img/logo_kibana.svg | 0 .../manifest.yml | 4 +- 24 files changed, 187 insertions(+), 282 deletions(-) delete mode 100644 packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log delete mode 100644 packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json create mode 100644 packages/platform_observability/_dev/build/build.yml rename packages/{platform-observability => platform_observability/_dev/build}/docs/README.md (83%) rename packages/{platform-observability => platform_observability}/changelog.yml (100%) create mode 100644 packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log create mode 100644 packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json rename packages/{platform-observability => platform_observability}/data_stream/kibana_audit/agent/stream/log.yml.hbs (100%) rename packages/{platform-observability => platform_observability}/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml (100%) rename packages/{platform-observability => platform_observability}/data_stream/kibana_audit/fields/base-fields.yml (100%) create mode 100644 packages/platform_observability/data_stream/kibana_audit/fields/beats.yml rename packages/{platform-observability => platform_observability}/data_stream/kibana_audit/fields/ecs.yml (87%) rename packages/{platform-observability => platform_observability}/data_stream/kibana_audit/manifest.yml (50%) rename packages/{platform-observability => platform_observability}/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log (52%) rename packages/{platform-observability => platform_observability}/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json (67%) rename packages/{platform-observability => platform_observability}/data_stream/kibana_log/agent/stream/log.yml.hbs (100%) rename packages/{platform-observability => platform_observability}/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml (100%) rename packages/{platform-observability => platform_observability}/data_stream/kibana_log/fields/base-fields.yml (100%) rename packages/{platform-observability => platform_observability}/data_stream/kibana_log/fields/ecs.yml (51%) rename packages/{platform-observability => platform_observability}/data_stream/kibana_log/manifest.yml (58%) create mode 100644 packages/platform_observability/docs/README.md rename packages/{platform-observability => platform_observability}/img/logo_kibana.svg (100%) rename packages/{platform-observability => platform_observability}/manifest.yml (84%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6a14d64cd70..a371c9535eb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -168,4 +168,4 @@ /packages/zscaler @elastic/security-external-integrations /packages/zscaler_zia @elastic/security-external-integrations /packages/zscaler_zpa @elastic/security-external-integrations -/packages/platform-observability @elastic/infra-monitoring-ui +/packages/platform_observability @elastic/infra-monitoring-ui diff --git a/packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log b/packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log deleted file mode 100644 index 605259fae86..00000000000 --- a/packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log +++ /dev/null @@ -1,4 +0,0 @@ -{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"localhost","path":"/internal/security/session","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"1c8c5808-d2d6-41fc-8cb7-998aa8996be9"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:03.742+00:00","message":"User is requesting [/internal/security/session] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"f8863d86567119e6"}} -{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/internal/bsearch","port":5601,"query":"compress=true","scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"abc8b4ad-5d96-42cf-9653-08aaeac0034e"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:08.178+00:00","message":"User is requesting [/internal/bsearch] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"3098796995e24283"}} -{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"post"}},"url":{"domain":"localhost","path":"/api/log_entries/summary","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"f4181218-b2d3-480e-b9da-78aef88683ff"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:08.187+00:00","message":"User is requesting [/api/log_entries/summary] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"c1480039d6e6e321"}} -{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"localhost","path":"/internal/security/session","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"83d80454-6b8a-4727-91ba-22e6ab27e476"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T13:14:30.142+00:00","message":"User is requesting [/internal/security/session] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"c1f684ff5fcf7eaf"}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json b/packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json deleted file mode 100644 index 86c1fd4ac49..00000000000 --- a/packages/platform-observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json +++ /dev/null @@ -1,221 +0,0 @@ -{ - "expected": [ - { - "@timestamp": "2022-06-29T12:05:03.742+00:00", - "data_stream": { - "dataset": "kibana-audit-log", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "action": "http_request", - "category": [ - "web" - ], - "dataset": "kibana-audit-log", - "outcome": "unknown" - }, - "http": { - "request": { - "method": "get" - } - }, - "kibana": { - "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", - "space_id": "default" - }, - "log": { - "level": "INFO", - "logger": "plugins.security.audit.ecs" - }, - "message": "User is requesting [/internal/security/session] endpoint", - "process": { - "pid": 7 - }, - "trace": { - "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9" - }, - "transaction": { - "id": "f8863d86567119e6" - }, - "url": { - "domain": "localhost", - "path": "/internal/security/session", - "port": 5601, - "scheme": "http" - }, - "user": { - "name": "elastic", - "roles": [ - "superuser" - ] - } - }, - { - "@timestamp": "2022-06-29T12:05:08.178+00:00", - "data_stream": { - "dataset": "kibana-audit-log", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "action": "http_request", - "category": [ - "web" - ], - "dataset": "kibana-audit-log", - "outcome": "unknown" - }, - "http": { - "request": { - "method": "post" - } - }, - "kibana": { - "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", - "space_id": "default" - }, - "log": { - "level": "INFO", - "logger": "plugins.security.audit.ecs" - }, - "message": "User is requesting [/internal/bsearch] endpoint", - "process": { - "pid": 7 - }, - "trace": { - "id": "abc8b4ad-5d96-42cf-9653-08aaeac0034e" - }, - "transaction": { - "id": "3098796995e24283" - }, - "url": { - "domain": "localhost", - "path": "/internal/bsearch", - "port": 5601, - "query": "compress=true", - "scheme": "http" - }, - "user": { - "name": "elastic", - "roles": [ - "superuser" - ] - } - }, - { - "@timestamp": "2022-06-29T12:05:08.187+00:00", - "data_stream": { - "dataset": "kibana-audit-log", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "action": "http_request", - "category": [ - "web" - ], - "dataset": "kibana-audit-log", - "outcome": "unknown" - }, - "http": { - "request": { - "method": "post" - } - }, - "kibana": { - "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", - "space_id": "default" - }, - "log": { - "level": "INFO", - "logger": "plugins.security.audit.ecs" - }, - "message": "User is requesting [/api/log_entries/summary] endpoint", - "process": { - "pid": 7 - }, - "trace": { - "id": "f4181218-b2d3-480e-b9da-78aef88683ff" - }, - "transaction": { - "id": "c1480039d6e6e321" - }, - "url": { - "domain": "localhost", - "path": "/api/log_entries/summary", - "port": 5601, - "scheme": "http" - }, - "user": { - "name": "elastic", - "roles": [ - "superuser" - ] - } - }, - { - "@timestamp": "2022-06-29T13:14:30.142+00:00", - "data_stream": { - "dataset": "kibana-audit-log", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "action": "http_request", - "category": [ - "web" - ], - "dataset": "kibana-audit-log", - "outcome": "unknown" - }, - "http": { - "request": { - "method": "get" - } - }, - "kibana": { - "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", - "space_id": "default" - }, - "log": { - "level": "INFO", - "logger": "plugins.security.audit.ecs" - }, - "message": "User is requesting [/internal/security/session] endpoint", - "process": { - "pid": 7 - }, - "trace": { - "id": "83d80454-6b8a-4727-91ba-22e6ab27e476" - }, - "transaction": { - "id": "c1f684ff5fcf7eaf" - }, - "url": { - "domain": "localhost", - "path": "/internal/security/session", - "port": 5601, - "scheme": "http" - }, - "user": { - "name": "elastic", - "roles": [ - "superuser" - ] - } - } - ] -} \ No newline at end of file diff --git a/packages/platform_observability/_dev/build/build.yml b/packages/platform_observability/_dev/build/build.yml new file mode 100644 index 00000000000..08d85edcf9a --- /dev/null +++ b/packages/platform_observability/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@1.12 diff --git a/packages/platform-observability/docs/README.md b/packages/platform_observability/_dev/build/docs/README.md similarity index 83% rename from packages/platform-observability/docs/README.md rename to packages/platform_observability/_dev/build/docs/README.md index b296edb5f56..6c3c7fee37f 100644 --- a/packages/platform-observability/docs/README.md +++ b/packages/platform_observability/_dev/build/docs/README.md @@ -14,13 +14,10 @@ The Kibana integration collects logs from [Kibana](https://www.elastic.co/guide/ Configure `Path` pointing to the location where audit logs will be created, based on the [Kibana Audit logging settings](https://www.elastic.co/guide/en/kibana/current/security-settings-kb.html#audit-logging-settings) in `kibana.yml` -**Exported fields** -TODO: define whether all ECS fields will be exported +{{fields "kibana_audit"}} #### Log - Configure `Path` pointing to the location where the logs will be created, based on the [Kibana logging settings](https://www.elastic.co/guide/en/kibana/current/logging-configuration.html#logging-appenders) in `kibana.yml` -**Exported fields** -TODO: define whether all ECS fields will be exported +{{fields "kibana_log"}} diff --git a/packages/platform-observability/changelog.yml b/packages/platform_observability/changelog.yml similarity index 100% rename from packages/platform-observability/changelog.yml rename to packages/platform_observability/changelog.yml diff --git a/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log new file mode 100644 index 00000000000..ed7cd51a51b --- /dev/null +++ b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log @@ -0,0 +1 @@ +{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"localhost","path":"/internal/security/session","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default","session_id":"ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k="},"trace":{"id":"1c8c5808-d2d6-41fc-8cb7-998aa8996be9"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T12:05:03.742+00:00","message":"User is requesting [/internal/security/session] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"f8863d86567119e6"}} \ No newline at end of file diff --git a/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json new file mode 100644 index 00000000000..28a5f5ff8c4 --- /dev/null +++ b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json @@ -0,0 +1,58 @@ +{ + "expected": [ + { + "@timestamp": "2022-06-29T12:05:03.742+00:00", + "data_stream": { + "dataset": "kibana-audit-log", + "namespace": "platform-observability", + "type": "logs" + }, + "ecs": { + "version": "8.0.0" + }, + "event": { + "action": "http_request", + "category": [ + "web" + ], + "dataset": "kibana-audit-log", + "outcome": "unknown" + }, + "http": { + "request": { + "method": "get" + } + }, + "kibana": { + "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=", + "space_id": "default" + }, + "log": { + "level": "INFO", + "logger": "plugins.security.audit.ecs" + }, + "message": "User is requesting [/internal/security/session] endpoint", + "process": { + "pid": 7 + }, + "trace": { + "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9" + }, + "transaction": { + "id": "f8863d86567119e6" + }, + "url": { + "domain": "localhost", + "path": "/internal/security/session", + "port": 5601, + "scheme": "http" + }, + "user": { + "name": "elastic", + "roles": [ + "superuser" + ] + } + } + ] +} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/kibana_audit/agent/stream/log.yml.hbs b/packages/platform_observability/data_stream/kibana_audit/agent/stream/log.yml.hbs similarity index 100% rename from packages/platform-observability/data_stream/kibana_audit/agent/stream/log.yml.hbs rename to packages/platform_observability/data_stream/kibana_audit/agent/stream/log.yml.hbs diff --git a/packages/platform-observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml b/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml similarity index 100% rename from packages/platform-observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml rename to packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml diff --git a/packages/platform-observability/data_stream/kibana_audit/fields/base-fields.yml b/packages/platform_observability/data_stream/kibana_audit/fields/base-fields.yml similarity index 100% rename from packages/platform-observability/data_stream/kibana_audit/fields/base-fields.yml rename to packages/platform_observability/data_stream/kibana_audit/fields/base-fields.yml diff --git a/packages/platform_observability/data_stream/kibana_audit/fields/beats.yml b/packages/platform_observability/data_stream/kibana_audit/fields/beats.yml new file mode 100644 index 00000000000..041301ae646 --- /dev/null +++ b/packages/platform_observability/data_stream/kibana_audit/fields/beats.yml @@ -0,0 +1,6 @@ +- name: kibana.session_id + description: The ID of the user session associated with this event. Each login attempt results in a unique session id + type: keyword +- name: kibana.space_id + description: The id of the space associated with this event. + type: keyword diff --git a/packages/platform-observability/data_stream/kibana_audit/fields/ecs.yml b/packages/platform_observability/data_stream/kibana_audit/fields/ecs.yml similarity index 87% rename from packages/platform-observability/data_stream/kibana_audit/fields/ecs.yml rename to packages/platform_observability/data_stream/kibana_audit/fields/ecs.yml index 4b0086ccb71..de3db8f9ab4 100644 --- a/packages/platform-observability/data_stream/kibana_audit/fields/ecs.yml +++ b/packages/platform_observability/data_stream/kibana_audit/fields/ecs.yml @@ -3,10 +3,6 @@ external: ecs - name: http.request.method external: ecs -- name: kibana.session_id - external: ecs -- name: kibana.space_id - external: ecs - name: log.level external: ecs - name: log.logger diff --git a/packages/platform-observability/data_stream/kibana_audit/manifest.yml b/packages/platform_observability/data_stream/kibana_audit/manifest.yml similarity index 50% rename from packages/platform-observability/data_stream/kibana_audit/manifest.yml rename to packages/platform_observability/data_stream/kibana_audit/manifest.yml index 2d2f5614562..1fc94ae115b 100644 --- a/packages/platform-observability/data_stream/kibana_audit/manifest.yml +++ b/packages/platform_observability/data_stream/kibana_audit/manifest.yml @@ -1,17 +1,17 @@ type: logs -title: Platform Observability Kibana Audit Logs +title: Platform Observability Kibana audit logs release: experimental streams: - input: logfile vars: - - name: path + - name: paths type: text - title: Path + title: Paths multi: true required: true show_user: true default: - - /tmp/service_logs/audit*.log + - /var/log/kibana/*_audit.json template_path: log.yml.hbs - title: Kibana Audit Logs - description: Collect Kibana Audit Logs + title: Kibana audit logs + description: Collect Kibana audit logs diff --git a/packages/platform-observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log similarity index 52% rename from packages/platform-observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log rename to packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log index aa08e9c6569..a49dd498911 100644 --- a/packages/platform-observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log +++ b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log @@ -1,4 +1,3 @@ {"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Trying to authenticate user request to /login.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} -{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Authorization header is not presented.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} -{"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Could not handle authentication attempt","log":{"level":"DEBUG","logger":"plugins.security.authentication"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} +{"http":{"request":{"id":"unknownId","method":"GET"},"response":{"body":{"bytes":118},"status_code":200}},"url":{"path":"/_nodes","query":"filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-07-14T10:35:25.366+00:00","message":"200 - 118.0B\nGET /_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip","log":{"level":"DEBUG","logger":"elasticsearch.query.data"},"process":{"pid":7},"trace":{"id":"0cd8dd5a3483159a43c07e9205432775"},"transaction":{"id":"6301eca88fba8d99"}} {"_tag":"Right","right":"update_aliases_succeeded","ecs":{"version":"8.0.0"},"@timestamp":"2022-07-04T09:17:38.611+00:00","message":"[.kibana] MARK_VERSION_INDEX_READY RESPONSE","log":{"level":"DEBUG","logger":"savedobjects-service"},"process":{"pid":7},"trace":{"id":"a167d1124764379d4121b357e20baee2"},"transaction":{"id":"14717ae6e3b30d5a"}} \ No newline at end of file diff --git a/packages/platform-observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json similarity index 67% rename from packages/platform-observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json rename to packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json index 4d20c4546da..658516af42d 100644 --- a/packages/platform-observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json +++ b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json @@ -29,7 +29,7 @@ } }, { - "@timestamp": "2022-06-29T11:24:17.898+00:00", + "@timestamp": "2022-07-14T10:35:25.366+00:00", "data_stream": { "dataset": "kibana-logs", "namespace": "platform-observability", @@ -41,47 +41,35 @@ "event": { "dataset": "kibana-logs" }, - "log": { - "level": "DEBUG", - "logger": "plugins.security.http" - }, - "message": "Authorization header is not presented.", - "process": { - "pid": 7 - }, - "trace": { - "id": "e6e1c25936546ec690b11a3b78b2a8db" - }, - "transaction": { - "id": "3be6994d7f6d5465" - } - }, - { - "@timestamp": "2022-06-29T11:24:17.898+00:00", - "data_stream": { - "dataset": "kibana-logs", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "dataset": "kibana-logs" + "http": { + "request": { + "id": "unknownId", + "method": "GET" + }, + "response": { + "body": { + "bytes": 118 + }, + "status_code": 200 + } }, "log": { "level": "DEBUG", - "logger": "plugins.security.authentication" + "logger": "elasticsearch.query.data" }, - "message": "Could not handle authentication attempt", + "message": "200 - 118.0B\nGET /_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", "process": { "pid": 7 }, "trace": { - "id": "e6e1c25936546ec690b11a3b78b2a8db" + "id": "0cd8dd5a3483159a43c07e9205432775" }, "transaction": { - "id": "3be6994d7f6d5465" + "id": "6301eca88fba8d99" + }, + "url": { + "path": "/_nodes", + "query": "filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip" } }, { diff --git a/packages/platform-observability/data_stream/kibana_log/agent/stream/log.yml.hbs b/packages/platform_observability/data_stream/kibana_log/agent/stream/log.yml.hbs similarity index 100% rename from packages/platform-observability/data_stream/kibana_log/agent/stream/log.yml.hbs rename to packages/platform_observability/data_stream/kibana_log/agent/stream/log.yml.hbs diff --git a/packages/platform-observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml b/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml similarity index 100% rename from packages/platform-observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml rename to packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml diff --git a/packages/platform-observability/data_stream/kibana_log/fields/base-fields.yml b/packages/platform_observability/data_stream/kibana_log/fields/base-fields.yml similarity index 100% rename from packages/platform-observability/data_stream/kibana_log/fields/base-fields.yml rename to packages/platform_observability/data_stream/kibana_log/fields/base-fields.yml diff --git a/packages/platform-observability/data_stream/kibana_log/fields/ecs.yml b/packages/platform_observability/data_stream/kibana_log/fields/ecs.yml similarity index 51% rename from packages/platform-observability/data_stream/kibana_log/fields/ecs.yml rename to packages/platform_observability/data_stream/kibana_log/fields/ecs.yml index 6df80b6f4f7..f070b041162 100644 --- a/packages/platform-observability/data_stream/kibana_log/fields/ecs.yml +++ b/packages/platform_observability/data_stream/kibana_log/fields/ecs.yml @@ -1,6 +1,14 @@ # only used for tests - name: ecs.version external: ecs +- name: http.request.id + external: ecs +- name: http.request.method + external: ecs +- name: http.response.body.bytes + external: ecs +- name: http.response.status_code + external: ecs - name: log.level external: ecs - name: log.logger @@ -13,3 +21,7 @@ external: ecs - name: transaction.id external: ecs +- name: url.path + external: ecs +- name: url.query + external: ecs diff --git a/packages/platform-observability/data_stream/kibana_log/manifest.yml b/packages/platform_observability/data_stream/kibana_log/manifest.yml similarity index 58% rename from packages/platform-observability/data_stream/kibana_log/manifest.yml rename to packages/platform_observability/data_stream/kibana_log/manifest.yml index 7d9f61ced34..37a237166b0 100644 --- a/packages/platform-observability/data_stream/kibana_log/manifest.yml +++ b/packages/platform_observability/data_stream/kibana_log/manifest.yml @@ -1,17 +1,17 @@ type: logs -title: Platform Observability Kibana Logs +title: Platform Observability Kibana logs release: experimental streams: - input: logfile vars: - - name: path + - name: paths type: text - title: Path + title: Paths multi: true required: true show_user: true default: - - /tmp/service_logs/kibana.log + - /var/log/kibana/kibana.stdout template_path: log.yml.hbs title: Kibana logs - description: Collect Kibana Logs + description: Collect Kibana logs diff --git a/packages/platform_observability/docs/README.md b/packages/platform_observability/docs/README.md new file mode 100644 index 00000000000..7ced5c7db8d --- /dev/null +++ b/packages/platform_observability/docs/README.md @@ -0,0 +1,70 @@ +# Platform Observability + +## Compatibility + +This package works with Kibana 8.2.0 and later. + +## Kibana logs + +The Kibana integration collects logs from [Kibana](https://www.elastic.co/guide/en/kibana/current/introduction.html) instance. + +### Logs + +#### Audit + +Configure `Path` pointing to the location where audit logs will be created, based on the [Kibana Audit logging settings](https://www.elastic.co/guide/en/kibana/current/security-settings-kb.html#audit-logging-settings) in `kibana.yml` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | +| kibana.session_id | The ID of the user session associated with this event. Each login attempt results in a unique session id | keyword | +| kibana.space_id | The id of the space associated with this event. | keyword | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | +| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| process.pid | Process id. | long | +| trace.id | Unique identifier of the trace. A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. | keyword | +| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | +| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| url.path | Path of the request, such as "/search". | wildcard | +| url.port | Port of the request, such as 443. | long | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | +| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | +| user.name | Short name or login of the user. | keyword | +| user.name.text | Multi-field of `user.name`. | match_only_text | +| user.roles | Array of user roles at the time of the event. | keyword | + + +#### Log + +Configure `Path` pointing to the location where the logs will be created, based on the [Kibana logging settings](https://www.elastic.co/guide/en/kibana/current/logging-configuration.html#logging-appenders) in `kibana.yml` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| http.request.id | A unique identifier for each HTTP request to correlate logs between clients and servers in transactions. The id may be contained in a non-standard HTTP header, such as `X-Request-ID` or `X-Correlation-ID`. | keyword | +| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | +| http.response.body.bytes | Size in bytes of the response body. | long | +| http.response.status_code | HTTP response status code. | long | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | +| log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| process.pid | Process id. | long | +| trace.id | Unique identifier of the trace. A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. | keyword | +| transaction.id | Unique identifier of the transaction within the scope of its trace. A transaction is the highest level of work measured within a service, such as a request to a server. | keyword | +| url.path | Path of the request, such as "/search". | wildcard | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | + diff --git a/packages/platform-observability/img/logo_kibana.svg b/packages/platform_observability/img/logo_kibana.svg similarity index 100% rename from packages/platform-observability/img/logo_kibana.svg rename to packages/platform_observability/img/logo_kibana.svg diff --git a/packages/platform-observability/manifest.yml b/packages/platform_observability/manifest.yml similarity index 84% rename from packages/platform-observability/manifest.yml rename to packages/platform_observability/manifest.yml index 8c5d75956d4..e92af05ccc5 100644 --- a/packages/platform-observability/manifest.yml +++ b/packages/platform_observability/manifest.yml @@ -4,7 +4,7 @@ title: "Platform Observability" version: 0.0.1 license: basic release: experimental -description: "Collect Stack Monitoring logs with Elastic Agent" +description: "Collect stack component logs with Elastic Agent" type: integration icons: - src: /img/logo_kibana.svg @@ -15,7 +15,7 @@ categories: ["elastic_stack"] conditions: kibana.version: "^8.0.0" policy_templates: - - name: platform-observability-kibana-logs + - name: platform_observability_kibana_logs title: Kibana logs description: Collect logs from Kibana inputs: From 0b03ec6bb022300bbcedd282715e2915e862b7c3 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 18 Jul 2022 10:06:23 +0200 Subject: [PATCH 09/18] Fix supported kibana version on readme --- packages/platform_observability/_dev/build/docs/README.md | 2 +- packages/platform_observability/docs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/platform_observability/_dev/build/docs/README.md b/packages/platform_observability/_dev/build/docs/README.md index 6c3c7fee37f..a522cdf1f08 100644 --- a/packages/platform_observability/_dev/build/docs/README.md +++ b/packages/platform_observability/_dev/build/docs/README.md @@ -2,7 +2,7 @@ ## Compatibility -This package works with Kibana 8.2.0 and later. +This package works with Kibana 8.0.0 and later. ## Kibana logs diff --git a/packages/platform_observability/docs/README.md b/packages/platform_observability/docs/README.md index 7ced5c7db8d..40fc3bee507 100644 --- a/packages/platform_observability/docs/README.md +++ b/packages/platform_observability/docs/README.md @@ -2,7 +2,7 @@ ## Compatibility -This package works with Kibana 8.2.0 and later. +This package works with Kibana 8.0.0 and later. ## Kibana logs From 33f32eb54f32992895a3dd21e6660aa4ddc3655b Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 18 Jul 2022 16:52:17 +0200 Subject: [PATCH 10/18] Remove experimental attribute from manifest; Fix changelog PR; Fix min version --- .../platform_observability/_dev/build/build.yml | 2 +- .../_dev/build/docs/README.md | 2 +- packages/platform_observability/changelog.yml | 2 +- .../data_stream/kibana_audit/manifest.yml | 1 - .../data_stream/kibana_log/manifest.yml | 1 - packages/platform_observability/docs/README.md | 16 ++++++++++++---- packages/platform_observability/manifest.yml | 3 +-- 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/platform_observability/_dev/build/build.yml b/packages/platform_observability/_dev/build/build.yml index 08d85edcf9a..5661d603a89 100644 --- a/packages/platform_observability/_dev/build/build.yml +++ b/packages/platform_observability/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@1.12 + reference: git@v8.3.0 diff --git a/packages/platform_observability/_dev/build/docs/README.md b/packages/platform_observability/_dev/build/docs/README.md index a522cdf1f08..721c56e8717 100644 --- a/packages/platform_observability/_dev/build/docs/README.md +++ b/packages/platform_observability/_dev/build/docs/README.md @@ -2,7 +2,7 @@ ## Compatibility -This package works with Kibana 8.0.0 and later. +This package works with Kibana 8.3.0 and later. ## Kibana logs diff --git a/packages/platform_observability/changelog.yml b/packages/platform_observability/changelog.yml index e00f8813359..1a514de9064 100644 --- a/packages/platform_observability/changelog.yml +++ b/packages/platform_observability/changelog.yml @@ -3,4 +3,4 @@ changes: - description: Initial draft of the package type: enhancement - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/3622 diff --git a/packages/platform_observability/data_stream/kibana_audit/manifest.yml b/packages/platform_observability/data_stream/kibana_audit/manifest.yml index 1fc94ae115b..31c7cf1655d 100644 --- a/packages/platform_observability/data_stream/kibana_audit/manifest.yml +++ b/packages/platform_observability/data_stream/kibana_audit/manifest.yml @@ -1,6 +1,5 @@ type: logs title: Platform Observability Kibana audit logs -release: experimental streams: - input: logfile vars: diff --git a/packages/platform_observability/data_stream/kibana_log/manifest.yml b/packages/platform_observability/data_stream/kibana_log/manifest.yml index 37a237166b0..00533fb01dd 100644 --- a/packages/platform_observability/data_stream/kibana_log/manifest.yml +++ b/packages/platform_observability/data_stream/kibana_log/manifest.yml @@ -1,6 +1,5 @@ type: logs title: Platform Observability Kibana logs -release: experimental streams: - input: logfile vars: diff --git a/packages/platform_observability/docs/README.md b/packages/platform_observability/docs/README.md index 40fc3bee507..ca6787c3f96 100644 --- a/packages/platform_observability/docs/README.md +++ b/packages/platform_observability/docs/README.md @@ -2,7 +2,7 @@ ## Compatibility -This package works with Kibana 8.0.0 and later. +This package works with Kibana 8.3.0 and later. ## Kibana logs @@ -23,8 +23,16 @@ Configure `Path` pointing to the location where audit logs will be created, base | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | -| kibana.session_id | The ID of the user session associated with this event. Each login attempt results in a unique session id | keyword | +| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | +| kibana.add_to_spaces | The set of space ids that a saved object was shared to. | keyword | +| kibana.authentication_provider | The authentication provider associated with a login event. | keyword | +| kibana.authentication_realm | The Elasticsearch authentication realm name which fulfilled a login event. | keyword | +| kibana.authentication_type | The authentication provider type associated with a login event. | keyword | +| kibana.delete_from_spaces | The set of space ids that a saved object was removed from. | keyword | +| kibana.lookup_realm | The Elasticsearch lookup realm which fulfilled a login event. | keyword | +| kibana.saved_object.id | The id of the saved object associated with this event. | keyword | +| kibana.saved_object.type | The type of the saved object associated with this event. | keyword | +| kibana.session_id | The ID of the user session associated with this event. Each login attempt results in a unique session id. | keyword | | kibana.space_id | The id of the space associated with this event. | keyword | | log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.logger | The name of the logger inside an application. This is usually the name of the class which initialized the logger, or can be a custom name. | keyword | @@ -56,7 +64,7 @@ Configure `Path` pointing to the location where the logs will be created, based | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | http.request.id | A unique identifier for each HTTP request to correlate logs between clients and servers in transactions. The id may be contained in a non-standard HTTP header, such as `X-Request-ID` or `X-Correlation-ID`. | keyword | -| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | +| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | | http.response.body.bytes | Size in bytes of the response body. | long | | http.response.status_code | HTTP response status code. | long | | log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | diff --git a/packages/platform_observability/manifest.yml b/packages/platform_observability/manifest.yml index e92af05ccc5..f0b31dd2444 100644 --- a/packages/platform_observability/manifest.yml +++ b/packages/platform_observability/manifest.yml @@ -3,7 +3,6 @@ name: platform_observability title: "Platform Observability" version: 0.0.1 license: basic -release: experimental description: "Collect stack component logs with Elastic Agent" type: integration icons: @@ -13,7 +12,7 @@ icons: type: image/svg+xml categories: ["elastic_stack"] conditions: - kibana.version: "^8.0.0" + kibana.version: "^8.3.0" policy_templates: - name: platform_observability_kibana_logs title: Kibana logs From 8e471603e1f1e09a486206de0dcd70de8c531f22 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 18 Jul 2022 17:09:44 +0200 Subject: [PATCH 11/18] Support current and old model for license --- packages/platform_observability/manifest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/platform_observability/manifest.yml b/packages/platform_observability/manifest.yml index f0b31dd2444..a54727700a5 100644 --- a/packages/platform_observability/manifest.yml +++ b/packages/platform_observability/manifest.yml @@ -3,6 +3,8 @@ name: platform_observability title: "Platform Observability" version: 0.0.1 license: basic +conditions: + elastic.subscription: 'basic' description: "Collect stack component logs with Elastic Agent" type: integration icons: From 3e74c73f2b056b6d4bdb119d20e5935a8d4f741a Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 18 Jul 2022 17:13:02 +0200 Subject: [PATCH 12/18] Fix manifest --- packages/platform_observability/manifest.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/platform_observability/manifest.yml b/packages/platform_observability/manifest.yml index a54727700a5..7a87abfcc55 100644 --- a/packages/platform_observability/manifest.yml +++ b/packages/platform_observability/manifest.yml @@ -3,8 +3,6 @@ name: platform_observability title: "Platform Observability" version: 0.0.1 license: basic -conditions: - elastic.subscription: 'basic' description: "Collect stack component logs with Elastic Agent" type: integration icons: @@ -15,6 +13,7 @@ icons: categories: ["elastic_stack"] conditions: kibana.version: "^8.3.0" + elastic.subscription: "basic" policy_templates: - name: platform_observability_kibana_logs title: Kibana logs From 14dc44ffe87323b3938eddf3d06dc39da404ec79 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 18 Jul 2022 17:14:07 +0200 Subject: [PATCH 13/18] Add event.ingested field; Improve Kibana audit fields documentation --- .../test-kibana-audit-logs.log-expected.json | 1 + .../elasticsearch/ingest_pipeline/default.yml | 3 ++ .../data_stream/kibana_audit/fields/beats.yml | 6 ---- .../kibana_audit/fields/package-fields.yml | 33 +++++++++++++++++++ .../test-kibana-logs.log-expected.json | 9 +++-- .../elasticsearch/ingest_pipeline/default.yml | 5 +++ 6 files changed, 48 insertions(+), 9 deletions(-) delete mode 100644 packages/platform_observability/data_stream/kibana_audit/fields/beats.yml create mode 100644 packages/platform_observability/data_stream/kibana_audit/fields/package-fields.yml diff --git a/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json index 28a5f5ff8c4..aecd6a2a90e 100644 --- a/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json +++ b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json @@ -16,6 +16,7 @@ "web" ], "dataset": "kibana-audit-log", + "ingested": "2022-07-18T15:10:15.717414176Z", "outcome": "unknown" }, "http": { diff --git a/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml b/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml index 870ef8fbae9..3006e93d76d 100644 --- a/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml @@ -63,6 +63,9 @@ processors: - remove: field: _ecs_json_message ignore_missing: true + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" on_failure: - set: field: error.message diff --git a/packages/platform_observability/data_stream/kibana_audit/fields/beats.yml b/packages/platform_observability/data_stream/kibana_audit/fields/beats.yml deleted file mode 100644 index 041301ae646..00000000000 --- a/packages/platform_observability/data_stream/kibana_audit/fields/beats.yml +++ /dev/null @@ -1,6 +0,0 @@ -- name: kibana.session_id - description: The ID of the user session associated with this event. Each login attempt results in a unique session id - type: keyword -- name: kibana.space_id - description: The id of the space associated with this event. - type: keyword diff --git a/packages/platform_observability/data_stream/kibana_audit/fields/package-fields.yml b/packages/platform_observability/data_stream/kibana_audit/fields/package-fields.yml new file mode 100644 index 00000000000..8099b58debb --- /dev/null +++ b/packages/platform_observability/data_stream/kibana_audit/fields/package-fields.yml @@ -0,0 +1,33 @@ +- name: kibana + type: group + fields: + - name: session_id + type: keyword + description: The ID of the user session associated with this event. Each login attempt results in a unique session id. + - name: space_id + type: keyword + description: The id of the space associated with this event. + - name: saved_object.type + type: keyword + description: The type of the saved object associated with this event. + - name: saved_object.id + type: keyword + description: The id of the saved object associated with this event. + - name: add_to_spaces + type: keyword + description: The set of space ids that a saved object was shared to. + - name: delete_from_spaces + type: keyword + description: The set of space ids that a saved object was removed from. + - name: authentication_provider + type: keyword + description: The authentication provider associated with a login event. + - name: authentication_type + type: keyword + description: The authentication provider type associated with a login event. + - name: authentication_realm + type: keyword + description: The Elasticsearch authentication realm name which fulfilled a login event. + - name: lookup_realm + type: keyword + description: The Elasticsearch lookup realm which fulfilled a login event. diff --git a/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json index 658516af42d..048bff76ec0 100644 --- a/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json +++ b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json @@ -11,7 +11,8 @@ "version": "8.0.0" }, "event": { - "dataset": "kibana-logs" + "dataset": "kibana-logs", + "ingested": "2022-07-18T15:10:15.862948593Z" }, "log": { "level": "DEBUG", @@ -39,7 +40,8 @@ "version": "8.0.0" }, "event": { - "dataset": "kibana-logs" + "dataset": "kibana-logs", + "ingested": "2022-07-18T15:10:15.862951843Z" }, "http": { "request": { @@ -83,7 +85,8 @@ "version": "8.0.0" }, "event": { - "dataset": "kibana-logs" + "dataset": "kibana-logs", + "ingested": "2022-07-18T15:10:15.862952759Z" }, "log": { "level": "DEBUG", diff --git a/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml b/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml index f201a199440..4e8a9875898 100644 --- a/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml @@ -63,12 +63,17 @@ processors: - remove: field: _ecs_json_message ignore_missing: true + - set: + field: event.ingested + value: "{{_ingest.timestamp}}" + ## some scenarios as update_aliases_succeeded type of operation, these 2 fields are present on the logs - remove: field: _tag ignore_missing: true - remove: field: right ignore_missing: true + on_failure: - set: field: error.message From 21524e84943e74f1c6b4da3b9e4c2982076a80a5 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 18 Jul 2022 17:32:36 +0200 Subject: [PATCH 14/18] Fix pipeline test --- .../kibana_audit/_dev/test/pipeline/test-common-config.yml | 2 ++ .../kibana_log/_dev/test/pipeline/test-common-config.yml | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-common-config.yml create mode 100644 packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-common-config.yml diff --git a/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-common-config.yml b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-common-config.yml b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" From 8cb34dae53a34ab63e35fdb72dedaca2fcdf0f14 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 19 Jul 2022 10:09:55 +0200 Subject: [PATCH 15/18] Extract ECS processor into a new file; Fix kibana logo; Add event.* fields --- .../test-kibana-audit-logs.log-expected.json | 3 +- .../elasticsearch/ingest_pipeline/default.yml | 39 +++++---------- .../ingest_pipeline/ecs-logs-pipeline.yml | 31 ++++++++++++ .../test-kibana-logs.log-expected.json | 10 ++-- .../elasticsearch/ingest_pipeline/default.yml | 47 ++++++++----------- .../ingest_pipeline/ecs-logs-pipeline.yml | 31 ++++++++++++ .../img/logo_kibana.svg | 10 ++-- 7 files changed, 105 insertions(+), 66 deletions(-) create mode 100644 packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/ecs-logs-pipeline.yml create mode 100644 packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/ecs-logs-pipeline.yml diff --git a/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json index aecd6a2a90e..e479b636cda 100644 --- a/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json +++ b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json @@ -16,7 +16,8 @@ "web" ], "dataset": "kibana-audit-log", - "ingested": "2022-07-18T15:10:15.717414176Z", + "ingested": "2022-07-19T08:03:36.301162346Z", + "kind": "event", "outcome": "unknown" }, "http": { diff --git a/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml b/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml index 3006e93d76d..88ebfe2fd6d 100644 --- a/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml @@ -7,6 +7,14 @@ processors: - remove: field: event.dataset ignore_missing: true + - pipeline: + name: '{{ IngestPipeline "ecs-logs-pipeline" }}' + if: |- + def message = ctx.message; + return message != null + && message.startsWith('{') + && message.endsWith('}') + && message.contains('"@timestamp"') - set: description: Uses event.dataset as a default for data_stream.dataset if the latter is not set. field: data_stream.dataset @@ -35,37 +43,12 @@ processors: - set: field: event.dataset copy_from: data_stream.dataset - - rename: - field: message - target_field: _ecs_json_message - if: |- - def message = ctx.message; - return message != null - && message.startsWith('{') - && message.endsWith('}') - && message.contains('"@timestamp"') - ignore_missing: true - - json: - field: _ecs_json_message - add_to_root: true - add_to_root_conflict_strategy: merge - allow_duplicate_keys: true - if: ctx.containsKey('_ecs_json_message') - on_failure: - - rename: - field: _ecs_json_message - target_field: message - ignore_missing: true - - set: - field: error.message - value: Error while parsing JSON - override: false - - remove: - field: _ecs_json_message - ignore_missing: true - set: field: event.ingested value: "{{_ingest.timestamp}}" + - set: + field: event.kind + value: event on_failure: - set: field: error.message diff --git a/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/ecs-logs-pipeline.yml b/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/ecs-logs-pipeline.yml new file mode 100644 index 00000000000..16db6d98a0f --- /dev/null +++ b/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/ecs-logs-pipeline.yml @@ -0,0 +1,31 @@ +--- +processors: + - rename: + field: message + target_field: _ecs_json_message + ignore_missing: true + - json: + field: _ecs_json_message + add_to_root: true + add_to_root_conflict_strategy: merge + allow_duplicate_keys: true + if: ctx.containsKey('_ecs_json_message') + on_failure: + - rename: + field: _ecs_json_message + target_field: message + ignore_missing: true + - set: + field: error.message + value: Error while parsing JSON + override: false + - remove: + field: _ecs_json_message + ignore_missing: true + - dot_expander: + field: "*" + override: true + - join: + field: error.stack_trace + separator: "\n" + if: ctx.error?.stack_trace instanceof Collection diff --git a/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json index 048bff76ec0..c2e19aac6e1 100644 --- a/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json +++ b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json @@ -12,7 +12,8 @@ }, "event": { "dataset": "kibana-logs", - "ingested": "2022-07-18T15:10:15.862948593Z" + "ingested": "2022-07-19T08:03:36.511837471Z", + "kind": "event" }, "log": { "level": "DEBUG", @@ -41,7 +42,9 @@ }, "event": { "dataset": "kibana-logs", - "ingested": "2022-07-18T15:10:15.862951843Z" + "ingested": "2022-07-19T08:03:36.511839513Z", + "kind": "event", + "outcome": "success" }, "http": { "request": { @@ -86,7 +89,8 @@ }, "event": { "dataset": "kibana-logs", - "ingested": "2022-07-18T15:10:15.862952759Z" + "ingested": "2022-07-19T08:03:36.511840346Z", + "kind": "event" }, "log": { "level": "DEBUG", diff --git a/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml b/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml index 4e8a9875898..c9a57de6ed1 100644 --- a/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml @@ -7,6 +7,14 @@ processors: - remove: field: event.dataset ignore_missing: true + - pipeline: + name: '{{ IngestPipeline "ecs-logs-pipeline" }}' + if: |- + def message = ctx.message; + return message != null + && message.startsWith('{') + && message.endsWith('}') + && message.contains('"@timestamp"') - set: description: Uses event.dataset as a default for data_stream.dataset if the latter is not set. field: data_stream.dataset @@ -35,37 +43,20 @@ processors: - set: field: event.dataset copy_from: data_stream.dataset - - rename: - field: message - target_field: _ecs_json_message - if: |- - def message = ctx.message; - return message != null - && message.startsWith('{') - && message.endsWith('}') - && message.contains('"@timestamp"') - ignore_missing: true - - json: - field: _ecs_json_message - add_to_root: true - add_to_root_conflict_strategy: merge - allow_duplicate_keys: true - if: ctx.containsKey('_ecs_json_message') - on_failure: - - rename: - field: _ecs_json_message - target_field: message - ignore_missing: true - - set: - field: error.message - value: Error while parsing JSON - override: false - - remove: - field: _ecs_json_message - ignore_missing: true - set: field: event.ingested value: "{{_ingest.timestamp}}" + - set: + field: event.kind + value: event + - set: + field: event.outcome + value: success + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + - set: + field: event.outcome + value: failure + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" ## some scenarios as update_aliases_succeeded type of operation, these 2 fields are present on the logs - remove: field: _tag diff --git a/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/ecs-logs-pipeline.yml b/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/ecs-logs-pipeline.yml new file mode 100644 index 00000000000..16db6d98a0f --- /dev/null +++ b/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/ecs-logs-pipeline.yml @@ -0,0 +1,31 @@ +--- +processors: + - rename: + field: message + target_field: _ecs_json_message + ignore_missing: true + - json: + field: _ecs_json_message + add_to_root: true + add_to_root_conflict_strategy: merge + allow_duplicate_keys: true + if: ctx.containsKey('_ecs_json_message') + on_failure: + - rename: + field: _ecs_json_message + target_field: message + ignore_missing: true + - set: + field: error.message + value: Error while parsing JSON + override: false + - remove: + field: _ecs_json_message + ignore_missing: true + - dot_expander: + field: "*" + override: true + - join: + field: error.stack_trace + separator: "\n" + if: ctx.error?.stack_trace instanceof Collection diff --git a/packages/platform_observability/img/logo_kibana.svg b/packages/platform_observability/img/logo_kibana.svg index bafebd9368c..8e5571acc1b 100644 --- a/packages/platform_observability/img/logo_kibana.svg +++ b/packages/platform_observability/img/logo_kibana.svg @@ -1,7 +1,5 @@ - - - - - - + + + + From 55c80e8eb8d4b0d8e433c121debf91166fc3f3b6 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 20 Jul 2022 10:39:47 +0200 Subject: [PATCH 16/18] Simplify ingest pipeline and clean-ups --- .../_dev/build/docs/README.md | 8 +- .../test-kibana-audit-logs.log-expected.json | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 22 ---- .../data_stream/kibana_audit/fields/ecs.yml | 13 ++- .../kibana_audit/sample_event.json | 48 ++++++++ .../_dev/test/pipeline/test-kibana-logs.log | 3 +- .../test-kibana-logs.log-expected.json | 34 +----- .../elasticsearch/ingest_pipeline/default.yml | 30 ----- .../data_stream/kibana_log/fields/ecs.yml | 9 +- .../data_stream/kibana_log/sample_event.json | 36 ++++++ .../platform_observability/docs/README.md | 108 +++++++++++++++++- 11 files changed, 220 insertions(+), 93 deletions(-) create mode 100644 packages/platform_observability/data_stream/kibana_audit/sample_event.json create mode 100644 packages/platform_observability/data_stream/kibana_log/sample_event.json diff --git a/packages/platform_observability/_dev/build/docs/README.md b/packages/platform_observability/_dev/build/docs/README.md index 721c56e8717..036ddc400d8 100644 --- a/packages/platform_observability/_dev/build/docs/README.md +++ b/packages/platform_observability/_dev/build/docs/README.md @@ -12,12 +12,16 @@ The Kibana integration collects logs from [Kibana](https://www.elastic.co/guide/ #### Audit -Configure `Path` pointing to the location where audit logs will be created, based on the [Kibana Audit logging settings](https://www.elastic.co/guide/en/kibana/current/security-settings-kb.html#audit-logging-settings) in `kibana.yml` +Audit logs collects the [Kibana audit logs](https://www.elastic.co/guide/en/kibana/current/security-settings-kb.html). + +{{event "kibana_audit"}} {{fields "kibana_audit"}} #### Log -Configure `Path` pointing to the location where the logs will be created, based on the [Kibana logging settings](https://www.elastic.co/guide/en/kibana/current/logging-configuration.html#logging-appenders) in `kibana.yml` +Log collects the [Kibana logs](https://www.elastic.co/guide/en/kibana/current/logging-configuration.html). + +{{event "kibana_log"}} {{fields "kibana_log"}} diff --git a/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json index e479b636cda..b5f9281a2f1 100644 --- a/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json +++ b/packages/platform_observability/data_stream/kibana_audit/_dev/test/pipeline/test-kibana-audit-logs.log-expected.json @@ -16,7 +16,7 @@ "web" ], "dataset": "kibana-audit-log", - "ingested": "2022-07-19T08:03:36.301162346Z", + "ingested": "2022-07-20T08:36:57.202942842Z", "kind": "event", "outcome": "unknown" }, diff --git a/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml b/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml index 88ebfe2fd6d..45755cf29fa 100644 --- a/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform_observability/data_stream/kibana_audit/elasticsearch/ingest_pipeline/default.yml @@ -1,12 +1,6 @@ --- description: Pipeline for parsing Kibana Audit ECS formatted logs processors: - - remove: - field: data_stream.dataset - ignore_missing: true - - remove: - field: event.dataset - ignore_missing: true - pipeline: name: '{{ IngestPipeline "ecs-logs-pipeline" }}' if: |- @@ -15,31 +9,15 @@ processors: && message.startsWith('{') && message.endsWith('}') && message.contains('"@timestamp"') - - set: - description: Uses event.dataset as a default for data_stream.dataset if the latter is not set. - field: data_stream.dataset - copy_from: event.dataset - if: ctx.event?.dataset instanceof String && ctx.event.dataset.length() > 1 - override: false - - script: - source: | - ctx.data_stream.dataset = /[\/*?"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_') - if: ctx.data_stream?.dataset != null - - script: - source: | - ctx.data_stream.namespace = /[\/*?"<>|, #:]/.matcher(ctx.data_stream.namespace).replaceAll('_') - if: ctx.data_stream?.namespace != null - set: field: data_stream.type value: logs - set: field: data_stream.dataset value: kibana-audit-log - override: false - set: field: data_stream.namespace value: platform-observability - override: false - set: field: event.dataset copy_from: data_stream.dataset diff --git a/packages/platform_observability/data_stream/kibana_audit/fields/ecs.yml b/packages/platform_observability/data_stream/kibana_audit/fields/ecs.yml index de3db8f9ab4..88a28b751bb 100644 --- a/packages/platform_observability/data_stream/kibana_audit/fields/ecs.yml +++ b/packages/platform_observability/data_stream/kibana_audit/fields/ecs.yml @@ -1,6 +1,17 @@ -# only used for tests - name: ecs.version external: ecs +- name: event.action + external: ecs +- name: event.category + external: ecs +- name: event.dataset + external: ecs +- name: event.kind + external: ecs +- name: event.ingested + external: ecs +- name: event.outcome + external: ecs - name: http.request.method external: ecs - name: log.level diff --git a/packages/platform_observability/data_stream/kibana_audit/sample_event.json b/packages/platform_observability/data_stream/kibana_audit/sample_event.json new file mode 100644 index 00000000000..85b071b660b --- /dev/null +++ b/packages/platform_observability/data_stream/kibana_audit/sample_event.json @@ -0,0 +1,48 @@ +{ + "event": { + "action": "http_request", + "category": [ + "web" + ], + "outcome": "unknown" + }, + "http": { + "request": { + "method": "get" + } + }, + "url": { + "domain": "localhost", + "path": "/internal/security/session", + "port": 5601, + "scheme": "http" + }, + "user": { + "name": "elastic", + "roles": [ + "superuser" + ] + }, + "kibana": { + "space_id": "default", + "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=" + }, + "trace": { + "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9" + }, + "ecs": { + "version": "8.0.0" + }, + "@timestamp": "2022-06-29T12:05:03.742+00:00", + "message": "User is requesting [/internal/security/session] endpoint", + "log": { + "level": "INFO", + "logger": "plugins.security.audit.ecs" + }, + "process": { + "pid": 7 + }, + "transaction": { + "id": "f8863d86567119e6" + } +} \ No newline at end of file diff --git a/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log index a49dd498911..3a1f92f7649 100644 --- a/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log +++ b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log @@ -1,3 +1,2 @@ {"ecs":{"version":"8.0.0"},"@timestamp":"2022-06-29T11:24:17.898+00:00","message":"Trying to authenticate user request to /login.","log":{"level":"DEBUG","logger":"plugins.security.http"},"process":{"pid":7},"trace":{"id":"e6e1c25936546ec690b11a3b78b2a8db"},"transaction":{"id":"3be6994d7f6d5465"}} -{"http":{"request":{"id":"unknownId","method":"GET"},"response":{"body":{"bytes":118},"status_code":200}},"url":{"path":"/_nodes","query":"filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-07-14T10:35:25.366+00:00","message":"200 - 118.0B\nGET /_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip","log":{"level":"DEBUG","logger":"elasticsearch.query.data"},"process":{"pid":7},"trace":{"id":"0cd8dd5a3483159a43c07e9205432775"},"transaction":{"id":"6301eca88fba8d99"}} -{"_tag":"Right","right":"update_aliases_succeeded","ecs":{"version":"8.0.0"},"@timestamp":"2022-07-04T09:17:38.611+00:00","message":"[.kibana] MARK_VERSION_INDEX_READY RESPONSE","log":{"level":"DEBUG","logger":"savedobjects-service"},"process":{"pid":7},"trace":{"id":"a167d1124764379d4121b357e20baee2"},"transaction":{"id":"14717ae6e3b30d5a"}} \ No newline at end of file +{"http":{"request":{"id":"unknownId","method":"GET"},"response":{"body":{"bytes":118},"status_code":200}},"url":{"path":"/_nodes","query":"filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-07-14T10:35:25.366+00:00","message":"200 - 118.0B\nGET /_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip","log":{"level":"DEBUG","logger":"elasticsearch.query.data"},"process":{"pid":7},"trace":{"id":"0cd8dd5a3483159a43c07e9205432775"},"transaction":{"id":"6301eca88fba8d99"}} \ No newline at end of file diff --git a/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json index c2e19aac6e1..a341db9656d 100644 --- a/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json +++ b/packages/platform_observability/data_stream/kibana_log/_dev/test/pipeline/test-kibana-logs.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "dataset": "kibana-logs", - "ingested": "2022-07-19T08:03:36.511837471Z", + "ingested": "2022-07-20T08:36:57.436382217Z", "kind": "event" }, "log": { @@ -42,7 +42,7 @@ }, "event": { "dataset": "kibana-logs", - "ingested": "2022-07-19T08:03:36.511839513Z", + "ingested": "2022-07-20T08:36:57.436384342Z", "kind": "event", "outcome": "success" }, @@ -76,36 +76,6 @@ "path": "/_nodes", "query": "filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip" } - }, - { - "@timestamp": "2022-07-04T09:17:38.611+00:00", - "data_stream": { - "dataset": "kibana-logs", - "namespace": "platform-observability", - "type": "logs" - }, - "ecs": { - "version": "8.0.0" - }, - "event": { - "dataset": "kibana-logs", - "ingested": "2022-07-19T08:03:36.511840346Z", - "kind": "event" - }, - "log": { - "level": "DEBUG", - "logger": "savedobjects-service" - }, - "message": "[.kibana] MARK_VERSION_INDEX_READY RESPONSE", - "process": { - "pid": 7 - }, - "trace": { - "id": "a167d1124764379d4121b357e20baee2" - }, - "transaction": { - "id": "14717ae6e3b30d5a" - } } ] } \ No newline at end of file diff --git a/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml b/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml index c9a57de6ed1..e7070f68af6 100644 --- a/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/platform_observability/data_stream/kibana_log/elasticsearch/ingest_pipeline/default.yml @@ -1,12 +1,6 @@ --- description: Pipeline for parsing Kibana ECS formatted logs processors: - - remove: - field: data_stream.dataset - ignore_missing: true - - remove: - field: event.dataset - ignore_missing: true - pipeline: name: '{{ IngestPipeline "ecs-logs-pipeline" }}' if: |- @@ -15,31 +9,15 @@ processors: && message.startsWith('{') && message.endsWith('}') && message.contains('"@timestamp"') - - set: - description: Uses event.dataset as a default for data_stream.dataset if the latter is not set. - field: data_stream.dataset - copy_from: event.dataset - if: ctx.event?.dataset instanceof String && ctx.event.dataset.length() > 1 - override: false - - script: - source: | - ctx.data_stream.dataset = /[\/*?"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_') - if: ctx.data_stream?.dataset != null - - script: - source: | - ctx.data_stream.namespace = /[\/*?"<>|, #:]/.matcher(ctx.data_stream.namespace).replaceAll('_') - if: ctx.data_stream?.namespace != null - set: field: data_stream.type value: logs - set: field: data_stream.dataset value: kibana-logs - override: false - set: field: data_stream.namespace value: platform-observability - override: false - set: field: event.dataset copy_from: data_stream.dataset @@ -57,14 +35,6 @@ processors: field: event.outcome value: failure if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" - ## some scenarios as update_aliases_succeeded type of operation, these 2 fields are present on the logs - - remove: - field: _tag - ignore_missing: true - - remove: - field: right - ignore_missing: true - on_failure: - set: field: error.message diff --git a/packages/platform_observability/data_stream/kibana_log/fields/ecs.yml b/packages/platform_observability/data_stream/kibana_log/fields/ecs.yml index f070b041162..6335dc892e4 100644 --- a/packages/platform_observability/data_stream/kibana_log/fields/ecs.yml +++ b/packages/platform_observability/data_stream/kibana_log/fields/ecs.yml @@ -1,6 +1,13 @@ -# only used for tests - name: ecs.version external: ecs +- name: event.dataset + external: ecs +- name: event.kind + external: ecs +- name: event.ingested + external: ecs +- name: event.outcome + external: ecs - name: http.request.id external: ecs - name: http.request.method diff --git a/packages/platform_observability/data_stream/kibana_log/sample_event.json b/packages/platform_observability/data_stream/kibana_log/sample_event.json new file mode 100644 index 00000000000..cc27916d3ef --- /dev/null +++ b/packages/platform_observability/data_stream/kibana_log/sample_event.json @@ -0,0 +1,36 @@ +{ + "http": { + "request": { + "id": "unknownId", + "method": "GET" + }, + "response": { + "body": { + "bytes": 118 + }, + "status_code": 200 + } + }, + "url": { + "path": "/_nodes", + "query": "filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip" + }, + "ecs": { + "version": "8.0.0" + }, + "@timestamp": "2022-07-14T10:35:25.366+00:00", + "message": "200 - 118.0B\nGET /_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", + "log": { + "level": "DEBUG", + "logger": "elasticsearch.query.data" + }, + "process": { + "pid": 7 + }, + "trace": { + "id": "0cd8dd5a3483159a43c07e9205432775" + }, + "transaction": { + "id": "6301eca88fba8d99" + } +} \ No newline at end of file diff --git a/packages/platform_observability/docs/README.md b/packages/platform_observability/docs/README.md index ca6787c3f96..bd116db6eca 100644 --- a/packages/platform_observability/docs/README.md +++ b/packages/platform_observability/docs/README.md @@ -12,7 +12,60 @@ The Kibana integration collects logs from [Kibana](https://www.elastic.co/guide/ #### Audit -Configure `Path` pointing to the location where audit logs will be created, based on the [Kibana Audit logging settings](https://www.elastic.co/guide/en/kibana/current/security-settings-kb.html#audit-logging-settings) in `kibana.yml` +Audit logs collects the [Kibana audit logs](https://www.elastic.co/guide/en/kibana/current/security-settings-kb.html). + +An example event for `kibana_audit` looks as following: + +```json +{ + "event": { + "action": "http_request", + "category": [ + "web" + ], + "outcome": "unknown" + }, + "http": { + "request": { + "method": "get" + } + }, + "url": { + "domain": "localhost", + "path": "/internal/security/session", + "port": 5601, + "scheme": "http" + }, + "user": { + "name": "elastic", + "roles": [ + "superuser" + ] + }, + "kibana": { + "space_id": "default", + "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=" + }, + "trace": { + "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9" + }, + "ecs": { + "version": "8.0.0" + }, + "@timestamp": "2022-06-29T12:05:03.742+00:00", + "message": "User is requesting [/internal/security/session] endpoint", + "log": { + "level": "INFO", + "logger": "plugins.security.audit.ecs" + }, + "process": { + "pid": 7 + }, + "transaction": { + "id": "f8863d86567119e6" + } +} +``` **Exported fields** @@ -23,6 +76,12 @@ Configure `Path` pointing to the location where audit logs will be created, base | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | | kibana.add_to_spaces | The set of space ids that a saved object was shared to. | keyword | | kibana.authentication_provider | The authentication provider associated with a login event. | keyword | @@ -52,7 +111,48 @@ Configure `Path` pointing to the location where audit logs will be created, base #### Log -Configure `Path` pointing to the location where the logs will be created, based on the [Kibana logging settings](https://www.elastic.co/guide/en/kibana/current/logging-configuration.html#logging-appenders) in `kibana.yml` +Log collects the [Kibana logs](https://www.elastic.co/guide/en/kibana/current/logging-configuration.html). + +An example event for `kibana_log` looks as following: + +```json +{ + "http": { + "request": { + "id": "unknownId", + "method": "GET" + }, + "response": { + "body": { + "bytes": 118 + }, + "status_code": 200 + } + }, + "url": { + "path": "/_nodes", + "query": "filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip" + }, + "ecs": { + "version": "8.0.0" + }, + "@timestamp": "2022-07-14T10:35:25.366+00:00", + "message": "200 - 118.0B\nGET /_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", + "log": { + "level": "DEBUG", + "logger": "elasticsearch.query.data" + }, + "process": { + "pid": 7 + }, + "trace": { + "id": "0cd8dd5a3483159a43c07e9205432775" + }, + "transaction": { + "id": "6301eca88fba8d99" + } +} +``` **Exported fields** @@ -63,6 +163,10 @@ Configure `Path` pointing to the location where the logs will be created, based | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | http.request.id | A unique identifier for each HTTP request to correlate logs between clients and servers in transactions. The id may be contained in a non-standard HTTP header, such as `X-Request-ID` or `X-Correlation-ID`. | keyword | | http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | | http.response.body.bytes | Size in bytes of the response body. | long | From 5dcc1fefb6c9b5b93568fbb04bc317ebaa605c2f Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Wed, 20 Jul 2022 11:04:13 +0200 Subject: [PATCH 17/18] Fix json format --- .../kibana_audit/sample_event.json | 90 +++++++++---------- .../data_stream/kibana_log/sample_event.json | 64 ++++++------- 2 files changed, 77 insertions(+), 77 deletions(-) diff --git a/packages/platform_observability/data_stream/kibana_audit/sample_event.json b/packages/platform_observability/data_stream/kibana_audit/sample_event.json index 85b071b660b..83e716705d4 100644 --- a/packages/platform_observability/data_stream/kibana_audit/sample_event.json +++ b/packages/platform_observability/data_stream/kibana_audit/sample_event.json @@ -1,48 +1,48 @@ { - "event": { - "action": "http_request", - "category": [ - "web" - ], - "outcome": "unknown" - }, - "http": { - "request": { - "method": "get" + "event": { + "action": "http_request", + "category": [ + "web" + ], + "outcome": "unknown" + }, + "http": { + "request": { + "method": "get" + } + }, + "url": { + "domain": "localhost", + "path": "/internal/security/session", + "port": 5601, + "scheme": "http" + }, + "user": { + "name": "elastic", + "roles": [ + "superuser" + ] + }, + "kibana": { + "space_id": "default", + "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=" + }, + "trace": { + "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9" + }, + "ecs": { + "version": "8.0.0" + }, + "@timestamp": "2022-06-29T12:05:03.742+00:00", + "message": "User is requesting [/internal/security/session] endpoint", + "log": { + "level": "INFO", + "logger": "plugins.security.audit.ecs" + }, + "process": { + "pid": 7 + }, + "transaction": { + "id": "f8863d86567119e6" } - }, - "url": { - "domain": "localhost", - "path": "/internal/security/session", - "port": 5601, - "scheme": "http" - }, - "user": { - "name": "elastic", - "roles": [ - "superuser" - ] - }, - "kibana": { - "space_id": "default", - "session_id": "ccZ0sbxrmmJwo+/y2Mn1tmGIrKOuZYaF8voUh0SkA/k=" - }, - "trace": { - "id": "1c8c5808-d2d6-41fc-8cb7-998aa8996be9" - }, - "ecs": { - "version": "8.0.0" - }, - "@timestamp": "2022-06-29T12:05:03.742+00:00", - "message": "User is requesting [/internal/security/session] endpoint", - "log": { - "level": "INFO", - "logger": "plugins.security.audit.ecs" - }, - "process": { - "pid": 7 - }, - "transaction": { - "id": "f8863d86567119e6" - } } \ No newline at end of file diff --git a/packages/platform_observability/data_stream/kibana_log/sample_event.json b/packages/platform_observability/data_stream/kibana_log/sample_event.json index cc27916d3ef..53f1d0d99eb 100644 --- a/packages/platform_observability/data_stream/kibana_log/sample_event.json +++ b/packages/platform_observability/data_stream/kibana_log/sample_event.json @@ -1,36 +1,36 @@ { - "http": { - "request": { - "id": "unknownId", - "method": "GET" + "http": { + "request": { + "id": "unknownId", + "method": "GET" + }, + "response": { + "body": { + "bytes": 118 + }, + "status_code": 200 + } }, - "response": { - "body": { - "bytes": 118 - }, - "status_code": 200 + "url": { + "path": "/_nodes", + "query": "filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip" + }, + "ecs": { + "version": "8.0.0" + }, + "@timestamp": "2022-07-14T10:35:25.366+00:00", + "message": "200 - 118.0B\nGET /_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", + "log": { + "level": "DEBUG", + "logger": "elasticsearch.query.data" + }, + "process": { + "pid": 7 + }, + "trace": { + "id": "0cd8dd5a3483159a43c07e9205432775" + }, + "transaction": { + "id": "6301eca88fba8d99" } - }, - "url": { - "path": "/_nodes", - "query": "filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip" - }, - "ecs": { - "version": "8.0.0" - }, - "@timestamp": "2022-07-14T10:35:25.366+00:00", - "message": "200 - 118.0B\nGET /_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", - "log": { - "level": "DEBUG", - "logger": "elasticsearch.query.data" - }, - "process": { - "pid": 7 - }, - "trace": { - "id": "0cd8dd5a3483159a43c07e9205432775" - }, - "transaction": { - "id": "6301eca88fba8d99" - } } \ No newline at end of file From 27a23aa1931ad0ac59b588155125d804c799fbd2 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Fri, 22 Jul 2022 09:05:12 +0200 Subject: [PATCH 18/18] Align default log file name with kibana docs --- .../platform_observability/data_stream/kibana_log/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/platform_observability/data_stream/kibana_log/manifest.yml b/packages/platform_observability/data_stream/kibana_log/manifest.yml index 00533fb01dd..cbd6fc42457 100644 --- a/packages/platform_observability/data_stream/kibana_log/manifest.yml +++ b/packages/platform_observability/data_stream/kibana_log/manifest.yml @@ -10,7 +10,7 @@ streams: required: true show_user: true default: - - /var/log/kibana/kibana.stdout + - /var/log/kibana/kibana.log template_path: log.yml.hbs title: Kibana logs description: Collect Kibana logs