diff --git a/packages/ecs_log/changelog.yml b/packages/ecs_log/changelog.yml new file mode 100644 index 00000000000..4faa1f186b4 --- /dev/null +++ b/packages/ecs_log/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial release of the package + type: enhancement + link: https://github.com/elastic/integrations/pull diff --git a/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log new file mode 100644 index 00000000000..112a60586a9 --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log @@ -0,0 +1,10 @@ +{"@timestamp":"2022-04-01T12:09:12.375Z", "log.level": "INFO", "message":"With event.dataset", "event.dataset": "foo"} +{"@timestamp":"2022-04-01T12:09:12.379Z", "log.level": "INFO", "message":"With data_stream.dataset", "data_stream.dataset": "bar"} +{"@timestamp":"2022-04-01T12:09:12.379Z", "log.level": "INFO", "message":"With invalid chars in dataset", "data_stream.dataset": "my-service"} +{"@timestamp":"2022-04-01T14:08:40.199Z", "log.level":"DEBUG", "message":"Without dataset"} +{"@timestamp":"2022-04-01T14:08:40.199Z", "log.level":"DEBUG", "message":"With stack trace", "error.stack_trace": "Exception in thread \"main\" java.lang.NullPointerException\n at com.example.myproject.Book.getTitle(Book.java:16)\n at com.example.myproject.Author.getBookTitles(Author.java:25)\n at com.example.myproject.Bootstrap.main(Bootstrap.java:14)"} +{"@timestamp":"2022-04-01T14:08:40.199Z", "log.level":"DEBUG", "message":"With stack trace as array", "error.stack_trace": [ + "Exception in thread \"main\" java.lang.NullPointerException\n", + " at com.example.myproject.Book.getTitle(Book.java:16)\n", + " at com.example.myproject.Author.getBookTitles(Author.java:25)\n", + " at com.example.myproject.Bootstrap.main(Bootstrap.java:14)"]} diff --git a/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-config.yml b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-config.yml new file mode 100644 index 00000000000..7464e902d0a --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-config.yml @@ -0,0 +1,11 @@ +multiline: + first_line_pattern: '^{' +fields: + ecs: + version: "1.5.0" + event: + dataset: ecs_router + data_stream: + type: logs + dataset: ecs_router + namespace: default diff --git a/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-expected.json b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-expected.json new file mode 100644 index 00000000000..de66933eb4a --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/_dev/test/pipeline/test-ecs-json.log-expected.json @@ -0,0 +1,118 @@ +{ + "expected": [ + { + "@timestamp": "2022-04-01T12:09:12.375Z", + "data_stream": { + "dataset": "foo", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "foo" + }, + "log": { + "level": "INFO" + }, + "message": "With event.dataset" + }, + { + "@timestamp": "2022-04-01T12:09:12.379Z", + "data_stream": { + "dataset": "bar", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "bar" + }, + "log": { + "level": "INFO" + }, + "message": "With data_stream.dataset" + }, + { + "@timestamp": "2022-04-01T12:09:12.379Z", + "data_stream": { + "dataset": "my_service", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "my_service" + }, + "log": { + "level": "INFO" + }, + "message": "With invalid chars in dataset" + }, + { + "@timestamp": "2022-04-01T14:08:40.199Z", + "data_stream": { + "dataset": "generic", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "generic" + }, + "log": { + "level": "DEBUG" + }, + "message": "Without dataset" + }, + { + "@timestamp": "2022-04-01T14:08:40.199Z", + "data_stream": { + "dataset": "generic", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "error": { + "stack_trace": "Exception in thread \"main\" java.lang.NullPointerException\n at com.example.myproject.Book.getTitle(Book.java:16)\n at com.example.myproject.Author.getBookTitles(Author.java:25)\n at com.example.myproject.Bootstrap.main(Bootstrap.java:14)" + }, + "event": { + "dataset": "generic" + }, + "log": { + "level": "DEBUG" + }, + "message": "With stack trace" + }, + { + "@timestamp": "2022-04-01T14:08:40.199Z", + "data_stream": { + "dataset": "generic", + "namespace": "default", + "type": "logs" + }, + "ecs": { + "version": "1.5.0" + }, + "error": { + "stack_trace": "Exception in thread \"main\" java.lang.NullPointerException\n\n at com.example.myproject.Book.getTitle(Book.java:16)\n\n at com.example.myproject.Author.getBookTitles(Author.java:25)\n\n at com.example.myproject.Bootstrap.main(Bootstrap.java:14)" + }, + "event": { + "dataset": "generic" + }, + "log": { + "level": "DEBUG" + }, + "message": "With stack trace as array" + } + ] +} diff --git a/packages/ecs_log/data_stream/ecs_router/agent/stream/stream.yml.hbs b/packages/ecs_log/data_stream/ecs_router/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..6055afd8c57 --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/agent/stream/stream.yml.hbs @@ -0,0 +1,11 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} + +processors: + - add_host_metadata: ~ + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ +{{custom}} diff --git a/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/default.yml b/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..fd56ce6b980 --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,50 @@ +--- +processors: +- remove: + description: | + This data stream is meant for routing only, we want to avoid that data is written to it. + We'll use the dataset that is specified in the ECS JSON log message, or use 'generic' as the default. + field: data_stream.dataset + ignore_missing: true +- remove: + field: event.dataset + ignore_missing: true +- pipeline: + name: '{{ IngestPipeline "logs-ecs-json-pipeline" }}' + if: |- + def message = ctx.message; + return message != null + && message.startsWith('{') + && message.endsWith('}') + && message.contains('"@timestamp"') +- set: + description: Uses event.dataset as a default for data_stream.dataset if the latter is not set. + field: data_stream.dataset + copy_from: event.dataset + if: ctx.event?.dataset instanceof String && ctx.event.dataset.length() > 1 + override: false +- script: + source: | + ctx.data_stream.dataset = /[\/*?"<>|, #:-]/.matcher(ctx.data_stream.dataset).replaceAll('_') + if: ctx.data_stream?.dataset != null +- script: + source: | + ctx.data_stream.namespace = /[\/*?"<>|, #:]/.matcher(ctx.data_stream.namespace).replaceAll('_') + if: ctx.data_stream?.namespace != null +- set: + field: data_stream.type + value: logs +- set: + field: data_stream.dataset + value: generic + override: false +- set: + field: data_stream.namespace + value: default + override: false +- set: + field: event.dataset + copy_from: data_stream.dataset +- set: + field: _index + value: logs-{{{data_stream.dataset}}}-{{{data_stream.namespace}}} diff --git a/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/logs-ecs-json-pipeline.yml b/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/logs-ecs-json-pipeline.yml new file mode 100644 index 00000000000..55d5b64e9a9 --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/elasticsearch/ingest_pipeline/logs-ecs-json-pipeline.yml @@ -0,0 +1,31 @@ +--- +processors: +- rename: + field: message + target_field: _ecs_json_message + ignore_missing: true +- json: + field: _ecs_json_message + add_to_root: true + add_to_root_conflict_strategy: merge + allow_duplicate_keys: true + if: ctx.containsKey('_ecs_json_message') + on_failure: + - rename: + field: _ecs_json_message + target_field: message + ignore_missing: true + - set: + field: error.message + value: Error while parsing JSON + override: false +- remove: + field: _ecs_json_message + ignore_missing: true +- dot_expander: + field: "*" + override: true +- join: + field: error.stack_trace + separator: "\n" + if: ctx.error?.stack_trace instanceof Collection diff --git a/packages/ecs_log/data_stream/ecs_router/fields/base-fields.yml b/packages/ecs_log/data_stream/ecs_router/fields/base-fields.yml new file mode 100644 index 00000000000..9de1ddcbf8b --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/fields/base-fields.yml @@ -0,0 +1,23 @@ +# no data is meant to be written to the data stream +# however, in order for package validation and tests to pass, we need to add some fields +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +# only used for tests +- name: ecs.version + external: ecs +- name: error.stack_trace + external: ecs +- name: log.level + external: ecs +- name: message + external: ecs diff --git a/packages/ecs_log/data_stream/ecs_router/manifest.yml b/packages/ecs_log/data_stream/ecs_router/manifest.yml new file mode 100644 index 00000000000..d9f31ea371f --- /dev/null +++ b/packages/ecs_log/data_stream/ecs_router/manifest.yml @@ -0,0 +1,21 @@ +title: ECS Log Router Dataset +type: logs +dataset: ecs_router +streams: + - input: logfile + description: Collect your custom log files. + title: Collect log files + vars: + - name: paths + required: true + title: Log file path + description: Path to log files to be collected + type: text + multi: true + - name: custom + title: Custom configurations + description: > + Here YAML configuration options can be used to be added to your configuration. Be careful using this as it might break your configuration file. + + type: yaml + default: "" diff --git a/packages/ecs_log/docs/README.md b/packages/ecs_log/docs/README.md new file mode 100644 index 00000000000..89c5fab8171 --- /dev/null +++ b/packages/ecs_log/docs/README.md @@ -0,0 +1,3 @@ +# ECS Log Package + +The ECS log package is used to ingest ECS log files. diff --git a/packages/ecs_log/img/icon.svg b/packages/ecs_log/img/icon.svg new file mode 100644 index 00000000000..173fdec5072 --- /dev/null +++ b/packages/ecs_log/img/icon.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/packages/ecs_log/manifest.yml b/packages/ecs_log/manifest.yml new file mode 100644 index 00000000000..3e42720317d --- /dev/null +++ b/packages/ecs_log/manifest.yml @@ -0,0 +1,24 @@ +format_version: 1.0.0 +name: ecs_log +title: ECS Logs +description: >- + Collect ECS logs with Elastic Agent. +type: integration +version: 0.0.1 +release: experimental +license: basic +categories: + - custom +policy_templates: + - name: logs + title: ECS logs + description: Collect your ECS log files. + inputs: + - type: logfile + title: ECS log file + description: Collect your ECS log files. +icons: + - src: "/img/icon.svg" + type: "image/svg+xml" +owner: + github: elastic/integrations