diff --git a/packages/cisco_meraki_metrics/_dev/build/docs/README.md b/packages/cisco_meraki_metrics/_dev/build/docs/README.md index 1a55ae9e0a2..28b93f79877 100644 --- a/packages/cisco_meraki_metrics/_dev/build/docs/README.md +++ b/packages/cisco_meraki_metrics/_dev/build/docs/README.md @@ -43,3 +43,15 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur {{fields "device_health"}} {{event "device_health"}} + +### Network Health + +The `network_health` dataset provides metrics related to the overall health and performance of Meraki networks. + +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + +{{fields "network_health"}} + +{{event "network_health"}} diff --git a/packages/cisco_meraki_metrics/changelog.yml b/packages/cisco_meraki_metrics/changelog.yml index a829163d711..df496880ac3 100644 --- a/packages/cisco_meraki_metrics/changelog.yml +++ b/packages/cisco_meraki_metrics/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.5.0" + changes: + - description: Add `network_health` data stream + type: enhancement + link: https://github.com/elastic/integrations/pull/15663 - version: "0.4.1" changes: - description: scale values in device channel utilization so they display correctly as percentages. diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs b/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..a999d22fdf8 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs @@ -0,0 +1,22 @@ +metricsets: ["network_health"] +apiBaseUrl: {{apiBaseUrl}} +apiKey: {{apiKey}} +organizations: +{{#each organizations as |organization|}} + - "{{organization}}" +{{/each}} +period: {{period}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..a9620ec0706 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,66 @@ +--- +description: Pipeline for parsing Cisco Meraki Network Health metrics. +processors: + - script: + lang: painless + source: > + // some values have unit 'percent' in the mappings; we need to scale them down from 0->100 to 0->1. + // we round to 4 decimal places to avoid floating point errors. + + if (ctx.meraki != null) { + if (ctx.meraki.uplink != null && ctx.meraki.uplink.loss != null && ctx.meraki.uplink.loss.pct != null) { + ctx.meraki.uplink.loss.pct = Math.round((ctx.meraki.uplink.loss.pct / 100) * 10000) / 10000.0; + } + + if (ctx.meraki.device != null && ctx.meraki.device.channel_utilization != null) { + def wifi0 = ctx.meraki.device.channel_utilization["2_4"]; + def wifi1 = ctx.meraki.device.channel_utilization["5"]; + + if (wifi0 != null) { + if (wifi0.utilization_80211 != null) { + wifi0.utilization_80211 = Math.round((wifi0.utilization_80211 / 100) * 10000) / 10000.0; + } + if (wifi0.utilization_non_80211 != null) { + wifi0.utilization_non_80211 = Math.round((wifi0.utilization_non_80211 / 100) * 10000) / 10000.0; + } + if (wifi0.utilization_total != null) { + wifi0.utilization_total = Math.round((wifi0.utilization_total / 100) * 10000) / 10000.0; + } + } + + if (wifi1 != null) { + if (wifi1.utilization_80211 != null) { + wifi1.utilization_80211 = Math.round((wifi1.utilization_80211 / 100) * 10000) / 10000.0; + } + if (wifi1.utilization_non_80211 != null) { + wifi1.utilization_non_80211 = Math.round((wifi1.utilization_non_80211 / 100) * 10000) / 10000.0; + } + if (wifi1.utilization_total != null) { + wifi1.utilization_total = Math.round((wifi1.utilization_total / 100) * 10000) / 10000.0; + } + } + } + } + - convert: + field: meraki.uplink.rsrp + type: float + ignore_missing: true + - convert: + field: meraki.uplink.rsrq + type: float + ignore_missing: true + - rename: + field: meraki.device.channel_utilization.2_4 + target_field: meraki.device.channel_utilization.wifi0 + ignore_missing: true + - rename: + field: meraki.device.channel_utilization.5 + target_field: meraki.device.channel_utilization.wifi1 + ignore_missing: true +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml new file mode 100644 index 00000000000..b549d5382f3 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml @@ -0,0 +1,23 @@ +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + dimension: true + description: 'Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' \ No newline at end of file diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml new file mode 100644 index 00000000000..2e74d6d8da3 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml @@ -0,0 +1,30 @@ +- name: meraki + type: group + fields: + - name: organization_id + type: keyword + dimension: true + - name: network + type: group + fields: + - name: id + type: keyword + dimension: true + - name: name + type: keyword + dimension: true + - name: vpn_peers + type: group + fields: + - name: network_id + type: keyword + dimension: true + - name: network_name + type: keyword + dimension: true + - name: usage_summary.received.bytes + type: long + unit: byte + - name: usage_summary.sent.bytes + type: long + unit: byte \ No newline at end of file diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml b/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml new file mode 100644 index 00000000000..5fad1940a6d --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml @@ -0,0 +1,44 @@ +title: "Cisco Meraki Network Health Metrics" +type: metrics +streams: + - input: meraki/metrics + title: Cisco Meraki Network Health Metrics + description: Collect network health metrics from the Meraki Dashboard API with Elastic Agent. + vars: + - name: apiKey + type: text + title: Meraki Dashboard API key + secret: true + required: true + - name: organizations + type: text + title: Meraki organization IDs + multi: true + required: true + - name: apiBaseUrl + type: url + title: Meraki Dashboard API base URL + default: api.meraki.com + - name: period + type: text + title: Collection interval + default: 60s + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - cisco_meraki_metrics-network_health + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. +elasticsearch: + source_mode: synthetic + index_mode: time_series diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json b/packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json new file mode 100644 index 00000000000..57d11dc63f0 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json @@ -0,0 +1,76 @@ +{ + "@timestamp": "2024-09-30T16:55:38.202Z", + "agent": { + "ephemeral_id": "11855dde-6a4a-48ce-ac32-087b1c7999a3", + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.15.2" + }, + "data_stream": { + "dataset": "cisco_meraki_metrics.network_health", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "snapshot": true, + "version": "8.15.2" + }, + "event": { + "agent_id_status": "verified", + "dataset": "cisco_meraki_metrics.network_health", + "duration": 12982553765, + "ingested": "2024-09-30T16:56:01Z", + "module": "meraki" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "c7f0ac74f5e24f78942164132c2c8ead", + "ip": "172.21.0.4", + "mac": "02-42-AC-15-00-04", + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "6.8.0-45-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" + } + }, + "meraki": { + "organization_id": "125432", + "network": { + "name": "BKYHUM", + "vpn_peers": [ + { + "network_id": "N_837204569103482715", + "network_name": "ZXVRNE", + "usage_summary.received.bytes": 12288, + "usage_summary.sent.bytes": 12288 + }, + { + "network_id": "N_294175608239471063", + "network_name": "QWMTJL", + "usage_summary.received.bytes": 0, + "usage_summary.sent.bytes": 79872 + } + ], + "id": "L_760194835627109284" + } + }, + "metricset": { + "name": "network_health", + "period": 60000 + }, + "service": { + "type": "meraki" + } +} \ No newline at end of file diff --git a/packages/cisco_meraki_metrics/docs/README.md b/packages/cisco_meraki_metrics/docs/README.md index 9f49428bf64..ef462bc4bf1 100644 --- a/packages/cisco_meraki_metrics/docs/README.md +++ b/packages/cisco_meraki_metrics/docs/README.md @@ -305,3 +305,112 @@ An example event for `device_health` looks as following: } } ``` + +### Network Health + +The `network_health` dataset provides metrics related to the overall health and performance of Meraki networks. + +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + +**Exported fields** + +| Field | Description | Type | Unit | +|---|---|---|---| +| @timestamp | Event timestamp. | date | | +| data_stream.dataset | Data stream dataset. | constant_keyword | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | +| data_stream.type | Data stream type. | constant_keyword | | +| host.ip | Host ip addresses. | ip | | +| host.mac | Host mac addresses. | keyword | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| meraki.network.id | | keyword | | +| meraki.network.name | | keyword | | +| meraki.network.vpn_peers.network_id | | keyword | | +| meraki.network.vpn_peers.network_name | | keyword | | +| meraki.network.vpn_peers.usage_summary.received.bytes | | long | byte | +| meraki.network.vpn_peers.usage_summary.sent.bytes | | long | byte | +| meraki.organization_id | | keyword | | + + +An example event for `network_health` looks as following: + +```json +{ + "@timestamp": "2024-09-30T16:55:38.202Z", + "agent": { + "ephemeral_id": "11855dde-6a4a-48ce-ac32-087b1c7999a3", + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.15.2" + }, + "data_stream": { + "dataset": "cisco_meraki_metrics.network_health", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "snapshot": true, + "version": "8.15.2" + }, + "event": { + "agent_id_status": "verified", + "dataset": "cisco_meraki_metrics.network_health", + "duration": 12982553765, + "ingested": "2024-09-30T16:56:01Z", + "module": "meraki" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "c7f0ac74f5e24f78942164132c2c8ead", + "ip": "172.21.0.4", + "mac": "02-42-AC-15-00-04", + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "6.8.0-45-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" + } + }, + "meraki": { + "organization_id": "125432", + "network": { + "name": "BKYHUM", + "vpn_peers": [ + { + "network_id": "N_837204569103482715", + "network_name": "ZXVRNE", + "usage_summary.received.bytes": 12288, + "usage_summary.sent.bytes": 12288 + }, + { + "network_id": "N_294175608239471063", + "network_name": "QWMTJL", + "usage_summary.received.bytes": 0, + "usage_summary.sent.bytes": 79872 + } + ], + "id": "L_760194835627109284" + } + }, + "metricset": { + "name": "network_health", + "period": 60000 + }, + "service": { + "type": "meraki" + } +} +``` diff --git a/packages/cisco_meraki_metrics/manifest.yml b/packages/cisco_meraki_metrics/manifest.yml index 282cd1a7356..60f95042608 100644 --- a/packages/cisco_meraki_metrics/manifest.yml +++ b/packages/cisco_meraki_metrics/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.2.0 name: cisco_meraki_metrics title: Cisco Meraki Metrics -version: 0.4.1 +version: 0.5.0 description: Collect metrics from Cisco Meraki with Elastic Agent. type: integration categories: @@ -10,7 +10,7 @@ categories: - security conditions: kibana: - version: "^8.15.2 || ^9.0.0" + version: "^9.1.0" elastic: subscription: "basic" screenshots: