From 87efffa9510f8c7ef0a433b3f1cafa5fab3202af Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Thu, 16 Oct 2025 11:12:13 +0300 Subject: [PATCH 1/3] add network_health data stream --- .../_dev/build/docs/README.md | 12 ++ packages/cisco_meraki_metrics/changelog.yml | 5 + .../agent/stream/stream.yml.hbs | 22 ++++ .../elasticsearch/ingest_pipeline/default.yml | 66 +++++++++++ .../network_health/fields/agent.yml | 23 ++++ .../network_health/fields/base-fields.yml | 12 ++ .../network_health/fields/fields.yml | 30 +++++ .../data_stream/network_health/manifest.yml | 44 +++++++ .../network_health/sample_event.json | 76 ++++++++++++ packages/cisco_meraki_metrics/docs/README.md | 109 ++++++++++++++++++ packages/cisco_meraki_metrics/manifest.yml | 2 +- 11 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml create mode 100644 packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json diff --git a/packages/cisco_meraki_metrics/_dev/build/docs/README.md b/packages/cisco_meraki_metrics/_dev/build/docs/README.md index 1a55ae9e0a2..28b93f79877 100644 --- a/packages/cisco_meraki_metrics/_dev/build/docs/README.md +++ b/packages/cisco_meraki_metrics/_dev/build/docs/README.md @@ -43,3 +43,15 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur {{fields "device_health"}} {{event "device_health"}} + +### Network Health + +The `network_health` dataset provides metrics related to the overall health and performance of Meraki networks. + +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + +{{fields "network_health"}} + +{{event "network_health"}} diff --git a/packages/cisco_meraki_metrics/changelog.yml b/packages/cisco_meraki_metrics/changelog.yml index a829163d711..50d8b2036c8 100644 --- a/packages/cisco_meraki_metrics/changelog.yml +++ b/packages/cisco_meraki_metrics/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.5.0" + changes: + - description: Add `network_health` data stream + type: enhancement + link: https://github.com/elastic/integrations/pull/999 - version: "0.4.1" changes: - description: scale values in device channel utilization so they display correctly as percentages. diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs b/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..a999d22fdf8 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs @@ -0,0 +1,22 @@ +metricsets: ["network_health"] +apiBaseUrl: {{apiBaseUrl}} +apiKey: {{apiKey}} +organizations: +{{#each organizations as |organization|}} + - "{{organization}}" +{{/each}} +period: {{period}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..a9620ec0706 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,66 @@ +--- +description: Pipeline for parsing Cisco Meraki Network Health metrics. +processors: + - script: + lang: painless + source: > + // some values have unit 'percent' in the mappings; we need to scale them down from 0->100 to 0->1. + // we round to 4 decimal places to avoid floating point errors. + + if (ctx.meraki != null) { + if (ctx.meraki.uplink != null && ctx.meraki.uplink.loss != null && ctx.meraki.uplink.loss.pct != null) { + ctx.meraki.uplink.loss.pct = Math.round((ctx.meraki.uplink.loss.pct / 100) * 10000) / 10000.0; + } + + if (ctx.meraki.device != null && ctx.meraki.device.channel_utilization != null) { + def wifi0 = ctx.meraki.device.channel_utilization["2_4"]; + def wifi1 = ctx.meraki.device.channel_utilization["5"]; + + if (wifi0 != null) { + if (wifi0.utilization_80211 != null) { + wifi0.utilization_80211 = Math.round((wifi0.utilization_80211 / 100) * 10000) / 10000.0; + } + if (wifi0.utilization_non_80211 != null) { + wifi0.utilization_non_80211 = Math.round((wifi0.utilization_non_80211 / 100) * 10000) / 10000.0; + } + if (wifi0.utilization_total != null) { + wifi0.utilization_total = Math.round((wifi0.utilization_total / 100) * 10000) / 10000.0; + } + } + + if (wifi1 != null) { + if (wifi1.utilization_80211 != null) { + wifi1.utilization_80211 = Math.round((wifi1.utilization_80211 / 100) * 10000) / 10000.0; + } + if (wifi1.utilization_non_80211 != null) { + wifi1.utilization_non_80211 = Math.round((wifi1.utilization_non_80211 / 100) * 10000) / 10000.0; + } + if (wifi1.utilization_total != null) { + wifi1.utilization_total = Math.round((wifi1.utilization_total / 100) * 10000) / 10000.0; + } + } + } + } + - convert: + field: meraki.uplink.rsrp + type: float + ignore_missing: true + - convert: + field: meraki.uplink.rsrq + type: float + ignore_missing: true + - rename: + field: meraki.device.channel_utilization.2_4 + target_field: meraki.device.channel_utilization.wifi0 + ignore_missing: true + - rename: + field: meraki.device.channel_utilization.5 + target_field: meraki.device.channel_utilization.wifi1 + ignore_missing: true +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml new file mode 100644 index 00000000000..b549d5382f3 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml @@ -0,0 +1,23 @@ +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + dimension: true + description: 'Name of the host. + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' \ No newline at end of file diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml b/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml new file mode 100644 index 00000000000..2e74d6d8da3 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml @@ -0,0 +1,30 @@ +- name: meraki + type: group + fields: + - name: organization_id + type: keyword + dimension: true + - name: network + type: group + fields: + - name: id + type: keyword + dimension: true + - name: name + type: keyword + dimension: true + - name: vpn_peers + type: group + fields: + - name: network_id + type: keyword + dimension: true + - name: network_name + type: keyword + dimension: true + - name: usage_summary.received.bytes + type: long + unit: byte + - name: usage_summary.sent.bytes + type: long + unit: byte \ No newline at end of file diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml b/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml new file mode 100644 index 00000000000..5fad1940a6d --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml @@ -0,0 +1,44 @@ +title: "Cisco Meraki Network Health Metrics" +type: metrics +streams: + - input: meraki/metrics + title: Cisco Meraki Network Health Metrics + description: Collect network health metrics from the Meraki Dashboard API with Elastic Agent. + vars: + - name: apiKey + type: text + title: Meraki Dashboard API key + secret: true + required: true + - name: organizations + type: text + title: Meraki organization IDs + multi: true + required: true + - name: apiBaseUrl + type: url + title: Meraki Dashboard API base URL + default: api.meraki.com + - name: period + type: text + title: Collection interval + default: 60s + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - cisco_meraki_metrics-network_health + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details. +elasticsearch: + source_mode: synthetic + index_mode: time_series diff --git a/packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json b/packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json new file mode 100644 index 00000000000..57d11dc63f0 --- /dev/null +++ b/packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json @@ -0,0 +1,76 @@ +{ + "@timestamp": "2024-09-30T16:55:38.202Z", + "agent": { + "ephemeral_id": "11855dde-6a4a-48ce-ac32-087b1c7999a3", + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.15.2" + }, + "data_stream": { + "dataset": "cisco_meraki_metrics.network_health", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "snapshot": true, + "version": "8.15.2" + }, + "event": { + "agent_id_status": "verified", + "dataset": "cisco_meraki_metrics.network_health", + "duration": 12982553765, + "ingested": "2024-09-30T16:56:01Z", + "module": "meraki" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "c7f0ac74f5e24f78942164132c2c8ead", + "ip": "172.21.0.4", + "mac": "02-42-AC-15-00-04", + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "6.8.0-45-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" + } + }, + "meraki": { + "organization_id": "125432", + "network": { + "name": "BKYHUM", + "vpn_peers": [ + { + "network_id": "N_837204569103482715", + "network_name": "ZXVRNE", + "usage_summary.received.bytes": 12288, + "usage_summary.sent.bytes": 12288 + }, + { + "network_id": "N_294175608239471063", + "network_name": "QWMTJL", + "usage_summary.received.bytes": 0, + "usage_summary.sent.bytes": 79872 + } + ], + "id": "L_760194835627109284" + } + }, + "metricset": { + "name": "network_health", + "period": 60000 + }, + "service": { + "type": "meraki" + } +} \ No newline at end of file diff --git a/packages/cisco_meraki_metrics/docs/README.md b/packages/cisco_meraki_metrics/docs/README.md index 9f49428bf64..ef462bc4bf1 100644 --- a/packages/cisco_meraki_metrics/docs/README.md +++ b/packages/cisco_meraki_metrics/docs/README.md @@ -305,3 +305,112 @@ An example event for `device_health` looks as following: } } ``` + +### Network Health + +The `network_health` dataset provides metrics related to the overall health and performance of Meraki networks. + +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + +**Exported fields** + +| Field | Description | Type | Unit | +|---|---|---|---| +| @timestamp | Event timestamp. | date | | +| data_stream.dataset | Data stream dataset. | constant_keyword | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | +| data_stream.type | Data stream type. | constant_keyword | | +| host.ip | Host ip addresses. | ip | | +| host.mac | Host mac addresses. | keyword | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | +| meraki.network.id | | keyword | | +| meraki.network.name | | keyword | | +| meraki.network.vpn_peers.network_id | | keyword | | +| meraki.network.vpn_peers.network_name | | keyword | | +| meraki.network.vpn_peers.usage_summary.received.bytes | | long | byte | +| meraki.network.vpn_peers.usage_summary.sent.bytes | | long | byte | +| meraki.organization_id | | keyword | | + + +An example event for `network_health` looks as following: + +```json +{ + "@timestamp": "2024-09-30T16:55:38.202Z", + "agent": { + "ephemeral_id": "11855dde-6a4a-48ce-ac32-087b1c7999a3", + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.15.2" + }, + "data_stream": { + "dataset": "cisco_meraki_metrics.network_health", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.0.0" + }, + "elastic_agent": { + "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e", + "snapshot": true, + "version": "8.15.2" + }, + "event": { + "agent_id_status": "verified", + "dataset": "cisco_meraki_metrics.network_health", + "duration": 12982553765, + "ingested": "2024-09-30T16:56:01Z", + "module": "meraki" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "c7f0ac74f5e24f78942164132c2c8ead", + "ip": "172.21.0.4", + "mac": "02-42-AC-15-00-04", + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "6.8.0-45-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" + } + }, + "meraki": { + "organization_id": "125432", + "network": { + "name": "BKYHUM", + "vpn_peers": [ + { + "network_id": "N_837204569103482715", + "network_name": "ZXVRNE", + "usage_summary.received.bytes": 12288, + "usage_summary.sent.bytes": 12288 + }, + { + "network_id": "N_294175608239471063", + "network_name": "QWMTJL", + "usage_summary.received.bytes": 0, + "usage_summary.sent.bytes": 79872 + } + ], + "id": "L_760194835627109284" + } + }, + "metricset": { + "name": "network_health", + "period": 60000 + }, + "service": { + "type": "meraki" + } +} +``` diff --git a/packages/cisco_meraki_metrics/manifest.yml b/packages/cisco_meraki_metrics/manifest.yml index 282cd1a7356..5e686025277 100644 --- a/packages/cisco_meraki_metrics/manifest.yml +++ b/packages/cisco_meraki_metrics/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.2.0 name: cisco_meraki_metrics title: Cisco Meraki Metrics -version: 0.4.1 +version: 0.5.0 description: Collect metrics from Cisco Meraki with Elastic Agent. type: integration categories: From e2527cadd99f015372b63a2c58a6a7079a951b41 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Fri, 17 Oct 2025 10:32:36 +0300 Subject: [PATCH 2/3] bump kibana version --- packages/cisco_meraki_metrics/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cisco_meraki_metrics/manifest.yml b/packages/cisco_meraki_metrics/manifest.yml index 5e686025277..60f95042608 100644 --- a/packages/cisco_meraki_metrics/manifest.yml +++ b/packages/cisco_meraki_metrics/manifest.yml @@ -10,7 +10,7 @@ categories: - security conditions: kibana: - version: "^8.15.2 || ^9.0.0" + version: "^9.1.0" elastic: subscription: "basic" screenshots: From a444bdb4dc0df9c6ae019fbc37f5f355c7022124 Mon Sep 17 00:00:00 2001 From: Gabriel Pop <94497545+gpop63@users.noreply.github.com> Date: Mon, 20 Oct 2025 10:46:43 +0300 Subject: [PATCH 3/3] Update packages/cisco_meraki_metrics/changelog.yml Co-authored-by: Mykola Kmet --- packages/cisco_meraki_metrics/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cisco_meraki_metrics/changelog.yml b/packages/cisco_meraki_metrics/changelog.yml index 50d8b2036c8..df496880ac3 100644 --- a/packages/cisco_meraki_metrics/changelog.yml +++ b/packages/cisco_meraki_metrics/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add `network_health` data stream type: enhancement - link: https://github.com/elastic/integrations/pull/999 + link: https://github.com/elastic/integrations/pull/15663 - version: "0.4.1" changes: - description: scale values in device channel utilization so they display correctly as percentages.