[cisco_meraki_metrics] Add network_health data stream

packages/cisco_meraki_metrics/_dev/build/docs/README.md

@@ -43,3 +43,15 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 {{fields "device_health"}}
 
 {{event "device_health"}}
+
+### Network Health
+
+The `network_health` dataset provides metrics related to the overall health and performance of Meraki networks.
+
+**ECS Field Reference**
+
+Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields.
+
+{{fields "network_health"}}
+
+{{event "network_health"}}

packages/cisco_meraki_metrics/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.5.0"
+  changes:
+    - description: Add `network_health` data stream
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15663
 - version: "0.4.1"
   changes:
     - description: scale values in device channel utilization so they display correctly as percentages.

packages/cisco_meraki_metrics/data_stream/network_health/agent/stream/stream.yml.hbs

@@ -0,0 +1,22 @@
+metricsets: ["network_health"]
+apiBaseUrl: {{apiBaseUrl}}
+apiKey: {{apiKey}}
+organizations:
+{{#each organizations as |organization|}}
+  - "{{organization}}"
+{{/each}}
+period: {{period}}
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/cisco_meraki_metrics/data_stream/network_health/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,66 @@
+---
+description: Pipeline for parsing Cisco Meraki Network Health metrics.
+processors:
+  - script:
+      lang: painless
+      source: >
+        // some values have unit 'percent' in the mappings; we need to scale them down from 0->100 to 0->1.
+        // we round to 4 decimal places to avoid floating point errors.
+
+        if (ctx.meraki != null) {
+            if (ctx.meraki.uplink != null && ctx.meraki.uplink.loss != null && ctx.meraki.uplink.loss.pct != null) {
+                ctx.meraki.uplink.loss.pct = Math.round((ctx.meraki.uplink.loss.pct / 100) * 10000) / 10000.0;
+            }
+
+            if (ctx.meraki.device != null && ctx.meraki.device.channel_utilization != null) {
+                def wifi0 = ctx.meraki.device.channel_utilization["2_4"];
+                def wifi1 = ctx.meraki.device.channel_utilization["5"];
+
+                if (wifi0 != null) {
+                    if (wifi0.utilization_80211 != null) {
+                        wifi0.utilization_80211 = Math.round((wifi0.utilization_80211 / 100) * 10000) / 10000.0;
+                    }
+                    if (wifi0.utilization_non_80211 != null) {
+                        wifi0.utilization_non_80211 = Math.round((wifi0.utilization_non_80211 / 100) * 10000) / 10000.0;
+                    }
+                    if (wifi0.utilization_total != null) {
+                        wifi0.utilization_total = Math.round((wifi0.utilization_total / 100) * 10000) / 10000.0;
+                    }
+                }
+
+                if (wifi1 != null) {
+                    if (wifi1.utilization_80211 != null) {
+                        wifi1.utilization_80211 = Math.round((wifi1.utilization_80211 / 100) * 10000) / 10000.0;
+                    }
+                    if (wifi1.utilization_non_80211 != null) {
+                        wifi1.utilization_non_80211 = Math.round((wifi1.utilization_non_80211 / 100) * 10000) / 10000.0;
+                    }
+                    if (wifi1.utilization_total != null) {
+                        wifi1.utilization_total = Math.round((wifi1.utilization_total / 100) * 10000) / 10000.0;
+                    }
+                }
+            }
+        }
+  - convert:
+      field: meraki.uplink.rsrp
+      type: float
+      ignore_missing: true
+  - convert:
+      field: meraki.uplink.rsrq
+      type: float
+      ignore_missing: true
+  - rename:
+      field: meraki.device.channel_utilization.2_4
+      target_field: meraki.device.channel_utilization.wifi0
+      ignore_missing: true
+  - rename:
+      field: meraki.device.channel_utilization.5
+      target_field: meraki.device.channel_utilization.wifi1
+      ignore_missing: true
+on_failure:
+  - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
+      field: error.message
+      value: '{{{ _ingest.on_failure_message }}}'

packages/cisco_meraki_metrics/data_stream/network_health/fields/agent.yml

@@ -0,0 +1,23 @@
+- name: host
+  title: Host
+  group: 2
+  description: 'A host is defined as a general computing instance.
+    ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.'
+  type: group
+  fields:
+    - name: ip
+      level: core
+      type: ip
+      description: Host ip addresses.
+    - name: mac
+      level: core
+      type: keyword
+      ignore_above: 1024
+      description: Host mac addresses.
+    - name: name
+      level: core
+      type: keyword
+      ignore_above: 1024
+      dimension: true
+      description: 'Name of the host.
+        It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.'

packages/cisco_meraki_metrics/data_stream/network_health/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/cisco_meraki_metrics/data_stream/network_health/fields/fields.yml

@@ -0,0 +1,30 @@
+- name: meraki
+  type: group
+  fields:
+    - name: organization_id
+      type: keyword
+      dimension: true
+    - name: network
+      type: group
+      fields:
+        - name: id
+          type: keyword
+          dimension: true
+        - name: name
+          type: keyword
+          dimension: true
+        - name: vpn_peers
+          type: group
+          fields:
+            - name: network_id
+              type: keyword
+              dimension: true
+            - name: network_name
+              type: keyword
+              dimension: true
+            - name: usage_summary.received.bytes
+              type: long
+              unit: byte
+            - name: usage_summary.sent.bytes
+              type: long
+              unit: byte

packages/cisco_meraki_metrics/data_stream/network_health/manifest.yml

@@ -0,0 +1,44 @@
+title: "Cisco Meraki Network Health Metrics"
+type: metrics
+streams:
+  - input: meraki/metrics
+    title: Cisco Meraki Network Health Metrics
+    description: Collect network health metrics from the Meraki Dashboard API with Elastic Agent.
+    vars:
+      - name: apiKey
+        type: text
+        title: Meraki Dashboard API key
+        secret: true
+        required: true
+      - name: organizations
+        type: text
+        title: Meraki organization IDs
+        multi: true
+        required: true
+      - name: apiBaseUrl
+        type: url
+        title: Meraki Dashboard API base URL
+        default: api.meraki.com
+      - name: period
+        type: text
+        title: Collection interval
+        default: 60s
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: true
+        show_user: false
+        default:
+          - forwarded
+          - cisco_meraki_metrics-network_health
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details.
+elasticsearch:
+  source_mode: synthetic
+  index_mode: time_series

packages/cisco_meraki_metrics/data_stream/network_health/sample_event.json

@@ -0,0 +1,76 @@
+{
+    "@timestamp": "2024-09-30T16:55:38.202Z",
+    "agent": {
+        "ephemeral_id": "11855dde-6a4a-48ce-ac32-087b1c7999a3",
+        "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e",
+        "name": "docker-fleet-agent",
+        "type": "metricbeat",
+        "version": "8.15.2"
+    },
+    "data_stream": {
+        "dataset": "cisco_meraki_metrics.network_health",
+        "namespace": "default",
+        "type": "metrics"
+    },
+    "ecs": {
+        "version": "8.0.0"
+    },
+    "elastic_agent": {
+        "id": "f06c246c-8375-47a9-b0f1-d0fc6c050e4e",
+        "snapshot": true,
+        "version": "8.15.2"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "dataset": "cisco_meraki_metrics.network_health",
+        "duration": 12982553765,
+        "ingested": "2024-09-30T16:56:01Z",
+        "module": "meraki"
+    },
+    "host": {
+        "architecture": "x86_64",
+        "containerized": false,
+        "hostname": "docker-fleet-agent",
+        "id": "c7f0ac74f5e24f78942164132c2c8ead",
+        "ip": "172.21.0.4",
+        "mac": "02-42-AC-15-00-04",
+        "name": "docker-fleet-agent",
+        "os": {
+            "codename": "focal",
+            "family": "debian",
+            "kernel": "6.8.0-45-generic",
+            "name": "Ubuntu",
+            "platform": "ubuntu",
+            "type": "linux",
+            "version": "20.04.6 LTS (Focal Fossa)"
+        }
+    },
+    "meraki": {
+        "organization_id": "125432",
+        "network": {
+            "name": "BKYHUM",
+            "vpn_peers": [
+                {
+                    "network_id": "N_837204569103482715",
+                    "network_name": "ZXVRNE",
+                    "usage_summary.received.bytes": 12288,
+                    "usage_summary.sent.bytes": 12288
+                },
+                {
+                    "network_id": "N_294175608239471063",
+                    "network_name": "QWMTJL",
+                    "usage_summary.received.bytes": 0,
+                    "usage_summary.sent.bytes": 79872
+                }
+            ],
+            "id": "L_760194835627109284"
+        }
+    },
+    "metricset": {
+        "name": "network_health",
+        "period": 60000
+    },
+    "service": {
+        "type": "meraki"
+    }
+}