From 6438b7003217457d467955bf0e4e050a43164755 Mon Sep 17 00:00:00 2001 From: Keith Cantrell Date: Tue, 4 Mar 2025 15:08:39 -0600 Subject: [PATCH 01/11] Initial Version. --- .../README.md | 279 + .../dashboards/FSxN_CW_Utilization.json | 509 + .../dashboards/FSxN_Clusters.json | 2464 +++ .../dashboards/FSxN_Data_protection.json | 1746 ++ .../dashboards/FSxN_LUN.json | 5472 +++++++ .../dashboards/FSxN_SVM.json | 13339 ++++++++++++++++ .../dashboards/FSxN_Volume.json | 7947 +++++++++ .../dashboards/dashboards.yaml | 11 + .../fsx_dashboards.zip | Bin 0 -> 49461 bytes 9 files changed, 31767 insertions(+) create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_CW_Utilization.json create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Clusters.json create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Data_protection.json create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_LUN.json create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_SVM.json create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Volume.json create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/dashboards.yaml create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/fsx_dashboards.zip diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md new file mode 100644 index 00000000..32407423 --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md @@ -0,0 +1,279 @@ +# Deploy NetApp Harvest on EC2 + +Harvest installation for monitoring Amazon FSxN using Promethues and Grafana stack, integrating AWS Secret Manager for FSxN credentials. + +## Introduction + +### What to Expect + +Harvest installation will result in the following: +* Install NetApp Harvest with the latest version on your EC2 instance. +* Collecting metrics about your FSxNs and adding existing Grafana dashboards for better visualization. + +### Prerequisites +* A FSx for ONTAP running in the same VPC. +* If not running an AWS based Linux, ensure that the `aws` command has been instealled and configured. + +## Installation Steps + +### 1. Create AWS Secret Manager with Username and Password for each FSxN + +```sh +aws secretsmanager create-secret --name --secret-string '{"username":"fsxadmin","password":""}' +``` + +### 2. Create Instance Profile with Permission to AWS Secret Manager and cloudwatch metrics + +#### 2.1. Create Policy with Permissions to AWS Secret Manager + +Edit the harvest-policy.json file found in this repo with the ARN of the AWS Secret Manager secret created above. + +```sh +POLICY_ARN=$(aws iam create-policy --policy-name harvest-policy --policy-document file://harvest-policy.json --query Policy.Arn --output text) +``` + +#### 2.2. Create Instance Profile Role + +```sh +aws iam create-role --role-name HarvestRole --assume-role-policy-document file://trust-policy.json +aws iam attach-role-policy --role-name HarvestRole --policy-arn $POLICY_ARN +aws iam create-instance-profile --instance-profile-name HarvestProfile +aws iam add-role-to-instance-profile --instance-profile-name HarvestProfile --role-name HarvestRole +``` + +Note that the `trust-policy.json` file can be found in this repo. + +### 3. Create EC2 Instance + +We recommend using a `t2.xlarge` instance type with 20GB disk and attaching the instance profile. + +If you already have an ec2 instance, you can use the following command to attach the instance profile: + +```sh +aws ec2 associate-iam-instance-profile --instance-id --iam-instance-profile Arn=,Name=HarvestProfile +``` +You should get the instance profile ARN from step 2.2 above. + +If your exiting ec2 instance already had an instance profile, then simply add the policy create in step 2.2 above. + +### 4. Install Docker and Docker Compose + +Use the following commands if you are running an Red Hat based Linux: +```sh +sudo yum install docker +sudo curl -L https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-compose-plugin-2.6.0-3.el7.x86_64.rpm -o ./compose-plugin.rpm +sudo yum install ./compose-plugin.rpm -y +sudo systemctl start docker +``` +If you aren't running a Red Hat based Linux, you can follow the instructions [here](https://docs.docker.com/engine/install/). + +To confirm that docker has been installed correctly, run the following command: + +```sh +sudo docker run hello-world +``` + +You should get output similar to the following: +``` +Unable to find image 'hello-world:latest' locally +latest: Pulling from library/hello-world +e6590344b1a5: Pull complete +Digest: sha256:bfbb0cc14f13f9ed1ae86abc2b9f11181dc50d779807ed3a3c5e55a6936dbdd5 +Status: Downloaded newer image for hello-world:latest + +Hello from Docker! +This message shows that your installation appears to be working correctly. + +To generate this message, Docker took the following steps: + 1. The Docker client contacted the Docker daemon. + 2. The Docker daemon pulled the "hello-world" image from the Docker Hub. + (amd64) + 3. The Docker daemon created a new container from that image which runs the + executable that produces the output you are currently reading. + 4. The Docker daemon streamed that output to the Docker client, which sent it + to your terminal. + +To try something more ambitious, you can run an Ubuntu container with: + $ docker run -it ubuntu bash + +Share images, automate workflows, and more with a free Docker ID: + https://hub.docker.com/ + +For more examples and ideas, visit: + https://docs.docker.com/get-started/ +``` +### 5. Install Harvest on EC2 + +To install Harvest on your EC2 instance following the following steps: + +#### 5.1. Generate Harvest Configuration File + +Create `harvest.yml` file with your cluster details, below is an example with annotated comments. Modify as needed for your scenario: + +```yaml +Exporters: + prometheus1: + exporter: Prometheus + port_range: 12990-14000 + add_meta_tags: false +Defaults: + use_insecure_tls: true +Pollers: + fsx01: + datacenter: fsx + addr: + collectors: + - Rest + - RestPerf + - Ems + exporters: + - prometheus1 + credentials_script: + path: /opt/fetch-credentails + schedule: 3h + timeout: 10s + fsx02: + datacenter: fsx + addr: + collectors: + - Rest + - RestPerf + - Ems + exporters: + - prometheus1 + credentials_script: + path: /opt/fetch-credentails + schedule: 3h + timeout: 10s +``` + +#### 5.2. Generate a Docker Compose from Harvest Configuration + +Run the following command to generate a Docker Compose file from the Harvest configuration: + +```sh +docker run --rm \ + --env UID=$(id -u) --env GID=$(id -g) \ + --entrypoint "bin/harvest" \ + --volume "$(pwd):/opt/temp" \ + --volume "$(pwd)/harvest.yml:/opt/harvest/harvest.yml" \ + ghcr.io/netapp/harvest \ + generate docker full \ + --output harvest-compose.yml +``` + +:warning:**NOTE** Ignore the command that it outputs used to start Harvest. + +#### 5.3. Replace Harvest images in the harvest-compose.yml: + +Replace the Harvest image that supports using AWS Secret Manager for FSxN credentials: + +```yaml +sed -i 's|ghcr.io/netapp/harvest:latest|ghcr.io/tlvdevops/harvest-fsx:latest|g' harvest-compose.yml +``` + +#### 5.4. Add AWS Secret Manager Names to Docker Compose Environment Variables + +`SECRET_NAME` and `AWS_REGION` are required for the credentials script. + +```yaml +services: + fsx01: + image: ghcr.io/tlvdevops/harvest-fsx:latest + container_name: poller-fsx01 + restart: unless-stopped + ports: + - "12990:12990" + command: '--poller fsx01 --promPort 12990 --config /opt/harvest.yml' + volumes: + - ./cert:/opt/harvest/cert + - ./harvest.yml:/opt/harvest.yml + - ./conf:/opt/harvest/conf + environment: + - SECRET_NAME= + - AWS_REGION= + networks: + - backend +``` +#### 5.5. Download FSxN dashboards and import into Grafana container: +The following commands will download the FSxN designed dashboards from this repo and replace the default Grafana dashboards with them: +```yaml +wget https://raw.githubusercontent.com/NetApp/FSx-ONTAP-samples-scripts/main/Monitoring/monitor_fsxn_with_grafana/fsx_dashboards.zip +unzip fsx_dashboards.zip +rm -rf grafana/dashboards +mv dashboards grafana/dashboards +``` + +#### 5.6. Configure Prometheus to use yet-another-exporter (yace) to gather AWS FSxN metrics +AWS has useful metrics regarding the FSxN file system that ONTAP doesn't provide. Therefore, it is recommended to install +an exporter that will expose these metrics. The following steps show how to install a recommended exporter. + +##### 5.6.1 Create the yace configuration file. +Use the text in the box below to create the configuration file named `yace-config.yaml`. Replace ``, in both places, with the region where your FSxN resides: + +```yaml +apiVersion: v1alpha1 +sts-region: +discovery: + jobs: + - type: AWS/FSx + regions: [] + period: 300 + length: 300 + metrics: + - name: DiskReadOperations + statistics: [Average] + - name: DiskWriteOperations + statistics: [Average] + - name: DiskReadBytes + statistics: [Average] + - name: DiskWriteBytes + statistics: [Average] + - name: DiskIopsUtilization + statistics: [Average] + - name: NetworkThroughputUtilization + statistics: [Average] + - name: FileServerDiskThroughputUtilization + statistics: [Average] + +``` + +##### 5.6.2 Add Yet-Another-Exporter to harvest-compose.yaml + +Copy the following to the end of the `harvest-compose.yml` file: +```yaml + yace: + image: quay.io/prometheuscommunity/yet-another-cloudwatch-exporter:latest + container_name: yace + restart: always + expose: + - 8080 + volumes: + - ./yace-config.yaml:/tmp/config.yml + - $HOME/.aws:/exporter/.aws:ro + command: + - -listen-address=:8080 + - -config.file=/tmp/config.yml + networks: + - backend +``` + +##### 5.6.3. Add Yet-Another-Exporter target to prometheus.yml: +```yaml +sudo sed -i -e "\$a\- job_name: 'yace'" -e '$a\ static_configs:' -e "\$a\ - targets: ['yace:8080']" container/prometheus/prometheus.yml +``` + +##### 6. Bring Everything Up + +```sh +sudo docker compose -f prom-stack.yml -f harvest-compose.yml up -d --remove-orphans +``` + +After bringing up the prom-stack.yml compose file, you can access Grafana at +http://IP_OF_GRAFANA:3000. + +You will be prompted to create a new password the first time you log in. Grafana's default credentials are: +``` +username: admin +password: admin +``` diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_CW_Utilization.json b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_CW_Utilization.json new file mode 100644 index 00000000..d6cc1f21 --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_CW_Utilization.json @@ -0,0 +1,509 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 35, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cdot" + ], + "targetBlank": false, + "title": "Related Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process all the operations on the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 11, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(aws_fsx_disk_read_operations_average{dimension_FileSystemId=~\"$Cluster\"}) by (dimension_FileSystemId)", + "interval": "", + "legendFormat": "{{dimension_FileSystemId}}_read_operations", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(aws_fsx_disk_write_operations_average{dimension_FileSystemId=~\"$Cluster\"}) by (dimension_FileSystemId)", + "hide": false, + "instant": false, + "legendFormat": "{{dimension_FileSystemId}}_write_operations", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(aws_fsx_disk_write_operations_average{dimension_FileSystemId=~\"$Cluster\"} + aws_fsx_disk_read_operations_average{dimension_FileSystemId=~\"$Cluster\"}) by (dimension_FileSystemId)", + "hide": false, + "instant": false, + "legendFormat": "{{dimension_FileSystemId}}_sum_operations", + "range": true, + "refId": "C" + } + ], + "title": "Disk Operations", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process all the operations on the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 13, + "x": 11, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(aws_fsx_disk_write_bytes_average{dimension_FileSystemId=~\"$Cluster\"}) by (dimension_FileSystemId) / 1048576", + "interval": "", + "legendFormat": "{{dimension_FileSystemId}}_write_megabytes", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(aws_fsx_disk_read_bytes_average{dimension_FileSystemId=~\"$Cluster\"}) by (dimension_FileSystemId) / 1048576", + "hide": false, + "instant": false, + "legendFormat": "{{dimension_FileSystemId}}_read_megabytes", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(aws_fsx_disk_read_bytes_average{dimension_FileSystemId=~\"$Cluster\"} + aws_fsx_disk_write_bytes_average{dimension_FileSystemId=~\"$Cluster\"} ) by (dimension_FileSystemId)", + "hide": false, + "instant": false, + "legendFormat": "{{dimension_FileSystemId}}_sum_megabytes", + "range": true, + "refId": "C" + } + ], + "title": "Disk Throughput", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process all the operations on the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "fs-038c109c5e023e533_network_throughput_utilization" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 19, + "w": 11, + "x": 5, + "y": 13 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(aws_fsx_disk_iops_utilization_average{dimension_FileSystemId=~\"$Cluster\"}) by (dimension_FileSystemId)", + "interval": "", + "legendFormat": "{{dimension_FileSystemId}}_iops_utilization", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg(aws_fsx_network_throughput_utilization_average{dimension_FileSystemId=~\"$Cluster\"}) by (dimension_FileSystemId)", + "hide": false, + "instant": false, + "legendFormat": "{{dimension_FileSystemId}}_network_throughput_utilization", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(aws_fsx_file_server_disk_throughput_utilization_average{dimension_FileSystemId=~\"$Cluster\"} + aws_fsx_disk_write_bytes_average{dimension_FileSystemId=~\"$Cluster\"} ) by (dimension_FileSystemId)", + "hide": false, + "instant": false, + "legendFormat": "{{dimension_FileSystemId}}_disk_throughput_utilization", + "range": true, + "refId": "C" + } + ], + "title": "Utilization Percentage", + "transformations": [], + "transparent": true, + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": [ + "" + ], + "value": [ + "" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(dimension_FileSystemId)", + "hide": 0, + "includeAll": false, + "multi": true, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(dimension_FileSystemId)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "FSxN: Utilization", + "uid": "c467bd93-f82b-4c34-8bc0-3a8b71e39898", + "version": 3, + "weekStart": "" +} diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Clusters.json b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Clusters.json new file mode 100644 index 00000000..f75219dd --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Clusters.json @@ -0,0 +1,2464 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 6, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cdot" + ], + "targetBlank": false, + "title": "Related Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 225, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of CIFS operations per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 239, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (cluster) (node_cifs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})\nand\ntopk($TopResources, sum by (cluster) (avg_over_time(node_cifs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[3h] @ end())))", + "interval": "", + "legendFormat": "{{cluster}}", + "refId": "A" + } + ], + "title": "Top $TopResources CIFS IOPs by Cluster", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of NFS operations per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 241, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (cluster) (node_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})\nand\ntopk($TopResources, sum by (cluster) (avg_over_time(node_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[3h] @ end())))", + "interval": "", + "legendFormat": "{{cluster}}", + "refId": "A" + } + ], + "title": "Top $TopResources NFS IOPs by Cluster", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of operations per second serviced by the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 240, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (cluster) (volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})\nand\ntopk($TopResources, sum by (cluster) (avg_over_time(volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[3h] @ end())))", + "interval": "", + "legendFormat": "{{cluster}}", + "refId": "A" + } + ], + "title": "Top $TopResources Total IOPs by Cluster", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average processor utilization across all processors in the system.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 247, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg by (cluster) (node_avg_processor_busy{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})\nand\ntopk($TopResources, avg by (cluster) (avg_over_time(node_avg_processor_busy{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[3h] @ end())))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{cluster}}", + "refId": "A" + } + ], + "title": "Top $TopResources Clusters by Average CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "System CPU resource utilization is a computed percentage that indicates how busy the system is based on a combination of the most heavily utilized domain and idle CPU cycles. This metric determines the amount of available CPU and will vary based on domains reaching their limits OR exhausting all idle CPU cycles, whichever comes first.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 271, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg by (cluster) (node_cpu_busy{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})\nand\ntopk($TopResources, avg by (cluster) (avg_over_time(node_cpu_busy{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[3h] @ end())))", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{cluster}}", + "refId": "A" + } + ], + "title": "Top $TopResources Clusters by CPU busy", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process all the operations on the volume; not including request processing or network communication time. node_volume_avg_latency is [volume_avg_latency](#volume_avg_latency) aggregated by `node`.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 234, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg by (cluster) (node_volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})\nand\ntopk($TopResources, avg by (cluster) (avg_over_time(node_volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}[3h] @ end())))", + "interval": "", + "legendFormat": "{{cluster}}", + "refId": "A" + } + ], + "title": "Top $TopResources Clusters by Average Latency", + "type": "timeseries" + } + ], + "title": "Cluster Metrics", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 226, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "SVM space used percent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-orange", + "value": 80 + }, + { + "color": "dark-red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Current" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cluster" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "displayName", + "value": "Cluster" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-cluster/ontap-cluster?orgId=1&${Datacenter:queryparam}&${__url_time_range}&var-Cluster=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 2 + }, + "id": 233, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Current" + } + ] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "(\n 100 * sum by (datacenter,cluster, svm) (\n volume_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}\n )\n / on(datacenter, cluster, svm)\n sum by (datacenter,cluster, svm) (\n volume_size_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}\n )\n)\nand on(datacenter,cluster, svm)\ntopk(\n $TopResources,\n (\n 100 * sum by (datacenter,cluster, svm) (\n avg_over_time(volume_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}[3h] @ end())\n )\n / on(datacenter,cluster, svm)\n sum by (datacenter,cluster, svm) (\n avg_over_time(volume_size_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}[3h] @ end())\n )\n )\n * on(datacenter,cluster, svm) group_left\n svm_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", root_svm=~\"No|$IncludeRoot\"}\n)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources SVMs by Capacity Used %", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Current" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Current", + "svm", + "cluster" + ] + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Volume space used percent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-orange", + "value": 80 + }, + { + "color": "dark-red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Current" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cluster" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "displayName", + "value": "Cluster" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-cluster/ontap-cluster?orgId=1&${Datacenter:queryparam}&${__url_time_range}&var-Cluster=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 2 + }, + "id": 242, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Current" + } + ] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "(\n 100 * volume_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\",volume=~\"$Volume\"} \n / on(datacenter,cluster,svm,aggr,node,volume,style) \n volume_size_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\",volume=~\"$Volume\"}\n)\nand on(datacenter,cluster, svm,aggr,node, volume,style)\ntopk(\n $TopResources,\n (\n (100 * avg_over_time(volume_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\",volume=~\"$Volume\"}[3h] @ end())) \n / on(datacenter,cluster,svm,aggr,node,volume,style) \n avg_over_time(volume_size_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\",volume=~\"$Volume\"}[3h] @ end())\n )\n * on(datacenter,cluster, svm, aggr,node,volume,style) group_left\n volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"}\n)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Capacity Used %", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Current" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "volume", + "Current", + "cluster" + ] + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Aggregate space used percent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-orange", + "value": 80 + }, + { + "color": "dark-red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Current" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cluster" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + }, + { + "id": "displayName", + "value": "Cluster" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-cluster/ontap-cluster?orgId=1&${Datacenter:queryparam}&${__url_time_range}&var-Cluster=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "aggr" + }, + "properties": [ + { + "id": "displayName", + "value": "Aggregate" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-aggregate/ontap-aggregate?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-Aggregate=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 2 + }, + "id": 243, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Current" + } + ] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "(\n 100 * aggr_space_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} \n / on(datacenter,cluster,aggr) \n aggr_space_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"}\n)\nand on(datacenter,cluster, aggr)\ntopk(\n $TopResources,\n (\n (100 * avg_over_time(aggr_space_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"}[3h] @ end())) \n / on(datacenter,cluster,aggr) \n avg_over_time(aggr_space_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"}[3h] @ end())\n )\n)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Aggregates by Capacity Used %", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "Value": 7, + "aggr": 2, + "cluster": 1, + "datacenter": 3, + "instance": 4, + "job": 5, + "node": 6 + }, + "renameByName": { + "Value": "Current" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Current", + "aggr", + "cluster" + ] + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "SVM space used percent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 272, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "(\n 100 * sum by (datacenter,cluster, svm) (\n volume_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}\n )\n / on(datacenter, cluster, svm)\n sum by (datacenter,cluster, svm) (\n volume_size_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}\n )\n)\nand on(datacenter,cluster, svm)\ntopk(\n $TopResources,\n (\n 100 * sum by (datacenter,cluster, svm) (\n avg_over_time(volume_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}[3h] @ end())\n )\n / on(datacenter,cluster, svm)\n sum by (datacenter,cluster, svm) (\n avg_over_time(volume_size_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}[3h] @ end())\n )\n )\n * on(datacenter,cluster, svm) group_left\n svm_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", root_svm=~\"No|$IncludeRoot\"}\n)", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources SVMs by Capacity Used %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Volume space used percent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 266, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "(\n 100 * volume_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\",volume=~\"$Volume\"} \n / on(datacenter,cluster,svm,aggr,node,volume,style) \n volume_size_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\",volume=~\"$Volume\"}\n)\nand on(datacenter,cluster, svm,aggr,node, volume,style)\ntopk(\n $TopResources,\n (\n (100 * avg_over_time(volume_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\",volume=~\"$Volume\"}[3h] @ end())) \n / on(datacenter,cluster,svm,aggr,node,volume,style) \n avg_over_time(volume_size_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\",volume=~\"$Volume\"}[3h] @ end())\n )\n * on(datacenter,cluster, svm, aggr,node,volume,style) group_left\n volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\",root_volume=~\"No|$IncludeRoot\"}\n)", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Capacity Used %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Aggregate space used percent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 267, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "(\n 100 * aggr_space_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} \n / on(datacenter,cluster,aggr) \n aggr_space_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"}\n)\nand on(datacenter,cluster, aggr)\ntopk(\n $TopResources,\n (\n (100 * avg_over_time(aggr_space_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"}[3h] @ end())) \n / on(datacenter,cluster,aggr) \n avg_over_time(aggr_space_total{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"}[3h] @ end())\n )\n)", + "interval": "", + "legendFormat": "{{cluster}} - {{aggr}}", + "refId": "A" + } + ], + "title": "Top $TopResources Aggregates by Capacity Used %", + "type": "timeseries" + } + ], + "title": "Capacity Metrics", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 256, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process all the operations on the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 268, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_vol_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(svm_vol_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Average Latency by SVMs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "SVM total throughput.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 269, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "(\n svm_vol_read_data{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\",svm=~\"$SVM\"} \n +\n svm_vol_write_data{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\",svm=~\"$SVM\"}\n)\nand on(datacenter,cluster, svm)\ntopk(\n $TopResources,\n (\n avg_over_time(svm_vol_read_data{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h])\n + avg_over_time(svm_vol_write_data{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h])\n )\n * on(datacenter,cluster, svm) group_left\n svm_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", root_svm=~\"No|$IncludeRoot\"}\n)", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Average Throughput by SVMs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of operations per second serviced by the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 270, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_vol_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(svm_vol_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources IOPs by SVMs", + "type": "timeseries" + } + ], + "title": "SVM Metrics", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 264, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process all the operations on the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 258, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}\nand on(datacenter,cluster,svm,volume)\ntopk($TopResources, avg_over_time(volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Average Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Volume total throughput.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 260, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "(\n volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"} \n +\n volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}\n)\nand on(datacenter,cluster,svm,volume)\ntopk(\n $TopResources,\n (\n avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[3h] @ end())\n + avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[3h] @ end())\n )\n)", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Average Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of operations per second serviced by the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 262, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}\nand on(datacenter,cluster,svm,volume)\ntopk($TopResources, avg_over_time(volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by IOPs", + "type": "timeseries" + } + ], + "title": "Volume Metrics", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 2, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": [ + "DC-01" + ], + "value": [ + "DC-01" + ] + }, + "datasource": "prometheus", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", + "includeAll": false, + "multi": true, + "name": "Datacenter", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", + "refId": "Prometheus-Datacenter-Variable-Query" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "includeAll": true, + "multi": true, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "label_values(svm_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"},svm)", + "includeAll": true, + "multi": true, + "name": "SVM", + "options": [], + "query": { + "query": "label_values(svm_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"},svm)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "label_values(volume_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"},volume)", + "includeAll": true, + "multi": true, + "name": "Volume", + "options": [], + "query": { + "query": "label_values(volume_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"},volume)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "5", + "value": "5" + }, + "includeAll": false, + "name": "TopResources", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "2", + "value": "2" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": false, + "text": "4", + "value": "4" + }, + { + "selected": true, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "6", + "value": "6" + }, + { + "selected": false, + "text": "8", + "value": "8" + }, + { + "selected": false, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "15", + "value": "15" + }, + { + "selected": false, + "text": "25", + "value": "25" + }, + { + "selected": false, + "text": "50", + "value": "50" + }, + { + "selected": false, + "text": "100", + "value": "100" + }, + { + "selected": false, + "text": "250", + "value": "250" + }, + { + "selected": false, + "text": "500", + "value": "500" + } + ], + "query": "1,2,3,4,5,6,8,10,15,25,50,100,250,500", + "type": "custom" + }, + { + "current": { + "text": "No", + "value": "No" + }, + "description": "", + "includeAll": false, + "name": "IncludeRoot", + "options": [ + { + "selected": true, + "text": "No", + "value": "No" + }, + { + "selected": false, + "text": "Yes", + "value": "Yes" + } + ], + "query": "No,Yes", + "type": "custom" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "FSxN: Clusters", + "uid": "cdot-cdot", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Data_protection.json b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Data_protection.json new file mode 100644 index 00000000..3c33fe0f --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Data_protection.json @@ -0,0 +1,1746 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 9, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cdot" + ], + "targetBlank": false, + "title": "Related Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 45, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Volumes Protected With Snapshot Copies (local).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Volumes protected" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Volumes not protected" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": false, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "count(volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{})) OR on() vector(0)", + "instant": true, + "interval": "", + "legendFormat": "Volumes protected", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "count((volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy=~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{}))) OR on() vector(0)", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "Volumes not protected", + "refId": "B" + } + ], + "title": "Protected Status", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of volumes that are protected.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 75, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "count(volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{})) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Volumes protected", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "description": "Volumes Breaching Snapshot Copy Reserve Space.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Volumes breached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Volumes not breached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": false, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "count(volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{}) * on (volume, svm, cluster) group_right() ( volume_snapshot_reserve_size{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} > volume_snapshot_reserve_size{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"})) OR on() vector(0)", + "instant": true, + "interval": "", + "legendFormat": "Volumes breached", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "(count((volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{})) * on (volume, svm, cluster) group_right() (volume_snapshot_reserve_size{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} <= volume_snapshot_reserve_size{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"})) OR on() vector(0)) + (count(volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{}) * on (volume, svm, cluster) group_right() ( volume_snapshot_reserve_size{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} == 0)) OR on() vector(0))", + "instant": true, + "interval": "", + "legendFormat": "Volumes not breached", + "refId": "B" + } + ], + "title": "Breached Status", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "description": "Total number of volumes whose snapshot size used is breached the snapshot reserve size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-red", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 77, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "count(volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{}) * on (volume, svm, cluster) group_right() ( volume_snapshot_reserve_size{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} > volume_snapshot_reserve_size{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"})) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Volumes breached", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "description": "Total number of volumes that are not protected.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "color": "rgb(21, 118, 171)", + "text": "0" + } + }, + "type": "special" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-yellow", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 26 + }, + "id": 79, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "count((volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy=~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{}))) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Volumes not protected", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "description": "Total number of volumes whose snapshot size used is not breached the snapshot reserve size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "color": "rgb(21, 118, 171)", + "text": "0" + } + }, + "type": "special" + } + ], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 26 + }, + "id": 81, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "(count((volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{})) * on (volume, svm, cluster) group_right() (volume_snapshot_reserve_size{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} > 0 and volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} > 0 and volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} <= volume_snapshot_reserve_size{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"})) OR on() vector(0)) + (count(volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{}) * on (volume, svm, cluster) group_right() ( volume_snapshot_reserve_size{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} == 0)) OR on() vector(0))", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Volumes not breached", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "description": "This panel displays volumes detail with protected status and snapshot policy.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "transparent", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "custom.width", + "value": 350 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "snapshot_policy" + }, + "properties": [ + { + "id": "displayName", + "value": "Snapshot Policy" + }, + { + "id": "custom.width", + "value": 220 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "custom.width", + "value": 250 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 83, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "label_replace(label_replace(volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", snapshot_policy!=\"\", volume!~\"MDV.*\"}, \"Status\", \"Protected\", \"snapshot_policy\", \"(.*)\") , \"Status\", \"Unprotected\", \"snapshot_policy\", \"(none.*)\") * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Volumes Protected With Snapshot Copies (local)", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "snapshot_policy", + "volume", + "Status", + "svm" + ] + } + } + }, + { + "id": "organize", + "options": { + "indexByName": { + "Status": 3, + "snapshot_policy": 2, + "svm": 1, + "volume": 0 + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "description": "Volumes details of snapshot copy reserve space.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "custom.width", + "value": 320 + }, + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Used" + }, + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Reserved" + }, + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "from": -1e+25, + "result": { + "index": 0, + "text": "Not Breached" + }, + "to": 0 + }, + "type": "range" + }, + { + "options": { + "from": 0, + "result": { + "index": 1, + "text": "Breached" + }, + "to": 1e+25 + }, + "type": "range" + } + ] + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "custom.width", + "value": 240 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 91, + "interval": "1m", + "maxDataPoints": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", snapshot_policy!=\"\", snapshot_policy!~\"none.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{}) * on (volume, svm, cluster) group_right() ( volume_snapshot_reserve_size{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} >= 0 and volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"} >= 0)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "volume_snapshots_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Volumes Breaching Snapshot Copy Reserve Space", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "greaterOrEqual", + "options": { + "value": 0 + } + }, + "fieldName": "Value #A" + } + ], + "match": "all", + "type": "include" + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "volume", + "Value #A", + "Value #B", + "svm" + ] + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Status", + "binary": { + "left": "Value #B", + "operator": "-", + "reducer": "sum", + "right": "Value #A" + }, + "mode": "binary", + "reduce": { + "include": [ + "Value #A", + "Value #B" + ], + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Status": 4, + "Value #A": 2, + "Value #B": 3, + "svm": 1, + "volume": 0 + }, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "title": "Snapshot Copies Overview", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 22, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "description": "Total number of volumes whose snapshot count is < 10.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 2 + }, + "id": 96, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "count((volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})) * on (volume,svm,cluster) (volume_snapshot_count{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"} > 0 and volume_snapshot_count{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"} < 10)) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": " <10 Copies ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "description": "Total number of volumes whose snapshot count is between 10 to 100.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 2 + }, + "id": 97, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "count((volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})) * on (volume,svm,cluster) (volume_snapshot_count{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"} >= 10 and volume_snapshot_count{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"} <= 100)) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "10-100 Copies", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "description": "Total number of volumes whose snapshot count is between 101 to 500.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 2 + }, + "id": 98, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "count((volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})) * on (volume,svm,cluster) (volume_snapshot_count{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"} > 100 and volume_snapshot_count{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"} <= 500)) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "101-500 Copies", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "description": "Total number of volumes whose snapshot count is > 500.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 2 + }, + "id": 99, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "count((volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})) * on (volume,svm,cluster) (volume_snapshot_count{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"} > 500)) OR on() vector(0)", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": ">500 Copies", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "description": "This panel displays volumes detail with snapshot count.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "transparent", + "mode": "fixed" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Bucket" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + }, + { + "id": "mappings", + "value": [ + { + "options": { + "from": 0, + "result": { + "index": 0, + "text": "< 10 Copies" + }, + "to": 9 + }, + "type": "range" + }, + { + "options": { + "from": 10, + "result": { + "index": 1, + "text": "10-100 Copies" + }, + "to": 100 + }, + "type": "range" + }, + { + "options": { + "from": 101, + "result": { + "index": 2, + "text": "101-500 Copies" + }, + "to": 500 + }, + "type": "range" + }, + { + "options": { + "from": 501, + "result": { + "index": 3, + "text": "> 500 Copies" + }, + "to": 1e+24 + }, + "type": "range" + } + ] + }, + { + "id": "custom.width", + "value": 290 + }, + { + "id": "displayName", + "value": "Snapshot Copies Bucket" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "custom.width", + "value": 500 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "custom.width", + "value": 500 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Snapshot Copies" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cluster" + }, + "properties": [ + { + "id": "displayName", + "value": "Cluster" + }, + { + "id": "custom.width", + "value": 400 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-cluster/ontap-cluster?orgId=1&${Datacenter:queryparam}&${__url_time_range}&var-Cluster=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 94, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ee9vj35on6mtcb" + }, + "exemplar": false, + "expr": "(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy!=\"\", snapshot_policy!~\"none.*\", volume!~\"MDV.*\"} * on (snapshot_policy) group_left () group by (snapshot_policy) (snapshot_policy_total_schedules{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})) * on (volume,svm,cluster) (volume_snapshot_count{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"} > 0)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Volume count by the number of Snapshot copies", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "svm", + "volume", + "Value", + "cluster" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 3, + "cluster": 2, + "svm": 1, + "volume": 0 + }, + "renameByName": { + "Value": "Snapshot Copies", + "volume": "" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Bucket", + "mode": "reduceRow", + "reduce": { + "include": [ + "Snapshot Copies" + ], + "reducer": "last" + } + } + } + ], + "type": "table" + } + ], + "title": "Snapshot Copies Analysis", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "ee9vj35on6mtcb" + }, + "hide": 2, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": [ + "DC-01" + ], + "value": [ + "DC-01" + ] + }, + "datasource": "prometheus", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", + "includeAll": false, + "multi": true, + "name": "Datacenter", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "includeAll": true, + "multi": true, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "FSxN: Data Protection", + "uid": "cdot-data-protection-snapshots", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_LUN.json b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_LUN.json new file mode 100644 index 00000000..1bd10975 --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_LUN.json @@ -0,0 +1,5472 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 18, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cdot" + ], + "targetBlank": false, + "title": "Related Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 20, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average Latency of top $TopResources volumes.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(topk($TopResources, lun_avg_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Average Read Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average Latency of top $TopResources volumes.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 36, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(topk($TopResources, lun_avg_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Average Write Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Throughput of top $TopResources volumes.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 23, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(topk($TopResources, lun_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Read Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Throughput of top $TopResources volumes.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 37, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(topk($TopResources, lun_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Write Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "IOPs of top $TopResources volumes.", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(topk($TopResources, lun_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Read IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "IOPs of top $TopResources volumes.", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 38, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(topk($TopResources, lun_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Write IOPs", + "type": "stat" + } + ], + "title": "Highlights", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 40, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average read latency in microseconds for all operations on the LUN.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false, + "width": 0 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Avg" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "lun" + }, + "properties": [ + { + "id": "displayName", + "value": "LUN" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-lun/ontap-lun?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${Volume:queryparam}&${__url_time_range}&var-LUN=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 2 + }, + "id": 42, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "volume" + } + ] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_avg_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \n and \ntopk($TopResources, lun_avg_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} @ end())", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Read Latency", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "instance", + "job", + "lun", + "svm", + "volume", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "cluster": true, + "datacenter": true, + "instance": true, + "job": true + }, + "indexByName": { + "Value": 7, + "cluster": 0, + "datacenter": 1, + "instance": 2, + "job": 3, + "lun": 6, + "svm": 4, + "volume": 5 + }, + "renameByName": { + "Value": "Avg", + "svm": "", + "volume": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Read bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "lun" + }, + "properties": [ + { + "id": "displayName", + "value": "LUN" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-lun/ontap-lun?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${Volume:queryparam}&${__url_time_range}&var-LUN=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 2 + }, + "id": 44, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \n and \ntopk($TopResources, lun_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} @ end())", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Read Throughput", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "instance", + "job", + "lun", + "svm", + "volume", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "cluster": true, + "datacenter": true, + "instance": true, + "job": true + }, + "indexByName": { + "Value": 7, + "cluster": 0, + "datacenter": 1, + "instance": 2, + "job": 3, + "lun": 6, + "svm": 4, + "volume": 5 + }, + "renameByName": { + "Value": "Avg", + "svm": "", + "volume": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of read operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "lun" + }, + "properties": [ + { + "id": "displayName", + "value": "LUN" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-lun/ontap-lun?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${Volume:queryparam}&${__url_time_range}&var-LUN=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 2 + }, + "id": 46, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \n and \ntopk($TopResources, lun_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} @ end())", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Read IOPS", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "instance", + "job", + "lun", + "svm", + "volume", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "cluster": true, + "datacenter": true, + "instance": true, + "job": true + }, + "indexByName": { + "Value": 7, + "cluster": 0, + "datacenter": 1, + "instance": 2, + "job": 3, + "lun": 6, + "svm": 4, + "volume": 5 + }, + "renameByName": { + "Value": "Avg", + "svm": "", + "volume": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average write latency in microseconds for all operations on the LUN.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Avg" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "lun" + }, + "properties": [ + { + "id": "displayName", + "value": "LUN" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-lun/ontap-lun?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${Volume:queryparam}&${__url_time_range}&var-LUN=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 43, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_avg_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \nand \ntopk($TopResources, lun_avg_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} @ end())", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{svm}}/{{volume}}/{{lun}}", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Write Latency", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "instance", + "job", + "lun", + "svm", + "volume", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "cluster": true, + "datacenter": true, + "instance": true, + "job": true + }, + "indexByName": { + "Value": 7, + "cluster": 0, + "datacenter": 1, + "instance": 2, + "job": 3, + "lun": 6, + "svm": 4, + "volume": 5 + }, + "renameByName": { + "Value": "Avg", + "svm": "", + "volume": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Write bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "lun" + }, + "properties": [ + { + "id": "displayName", + "value": "LUN" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-lun/ontap-lun?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${Volume:queryparam}&${__url_time_range}&var-LUN=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 45, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \nand \ntopk($TopResources, lun_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} @ end())", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Write Throughput", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "instance", + "job", + "lun", + "svm", + "volume", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "cluster": true, + "datacenter": true, + "instance": true, + "job": true + }, + "indexByName": { + "Value": 7, + "cluster": 0, + "datacenter": 1, + "instance": 2, + "job": 3, + "lun": 6, + "svm": 4, + "volume": 5 + }, + "renameByName": { + "Value": "Avg", + "svm": "", + "volume": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of write operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "lun" + }, + "properties": [ + { + "id": "displayName", + "value": "LUN" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-lun/ontap-lun?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${Volume:queryparam}&${__url_time_range}&var-LUN=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 47, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \nand \ntopk($TopResources, lun_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} @ end())", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Write IOPS", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "instance", + "job", + "lun", + "svm", + "volume", + "Value" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "cluster": true, + "datacenter": true, + "instance": true, + "job": true + }, + "indexByName": { + "Value": 7, + "cluster": 0, + "datacenter": 1, + "instance": 2, + "job": 3, + "lun": 6, + "svm": 4, + "volume": 5 + }, + "renameByName": { + "Value": "Avg", + "svm": "", + "volume": "" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "LUN details.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "1": { + "text": "online" + } + }, + "type": "value" + }, + { + "options": { + "from": 0, + "result": { + "text": "offline" + }, + "to": 0.99 + }, + "type": "range" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(83, 179, 59)", + "value": null + }, + { + "color": "semi-dark-red", + "value": 5 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "custom.width", + "value": 153 + }, + { + "id": "displayName", + "value": "status" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "displayName", + "value": "size" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "displayName", + "value": "size used" + }, + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "max", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "node" + }, + "properties": [ + { + "id": "displayName", + "value": "Node" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-node/ontap-node?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-Node=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "lun" + }, + "properties": [ + { + "id": "displayName", + "value": "LUN" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-lun/ontap-lun?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${Volume:queryparam}&${__url_time_range}&var-LUN=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 18, + "interval": "1m", + "maxDataPoints": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "space used" + } + ] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_new_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "D" + } + ], + "title": "LUNS in Cluster", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "cluster": true, + "datacenter": true, + "instance": true, + "job": true, + "state": true + }, + "indexByName": { + "Time": 0, + "Value": 12, + "__name__": 1, + "aggr": 4, + "cluster": 5, + "datacenter": 6, + "instance": 7, + "job": 8, + "node": 2, + "state": 9, + "style": 11, + "svm": 3, + "volume": 10 + }, + "renameByName": {} + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "node", + "svm", + "lun", + "Value #A", + "Value #C", + "Value #D", + "volume" + ] + } + } + } + ], + "type": "table" + } + ], + "title": "LUN Table", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 28, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average read latency in microseconds for all operations on the LUN.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 48, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_avg_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \n and \ntopk($TopResources, avg_over_time(lun_avg_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}}", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Average Read Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Read bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \n and \ntopk($TopResources, avg_over_time(lun_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}[3h] @ end()))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{svm}} / {{volume}} / {{lun}}", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Read Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of read operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \n and \ntopk($TopResources, avg_over_time(lun_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}}", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Read IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average write latency in microseconds for all operations on the LUN.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 51, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_avg_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \n and \ntopk($TopResources, avg_over_time(lun_avg_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}}", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Average Write Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Write bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \n and \ntopk($TopResources, avg_over_time(lun_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}[3h] @ end()))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{svm}} / {{volume}} / {{lun}}", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Write Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of write operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 53, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \n and \ntopk($TopResources, avg_over_time(lun_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}}", + "refId": "A" + } + ], + "title": "Top $TopResources Luns by Write IOPs", + "type": "timeseries" + } + ], + "title": "Top LUN Performance", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 61, + "panels": [ + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 73, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This dashboard requires Workload/QOS counters templates. Two actions are required to use this dashboard:
\n1. If you are using ZapiPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/zapiperf/default.yaml.
\n2. If you are using RestPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/restperf/default.yaml.
\n\nRefer ONTAP Performance - Resolution Guide https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/ONTAP_9_Performance_-_Resolution_Guide", + "mode": "markdown" + }, + "pluginVersion": "11.3.0", + "title": "Important Information on Top Volume QoS Latency Graphs", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 63, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"network\",volume!=\"\"} \n and \ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"network\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Network", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 64, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"throttle\",volume!=\"\"} \n and \ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"throttle\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Throttle", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 95, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"qos_min\",volume!=\"\"} \n and \ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"qos_min\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Qos Min", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 66, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"cluster\",volume!=\"\"} \n and \ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"cluster\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Cluster", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 67, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"backend\",volume!=\"\"} \n and \ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"backend\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Backend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 97, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"cp\",volume!=\"\"} \n and \ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"cp\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from CP (Consistency Point)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 26 + }, + "id": 69, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"suspend\",volume!=\"\"} \n and \ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"suspend\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Suspend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 26 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"cloud\",volume!=\"\"} \n and \ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"cloud\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Cloud", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 26 + }, + "id": 65, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"frontend\",volume!=\"\"} \n and \ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"frontend\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Frontend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 35 + }, + "id": 71, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"nvlog\",volume!=\"\"} \n and \ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"nvlog\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from NVLog Mirroring", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 35 + }, + "id": 68, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"disk\",volume!=\"\"} \n and \ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",resource=\"disk\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Disk", + "type": "timeseries" + } + ], + "title": "Top Volume Latency from QoS", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 75, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "percentage of utilized storage space in a volume relative to its total capacity.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 77, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_size_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\"} \n and \ntopk($TopResources, avg_over_time(volume_size_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Used %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Percentage of snapshot reserve size that has been used.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 78, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_snapshot_reserve_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\"} \n and \ntopk($TopResources, avg_over_time(volume_snapshot_reserve_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Snapshot Used %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Percentage of the total disk space that is saved by deduplication and file cloning.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 80, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_sis_dedup_saved_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\"} \n and \ntopk($TopResources, avg_over_time(volume_sis_dedup_saved_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Deduplication Percent Saved", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Percentage of the total disk space that is saved by compressing blocks on the referenced file system.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 81, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_sis_compress_saved_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\"} \n and \ntopk($TopResources, avg_over_time(volume_sis_compress_saved_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Compression Percent Saved", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "LUN used percent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 98, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_size_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \n and \ntopk($TopResources, avg_over_time(lun_size_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}[3h] @ end()))", + "instant": false, + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}}", + "refId": "A" + } + ], + "title": "Top $TopResources LUNs by Percent Most Filled", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "LUN used percent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 83, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_size_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"} \n and \nbottomk($TopResources, avg_over_time(lun_size_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}[3h] @ end()))", + "instant": false, + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}}", + "refId": "A" + } + ], + "title": "Top $TopResources LUNs by Percent Least Filled", + "type": "timeseries" + } + ], + "title": "Top Volume and LUN Capacity", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 85, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This panels displays average read/write latency in microseconds for all operations on the LUN.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 87, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_avg_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}} Read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_avg_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}} Write", + "refId": "B" + } + ], + "title": "Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This panel displays Read/Write bytes data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 88, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}} Read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}} Write", + "refId": "B" + } + ], + "title": "Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This panel displays number of read/write operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 89, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}} Read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}} Write", + "refId": "B" + } + ], + "title": "IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This panel displays detail of I/O to or from a LUN which is not owned by the storage system handling the I/O and number of operations received by a storage system that does not own the LUN targeted by the operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "iops" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 90, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_remote_bytes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}} Remote Data", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_remote_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}} Remote IOPs", + "refId": "B" + } + ], + "title": "Indirect Access", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This panel displays detail of number of compare and write requests, number of unmap command requests, number of write same command requests, number of write same commands requests with unmap bit set and total number of xcopy operations on the LUN.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 91, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_caw_reqs{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "interval": "", + "legendFormat": "Compare and Write", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_unmap_reqs{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "hide": false, + "interval": "", + "legendFormat": "Unmap", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_writesame_reqs{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "hide": false, + "interval": "", + "legendFormat": "Write Same", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_writesame_unmap_reqs{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "hide": false, + "interval": "", + "legendFormat": "Write Same Unmap", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_xcopy_reqs{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}", + "hide": false, + "interval": "", + "legendFormat": "Xcopy", + "refId": "E" + } + ], + "title": "vStorage Offload Operations", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "include": [ + "Compare and Write", + "Unmap", + "Write Same", + "Write Same Unmap", + "Xcopy" + ], + "reducer": "sum" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "deckbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 93, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_read_data/lun_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}/1024", + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}} Read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "lun_write_data/lun_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume|\",lun=~\"$LUN\"}/1024", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} / {{volume}} / {{lun}} Write", + "refId": "B" + } + ], + "title": "IO Size", + "type": "timeseries" + } + ], + "title": "Per LUN (Must Select Cluster/SVM/Volume/LUN)", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 2, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": [ + "DC-01" + ], + "value": [ + "DC-01" + ] + }, + "datasource": "prometheus", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\"}, datacenter)", + "includeAll": false, + "multi": true, + "name": "Datacenter", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\"}, datacenter)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "includeAll": true, + "multi": true, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "prometheus", + "definition": "label_values(lun_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)", + "includeAll": true, + "multi": true, + "name": "SVM", + "options": [], + "query": { + "query": "label_values(lun_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "prometheus", + "definition": "label_values(lun_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}, volume)", + "includeAll": true, + "multi": true, + "name": "Volume", + "options": [], + "query": { + "query": "label_values(lun_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}, volume)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "prometheus", + "definition": "label_values(lun_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\", volume=~\"$Volume|\"},lun)", + "includeAll": true, + "multi": true, + "name": "LUN", + "options": [], + "query": { + "query": "label_values(lun_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\", volume=~\"$Volume|\"},lun)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "5", + "value": "5" + }, + "includeAll": false, + "name": "TopResources", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "2", + "value": "2" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": false, + "text": "4", + "value": "4" + }, + { + "selected": true, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "6", + "value": "6" + }, + { + "selected": false, + "text": "8", + "value": "8" + }, + { + "selected": false, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "15", + "value": "15" + }, + { + "selected": false, + "text": "25", + "value": "25" + }, + { + "selected": false, + "text": "50", + "value": "50" + }, + { + "selected": false, + "text": "100", + "value": "100" + }, + { + "selected": false, + "text": "250", + "value": "250" + }, + { + "selected": false, + "text": "500", + "value": "500" + } + ], + "query": "1,2,3,4,5,6,8,10,15,25,50,100,250,500", + "type": "custom" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "FSxN: LUN", + "uid": "cdot-lun", + "version": 4, + "weekStart": "" +} \ No newline at end of file diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_SVM.json b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_SVM.json new file mode 100644 index 00000000..f9bb375d --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_SVM.json @@ -0,0 +1,13339 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 36, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cdot" + ], + "targetBlank": false, + "title": "Related Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 68, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process all the operations on the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 64, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_vol_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM Average Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total throughput.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 65, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_vol_read_data+svm_vol_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of operations per second serviced by the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 66, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_vol_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process read request to the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 69, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_vol_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM Average Read Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process write request to the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 72, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_vol_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM Average Write Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Bytes read per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 70, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_vol_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM Read Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Bytes written per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 5 + }, + "id": 73, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_vol_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM Write Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of read operations per second from the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 5 + }, + "id": 71, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_vol_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM Read IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of write operations per second to the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 5 + }, + "id": 74, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_vol_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM Write IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process read/write/other request to the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 75, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_vol_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_vol_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Write", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_vol_other_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Other", + "refId": "C" + } + ], + "title": "SVM Average Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Bytes read/written per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 76, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_vol_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_vol_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "title": "SVM Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of read/write/other operations per second from the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 77, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_vol_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_vol_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Write", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_vol_other_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Other", + "refId": "C" + } + ], + "title": "SVM IOPs", + "type": "timeseries" + } + ], + "title": "Highlights", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 607, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This panel displays the top $TopResources SVMs based on the space used across their volumes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Used", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 631, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (datacenter, cluster, svm) (\n volume_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", style=~\"flexvol|flexgroup\"}\n)\nand on(datacenter, cluster, svm)\ntopk(\n $TopResources,\n sum by (datacenter, cluster, svm) (\n avg_over_time(volume_size_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", style=~\"flexvol|flexgroup\"}[3h])\n )\n)", + "hide": false, + "interval": "", + "legendFormat": "{{cluster}} - {{svm}}", + "refId": "B" + } + ], + "title": "Top $TopResources SVMs by Volume Space Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This panel displays the top $TopResources SVMs based on the logical space used across their volumes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Used", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 630, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (datacenter, cluster, svm) (volume_space_logical_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", style=~\"flexvol|flexgroup\"})\nand on(datacenter, cluster, svm)\ntopk(\n $TopResources,\n sum by (datacenter, cluster, svm) (\n avg_over_time(volume_space_logical_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", style=~\"flexvol|flexgroup\"}[3h])\n )\n)", + "hide": false, + "interval": "", + "legendFormat": "{{cluster}} - {{svm}}", + "refId": "B" + } + ], + "title": "Top $TopResources SVMs by Logical Space Usage Across Volumes", + "type": "timeseries" + } + ], + "title": "Capacity", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 79, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process read request to the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 367 + }, + "id": 442, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{volume}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Read Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Bytes read per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 367 + }, + "id": 443, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{volume}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Read Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of read operations per second from the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 367 + }, + "id": 444, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{volume}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Read IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process write request to the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 378 + }, + "id": 339, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{volume}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Write Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Bytes written per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 378 + }, + "id": 390, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{volume}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Write Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of write operations per second to the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 378 + }, + "id": 391, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{volume}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Write IOPs", + "type": "timeseries" + } + ], + "title": "Volume Performance", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 138, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Size that is physically used in the volume, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Used", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 390 + }, + "id": 310, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_space_physical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_space_physical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "B" + } + ], + "title": "Top $TopResources Volumes Per Physical Space Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Sum of (physical-used, shared_refs, compression_saved_in_plane0, vbn_zero, future_blk_cnt), in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Used", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 390 + }, + "id": 311, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_space_logical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_space_logical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "C" + } + ], + "title": "Top $TopResources Volumes Per Logical Space Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The total disk space (in bytes) that is saved by deduplication and file cloning.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 402 + }, + "id": 238, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_sis_dedup_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_sis_dedup_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{volume}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Deduplication Savings", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The total disk space (in bytes) that is saved by compressing blocks on the referenced file system.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 402 + }, + "id": 239, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_sis_compress_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_sis_compress_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{volume}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Compression Savings", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total space saved (in bytes) in the volume due to deduplication, compression, and file cloning.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 410 + }, + "id": 240, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_sis_total_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_sis_total_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{volume}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Total Efficiency Savings", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The virtual space used (includes volume reserves) before storage efficiency, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Used", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 420 + }, + "id": 312, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes Per Volume Size Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Physical size of the volume, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 420 + }, + "id": 313, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes Per Volume Total Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Available space for Snapshot copies from snap-reserve, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 432 + }, + "id": 314, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_snapshots_size_available{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_snapshots_size_available{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "D" + } + ], + "title": "Top $TopResources Volumes Per Snapshot Size Available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Size in the volume that has been set aside as a reserve for Snapshot copy usage, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 432 + }, + "id": 315, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_snapshot_reserve_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_snapshot_reserve_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "B" + } + ], + "title": "Top $TopResources Volumes Per Snapshot Reserve Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Size available for Snapshot copies within the Snapshot copy reserve, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 444 + }, + "id": 316, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_snapshot_reserve_available{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_snapshot_reserve_available{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "C" + } + ], + "title": "Top $TopResources Volumes Per Snapshot Reserve Available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The total space used by Snapshot copies in the volume, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 444 + }, + "id": 317, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_snapshots_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(volume_snapshots_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes Per Snapshot Size Used", + "type": "timeseries" + } + ], + "title": "Volume Capacity", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 98, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency for CIFS operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 996 + }, + "id": 99, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_cifs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM CIFS Average Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "SVM cifs total Iops.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 996 + }, + "id": 103, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_cifs_op_count{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM CIFS IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "SVM CIFS IOPs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 996 + }, + "id": 107, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_cifs_connections{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}", + "interval": "", + "legendFormat": "Connections", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_cifs_open_files{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}", + "hide": false, + "interval": "", + "legendFormat": "Open Files", + "refId": "B" + } + ], + "title": "SVM CIFS Connections and Open Files", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency for CIFS read operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1000 + }, + "id": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_cifs_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM CIFS Average Read Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency for CIFS write operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1000 + }, + "id": 101, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_cifs_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM CIFS Average Write Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of CIFS read operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1000 + }, + "id": 104, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_cifs_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM CIFS Read IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of CIFS write operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1000 + }, + "id": 105, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_cifs_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM CIFS Write IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency for CIFS read/write/average operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 1004 + }, + "id": 102, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_cifs_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_cifs_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Write", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_cifs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Average", + "refId": "C" + } + ], + "title": "SVM CIFS Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "SVM CIFS IOPs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 1004 + }, + "id": 106, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_cifs_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Read", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_cifs_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Write", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_cifs_op_count{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "Total", + "refId": "A" + } + ], + "title": "SVM CIFS IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Array of select CIFS operation counts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 1004 + }, + "id": 108, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_cifs_op_count{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}", + "interval": "", + "legendFormat": "{{metric}}", + "refId": "A" + } + ], + "title": "SVM CIFS IOP by Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "dispatch_latency: Latency by ONTAP to select a scanner and until the selected scanner pulls the request.\n\nscan_latency: Latency by the scanner after it pulls the request and until it responds, including scan time of the AV software.\n\nNote, the scan latency is per request (file) and not per IOP.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1163 + }, + "id": 242, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_vscan_dispatch_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}} - dispatch latency", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_vscan_scan_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}", + "hide": false, + "interval": "", + "legendFormat": "{{cluster}} - {{svm}}", + "refId": "B" + } + ], + "title": "Virus Scan Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "scan_request_dispatched_rate: Number of virus-scanning requests sent from Data ONTAP to the Vscan servers per second.\n\nscan_noti_received_rate: Number of virus-scanning requests received back by Data ONTAP from the Vscan servers per second.\n\nIf (dispatched > received) then requests are failing or delayed and deeper troubleshooting is recommended.\n\nNote, the rate is per request (file) and not per IOP.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1163 + }, + "id": 243, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_vscan_scan_noti_received_rate{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_vscan_scan_request_dispatched_rate{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}", + "hide": false, + "interval": "", + "legendFormat": "{{cluster}} - {{svm}}", + "refId": "B" + } + ], + "title": "Virus Scan Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "connections_active: Number of active connections from ONTAP nodes to virus scan servers.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1163 + }, + "id": 244, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_vscan_connections_active{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}", + "interval": "", + "legendFormat": "{{cluster}} - {{svm}}", + "refId": "A" + } + ], + "title": "Virus Scan Connections Active", + "type": "timeseries" + } + ], + "title": "CIFS", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 124, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency for iSCSI operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 191, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(iscsi_lif_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM iSCSI Average Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Iscsi total throughput.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 196, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(iscsi_lif_read_data+iscsi_lif_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM iSCSI Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Iscsi total Iops.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 197, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(iscsi_lif_iscsi_read_ops+iscsi_lif_iscsi_write_ops+iscsi_lif_iscsi_other_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM iSCSI IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency for read operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 10 + }, + "id": 192, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(iscsi_lif_avg_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM iSCSI Average Read Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency for write operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 10 + }, + "id": 202, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(iscsi_lif_avg_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}) / 1000", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM iSCSI Average Write Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Amount of data read from the storage system in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 10 + }, + "id": 195, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(iscsi_lif_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM iSCSI Read Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Amount of data written to the storage system in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 10 + }, + "id": 203, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(iscsi_lif_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM iSCSI Write Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "iSCSI read operations per second on this logical interface (LIF).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 10 + }, + "id": 198, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(iscsi_lif_iscsi_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM iSCSI Read IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "iSCSI write operations per second on this logical interface (LIF).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 10 + }, + "id": 204, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(iscsi_lif_iscsi_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM iSCSI Write IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Amount of data read from the storage system in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 194, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "iscsi_lif_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(iscsi_lif_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{node}} - {{lif}}", + "refId": "A" + } + ], + "title": "Top $TopResources iSCSI LIFs by Send Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Amount of data written to the storage system in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 200, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "iscsi_lif_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(iscsi_lif_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{node}} - {{lif}}", + "refId": "A" + } + ], + "title": "Top $TopResources iSCSI LIFs by Receive Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency for read/write/other operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 1000 + }, + "id": 193, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(iscsi_lif_avg_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(iscsi_lif_avg_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Write", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(iscsi_lif_avg_other_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Other", + "refId": "C" + } + ], + "title": "SVM iSCSI Average Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Amount of data read from the storage system/written to the storage system in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 1000 + }, + "id": 201, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(iscsi_lif_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "Read", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(iscsi_lif_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "Write", + "refId": "B" + } + ], + "title": "SVM iSCSI Throughput", + "type": "timeseries" + } + ], + "title": "iSCSI", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 37, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of NFSv4 requests. This counter keeps track of the average response time of NFSv4 requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 25 + }, + "id": 39, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv3 Avg Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv3 data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 25 + }, + "id": 50, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv3 Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of NFSv3 procedure requests per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 25 + }, + "id": 46, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv3 IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of READ operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 161 + }, + "id": 48, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv3 Read Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of WRITE operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 161 + }, + "id": 47, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv3 Write Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv3 read data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 161 + }, + "id": 150, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv3 Read Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "data transfers.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 161 + }, + "id": 151, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv3 Write Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total observed NFSv3 read operations per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 161 + }, + "id": 152, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv3 Read IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total observed NFSv3 write operations per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 161 + }, + "id": 153, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv3 Write IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of NFSv3 Read/Write procedure requests. The counter keeps track of the average response time of Read/Write requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 268 + }, + "id": 51, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read Latency" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write Latency" + } + ], + "title": "Top $TopResources NFSv3 SVMs by Read and Write Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv3 read/write/total data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 268 + }, + "id": 53, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Total - {{svm}}", + "refId": "Total" + } + ], + "title": "Top $TopResources NFSv3 SVMs by Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total observed NFSv3 read/write/total operations per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 268 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read Ops" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write Ops" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[3h] @ end()))", + "interval": "", + "legendFormat": "Total - {{svm}}", + "refId": "Total Ops" + } + ], + "title": "Top $TopResources NFSv3 SVMs by IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue" + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 276 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg by(__name__) ({__name__=~\"svm_nfs_.+_avg_latency\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"})", + "hide": false, + "interval": "", + "legendFormat": "{{__name__}}", + "refId": "Read Throughput" + } + ], + "title": "NFSv3 Latency by Op Type", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "svm_nfs_(.*)_avg_latency", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue" + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 276 + }, + "id": 54, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (__name__) ({__name__=~\"svm_nfs_.+_total\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"})", + "hide": false, + "interval": "", + "legendFormat": "{{__name__}}", + "refId": "Read Throughput" + } + ], + "title": "NFSv3 IOPs per Type", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "svm_nfs_(.*)_total", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + } + ], + "title": "NFSv3", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 126, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of NFSv4 requests. This counter keeps track of the average response time of NFSv4 requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 285 + }, + "id": 154, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4 Avg Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv3 data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 285 + }, + "id": 155, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4 Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of NFSv3 procedure requests per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 285 + }, + "id": 166, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv4 IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of READ operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 290 + }, + "id": 158, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4 Read Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of WRITE operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 290 + }, + "id": 159, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4 Write Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv3 read data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 290 + }, + "id": 160, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4 Read Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "data transfers.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 290 + }, + "id": 161, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4 Write Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of READ operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 290 + }, + "id": 162, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_read_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv4 Read IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of WRITE procedures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 290 + }, + "id": 163, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_write_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv4 Write IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of NFSv4 Read/Write procedure requests. The counter keeps track of the average response time of Read/Write requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 295 + }, + "id": 145, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read Latency" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write Latency" + } + ], + "title": "Top $TopResources NFSv4 SVMs by Read and Write Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv4 read/write data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 295 + }, + "id": 146, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read Latency" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write Latency" + } + ], + "title": "Top $TopResources NFSv4 SVMs by Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total observed NFSv4 read/write/total operations per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue" + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 295 + }, + "id": 147, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read Ops" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write Ops" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[3h] @ end()))", + "interval": "", + "legendFormat": "Total - {{svm}}", + "refId": "Total Ops" + } + ], + "title": "Top $TopResources NFSv4 SVMs by IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 303 + }, + "id": 157, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg by(__name__) ({__name__=~\"svm_nfs_.+_avg_latency\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", + "hide": false, + "interval": "", + "legendFormat": "{{__name__}}", + "refId": "Read Throughput" + } + ], + "title": "NFSv4 Latency by Op Type", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "svm_nfs_(.*)_avg_latency", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 303 + }, + "id": 149, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (__name__) ({__name__=~\"svm_nfs_.+_total\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", + "hide": false, + "interval": "", + "legendFormat": "{{__name__}}", + "refId": "Read Throughput" + } + ], + "title": "NFSv4 IOPs per Type", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "svm_nfs_(.*)_total", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + } + ], + "title": "NFSv4", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 130, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of NFSv4 requests. This counter keeps track of the average response time of NFSv4 requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 312 + }, + "id": 164, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4.1 Avg Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv3 data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 312 + }, + "id": 165, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4.1 Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of NFSv3 procedure requests per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 312 + }, + "id": 156, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv4.1 IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of READ operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 317 + }, + "id": 140, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4.1 Read Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of WRITE operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 317 + }, + "id": 167, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4.1 Write Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv4 read data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 317 + }, + "id": 168, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": " NFSv4.1 Read Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "data transfers.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 317 + }, + "id": 169, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": " NFSv4.1 Write Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of READ operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 317 + }, + "id": 170, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_read_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv4.1 Read IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of WRITE procedures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 317 + }, + "id": 171, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_write_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv4.1 Write IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of NFSv4.1 Read/Write procedure requests. The counter keeps track of the average response time of Read/Write requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 322 + }, + "id": 172, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read Latency" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write Latency" + } + ], + "title": "Top $TopResources NFSv4.1 SVMs by Read and Write Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv4.1 read/write data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 322 + }, + "id": 173, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read Latency" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write Latency" + } + ], + "title": "Top $TopResources NFSv4.1 SVMs by Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total observed NFSv4.1 read/write/total operations per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 322 + }, + "id": 174, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read Ops" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write Ops" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[3h] @ end()))", + "interval": "", + "legendFormat": "Total - {{svm}}", + "refId": "Total Ops" + } + ], + "title": "Top $TopResources NFSv4.1 SVMs by IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 330 + }, + "id": 175, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg by(__name__) ({__name__=~\"svm_nfs_.+_avg_latency\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", + "hide": false, + "interval": "", + "legendFormat": "{{__name__}}", + "refId": "Read Throughput" + } + ], + "title": "NFSv4.1 Latency by Op Type", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "svm_nfs_(.*)_avg_latency", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 330 + }, + "id": 176, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (__name__) ({__name__=~\"svm_nfs_.+_total\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", + "hide": false, + "interval": "", + "legendFormat": "{{__name__}}", + "refId": "Read Throughput" + } + ], + "title": "NFSv4.1 IOPs per Type", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "svm_nfs_(.*)_total", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + } + ], + "title": "NFSv4.1", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 486, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of NFSv4 requests. This counter keeps track of the average response time of NFSv4 requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 313 + }, + "id": 528, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4.2 Avg Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv3 data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 313 + }, + "id": 530, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4.2 Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of NFSv3 procedure requests per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 313 + }, + "id": 532, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv4.2 IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of READ operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 318 + }, + "id": 534, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4.2 Read Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of WRITE operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 318 + }, + "id": 536, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "NFSv4.2 Write Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv4 read data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 318 + }, + "id": 538, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": " NFSv4.2 Read Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "data transfers.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 318 + }, + "id": 540, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": " NFSv4.2 Write Throughput", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of READ operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 318 + }, + "id": 542, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_read_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv4.2 Read IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of WRITE procedures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "Value #Read Ops": { + "index": 0, + "text": "-read-" + }, + "Value #Total Ops": { + "index": 2, + "text": "TOTAL" + }, + "Value #Write Ops": { + "index": 1, + "text": "-write-" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 318 + }, + "id": 544, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(svm_nfs_write_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "Total Ops" + } + ], + "title": "NFSv4.2 Write IOPs", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Field": "Metric", + "Last (not null)": "Avg" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency of NFSv4.2 Read/Write procedure requests. The counter keeps track of the average response time of Read/Write requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 323 + }, + "id": 546, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read Latency" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write Latency" + } + ], + "title": "Top $TopResources NFSv4.2 SVMs by Read and Write Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Rate of NFSv4.2 read/write data transfers per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 323 + }, + "id": 548, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read Latency" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_throughput{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write Latency" + } + ], + "title": "Top $TopResources NFSv4.2 SVMs by Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total observed NFSv4.2 read/write/total operations per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 323 + }, + "id": 550, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_read_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_read_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "Read Ops" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_write_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_write_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "Write Ops" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}\nand\ntopk($TopResources, avg_over_time(svm_nfs_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"}[3h] @ end()))", + "interval": "", + "legendFormat": "Total - {{svm}}", + "refId": "Total Ops" + } + ], + "title": "Top $TopResources NFSv4.2 SVMs by IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 331 + }, + "id": 552, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg by(__name__) ({__name__=~\"svm_nfs_.+_avg_latency\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"})", + "hide": false, + "interval": "", + "legendFormat": "{{__name__}}", + "refId": "Read Throughput" + } + ], + "title": "NFSv4.2 Latency by Op Type", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "svm_nfs_(.*)_avg_latency", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 331 + }, + "id": 554, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (__name__) ({__name__=~\"svm_nfs_.+_total\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.2\"})", + "hide": false, + "interval": "", + "legendFormat": "{{__name__}}", + "refId": "Read Throughput" + } + ], + "title": "NFSv4.2 IOPs per Type", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "svm_nfs_(.*)_total", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + } + ], + "title": "NFSv4.2", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 134, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the average response time for requests that were initiated by the workload.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 209, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(qos_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Average QoS Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Qos total throughput.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 212, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(qos_read_data+qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "QoS Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This field is the workload's rate of operations that completed during the measurement interval; measured per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 31 + }, + "id": 210, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(qos_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "QoS IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the average response time for read requests that were initiated by the workload.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 158 + }, + "id": 211, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(qos_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Average QoS Read Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the average response time for write requests that were initiated by the workload.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 158 + }, + "id": 213, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(qos_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Average QoS Write Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the amount of data read per second from the filer by the workload.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 158 + }, + "id": 214, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(qos_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "QoS Read Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the amount of data written per second to the filer by the workload.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 158 + }, + "id": 215, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "QoS Write Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the rate of this workload's read operations that completed during the measurement interval.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 158 + }, + "id": 216, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(qos_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "QoS Read IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the workload's write operations that completed during the measurement interval; measured per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 158 + }, + "id": 217, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(qos_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "QoS Write IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the average response time for read/write requests that were initiated by the workload.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 162 + }, + "id": 218, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg by (svm) (qos_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})\nand\ntopk($TopResources, avg by (svm) (avg_over_time(qos_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end())))", + "hide": false, + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg by (svm) (qos_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})\nand\ntopk($TopResources, avg by (svm) (avg_over_time(qos_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end())))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "B" + } + ], + "title": "Top $TopResources SVMs by QoS Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the amount of data read/written per second from the filer by the workload.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 162 + }, + "id": 219, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (svm) (qos_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})\nand\ntopk($TopResources, sum by (svm) (avg_over_time(qos_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",svm!=\"\"}[3h] @ end())))", + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (svm) (qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})\nand\ntopk($TopResources, sum by (svm) (avg_over_time(qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",svm!=\"\"}[3h] @ end())))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "B" + } + ], + "title": "Top $TopResources SVMs by QoS Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the rate of this workload's read/write operations that completed during the measurement interval.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 162 + }, + "id": 220, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (svm) (qos_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})\nand\ntopk($TopResources, sum by (svm) (avg_over_time(qos_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",svm!=\"\"}[3h] @ end())))", + "interval": "", + "legendFormat": "Read - {{svm}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (svm) (qos_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})\nand\ntopk($TopResources, sum by (svm) (avg_over_time(qos_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",svm!=\"\"}[3h] @ end())))", + "hide": false, + "interval": "", + "legendFormat": "Write - {{svm}}", + "refId": "B" + } + ], + "title": "Top $TopResources SVMs by QoS IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the percentage of reads, performed on behalf of the workload, that were sequential.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 170 + }, + "id": 222, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_sequential_reads{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(qos_sequential_reads{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} - {{workload}}", + "refId": "A" + } + ], + "title": "Top $TopResources Workloads by Sequential Reads (%)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the percentage of writes, performed on behalf of the workload, that were sequential.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 170 + }, + "id": 585, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_sequential_writes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}\nand\ntopk($TopResources, avg_over_time(qos_sequential_writes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} - {{workload}}", + "refId": "A" + } + ], + "title": "Top $TopResources Workloads by Sequential Writes (%)", + "type": "timeseries" + } + ], + "title": "QoS Policy Group", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 136, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 32 + }, + "id": 224, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",resource=\"network\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume!=\"\",resource=\"network\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Workloads by Latency from Network", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 32 + }, + "id": 225, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",resource=\"throttle\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume!=\"\",resource=\"throttle\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Workloads by Latency from Throttle", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 32 + }, + "id": 227, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",resource=\"frontend\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume!=\"\",resource=\"frontend\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Workloads by Latency from Frontend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 64 + }, + "id": 226, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",resource=\"backend\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume!=\"\",resource=\"backend\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Workloads by Latency from Backend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 64 + }, + "id": 228, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",resource=\"cluster\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume!=\"\",resource=\"cluster\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Workloads by Latency from Cluster", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 64 + }, + "id": 229, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",resource=\"disk\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume!=\"\",resource=\"disk\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Workloads by Latency from Disk", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 72 + }, + "id": 230, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",resource=\"nvlog\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume!=\"\",resource=\"nvlog\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Workloads by Latency from NVLog Mirroring", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 72 + }, + "id": 231, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",resource=\"suspend\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume!=\"\",resource=\"suspend\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Workloads by Latency from Suspend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "average latency for workload on Data ONTAP subsystems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 72 + }, + "id": 232, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",resource=\"cloud\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume!=\"\",resource=\"cloud\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Workloads by Latency from Cloud", + "type": "timeseries" + } + ], + "title": "QoS Policy Group Latency from Resource", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 2, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": [ + "DC-01" + ], + "value": [ + "DC-01" + ] + }, + "datasource": "prometheus", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\"}, datacenter)", + "includeAll": false, + "multi": true, + "name": "Datacenter", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\"}, datacenter)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "includeAll": true, + "multi": true, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "label_values(svm_labels{cluster=~\"$Cluster\"}, svm)", + "includeAll": true, + "multi": true, + "name": "SVM", + "options": [], + "query": { + "query": "label_values(svm_labels{cluster=~\"$Cluster\"}, svm)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "5", + "value": "5" + }, + "includeAll": false, + "name": "TopResources", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "2", + "value": "2" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": false, + "text": "4", + "value": "4" + }, + { + "selected": true, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "6", + "value": "6" + }, + { + "selected": false, + "text": "8", + "value": "8" + }, + { + "selected": false, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "15", + "value": "15" + }, + { + "selected": false, + "text": "25", + "value": "25" + }, + { + "selected": false, + "text": "50", + "value": "50" + }, + { + "selected": false, + "text": "100", + "value": "100" + }, + { + "selected": false, + "text": "250", + "value": "250" + }, + { + "selected": false, + "text": "500", + "value": "500" + } + ], + "query": "1,2,3,4,5,6,8,10,15,25,50,100,250,500", + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "FSxN: Storage Virtual Machine", + "uid": "cdot-svm", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Volume.json b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Volume.json new file mode 100644 index 00000000..eae23a16 --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/FSxN_Volume.json @@ -0,0 +1,7947 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 37, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cdot" + ], + "targetBlank": false, + "title": "Related Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 20, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average Latency of top $TopResources volumes.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg(topk($TopResources, volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Volume Average Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Throughput of top $TopResources volumes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 23, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(\n topk(\n $TopResources,\n volume_read_data{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", style!=\"flexgroup_constituent\"}\n * on(datacenter, cluster, svm, volume) group_left(node)\n volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", tags=~\".*$Tag.*\"}\n )\n)\n+\nsum(\n topk(\n $TopResources,\n volume_write_data{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", style!=\"flexgroup_constituent\"}\n * on(datacenter, cluster, svm, volume) group_left(node)\n volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", tags=~\".*$Tag.*\"}\n )\n)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes Total Throughput", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "IOPs of top $TopResources volumes.", + "fieldConfig": { + "defaults": { + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum(topk($TopResources, volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Total IOPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process all the operations on the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_avg_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Average Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total throughput.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 5 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "(\n volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} \n +\n volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\n)\nand on (datacenter, cluster, svm, volume)\ntopk(\n $TopResources,\n (\n avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"} \n + avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}\n )\n)", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Average Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of operations per second serviced by the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_total_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Total IOPs", + "type": "timeseries" + } + ], + "title": "Highlights", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 37, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Volume details.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 176, 196)", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "offline" + }, + "1": { + "color": "green", + "index": 0, + "text": "online" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "custom.width" + }, + { + "id": "displayName", + "value": "Status" + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "displayName", + "value": "space used" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "max", + "value": 100 + }, + { + "id": "min", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dedupe Space Saved" + }, + "properties": [ + { + "id": "custom.width" + }, + { + "id": "displayName", + "value": "Dedupe Space Saved" + }, + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Compression Space Saved" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Space Saved" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Logical Space Used" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Physical Space Used" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Clone Split Estimate" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "node" + }, + "properties": [ + { + "id": "displayName", + "value": "Node" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-node/ontap-node?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-Node=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "aggr" + }, + "properties": [ + { + "id": "displayName", + "value": "Aggregate" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-aggregate/ontap-aggregate?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-Aggregate=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "datacenter" + }, + "properties": [ + { + "id": "displayName", + "value": "Datacenter" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-datacenter/ontap-datacenter?orgId=1&${__url_time_range}&var-Datacenter=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cluster" + }, + "properties": [ + { + "id": "displayName", + "value": "Cluster" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-cluster/ontap-cluster?orgId=1&${Datacenter:queryparam}&${__url_time_range}&var-Cluster=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 18, + "interval": "1m", + "maxDataPoints": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "space used" + } + ] + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",tags=~\".*$Tag.*\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "volume_new_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "volume_size_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "volume_size_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_sis_dedup_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_sis_compress_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_sis_dedup_saved+volume_sis_compress_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_space_logical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_space_physical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_clone_split_estimate{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"} * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "J" + } + ], + "title": "Volumes in Cluster", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "aggr", + "clone_parent_snapshot", + "clone_parent_svm", + "clone_parent_volume", + "junction_path", + "node", + "svm", + "volume", + "Value #A", + "Value #C", + "Value #D", + "Value #E", + "Value #F", + "Value #G", + "Value #H", + "Value #I", + "Value #J", + "datacenter", + "cluster", + "tags" + ] + } + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "Value #C": false, + "__name__": true, + "cluster": false, + "datacenter": false, + "instance": true, + "job": true, + "state": true + }, + "indexByName": { + "Value #A": 6, + "Value #C": 8, + "Value #D": 9, + "Value #E": 12, + "Value #F": 13, + "Value #G": 14, + "Value #H": 10, + "Value #I": 11, + "Value #J": 19, + "aggr": 5, + "clone_parent_snapshot": 18, + "clone_parent_svm": 16, + "clone_parent_volume": 17, + "cluster": 1, + "datacenter": 0, + "junction_path": 15, + "node": 4, + "svm": 2, + "tags": 7, + "volume": 3 + }, + "renameByName": { + "Value #C": "Size", + "Value #D": "", + "Value #E": "Dedupe Space Saved", + "Value #F": "Compression Space Saved", + "Value #G": "Total Space Saved", + "Value #H": "Logical Space Used", + "Value #I": "Physical Space Used", + "Value #J": "Clone Split Estimate", + "clone_parent_snapshot": "Clone Parent Snapshot", + "clone_parent_svm": "Clone Parent SVM", + "clone_parent_volume": "Clone Parent Volume", + "junction_path": "Junction Path", + "tags": "Tags" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process read request to the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "unit", + "value": "µs" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 477 + }, + "id": 39, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Read Latency", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "SVM / Volume", + "binary": { + "left": "svm", + "operator": "+", + "reducer": "sum", + "right": "volume" + }, + "mode": "binary", + "reduce": { + "include": [ + "svm", + "volume" + ], + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Avg", + "type": "{{svm}} / {{volume}}", + "volume": "" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "svm", + "volume", + "Avg" + ] + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Bytes read per second from the volume.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "transparent", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 477 + }, + "id": 41, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Read Throughput", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "SVM / Volume", + "binary": { + "left": "svm", + "operator": "+", + "reducer": "sum", + "right": "volume" + }, + "mode": "binary", + "reduce": { + "include": [ + "svm", + "volume" + ], + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Avg", + "type": "{{svm}} / {{volume}}", + "volume": "" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "svm", + "volume", + "Avg" + ] + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of reads per second from the volume.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "transparent", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "unit", + "value": "iops" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 477 + }, + "id": 43, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Read IOPS", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "SVM / Volume", + "binary": { + "left": "svm", + "operator": "+", + "reducer": "sum", + "right": "volume" + }, + "mode": "binary", + "reduce": { + "include": [ + "svm", + "volume" + ], + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Avg", + "type": "{{svm}} / {{volume}}", + "volume": "" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "svm", + "volume", + "Avg" + ] + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process write request to the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "unit", + "value": "µs" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 20000 + }, + { + "color": "red", + "value": 30000 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 484 + }, + "id": 40, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Write Latency", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "SVM / Volume", + "binary": { + "left": "svm", + "operator": "+", + "reducer": "sum", + "right": "volume" + }, + "mode": "binary", + "reduce": { + "include": [ + "svm", + "volume" + ], + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Avg", + "type": "{{svm}} / {{volume}}", + "volume": "" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "svm", + "volume", + "Avg" + ] + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Bytes written per second to the volume.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "transparent", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 484 + }, + "id": 42, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Write Throughput", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "SVM / Volume", + "binary": { + "left": "svm", + "operator": "+", + "reducer": "sum", + "right": "volume" + }, + "mode": "binary", + "reduce": { + "include": [ + "svm", + "volume" + ], + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Avg", + "type": "{{svm}} / {{volume}}", + "volume": "" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "svm", + "volume", + "Avg" + ] + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of writes per second to the volume.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "transparent", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "unit", + "value": "iops" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svm" + }, + "properties": [ + { + "id": "displayName", + "value": "SVM" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-svm/ontap-svm?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${__url_time_range}&var-SVM=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 484 + }, + "id": 44, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{svm}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Write IOPS", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "SVM / Volume", + "binary": { + "left": "svm", + "operator": "+", + "reducer": "sum", + "right": "volume" + }, + "mode": "binary", + "reduce": { + "include": [ + "svm", + "volume" + ], + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Avg", + "type": "{{svm}} / {{volume}}", + "volume": "" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "svm", + "volume", + "Avg" + ] + } + } + } + ], + "type": "table" + } + ], + "title": "Volume Table", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 28, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process read request to the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Read Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process write request to the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Write Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency in microseconds for the WAFL filesystem to process other operations to the volume; not including request processing or network communication time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 116, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_other_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_other_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Other Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of reads per second from the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 442 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Read IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of writes per second to the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 442 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Write IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of other operations per second to the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 442 + }, + "id": 115, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_other_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_other_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Other IOPs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Bytes read per second from the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 451 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Read Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Bytes written per second to the volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 451 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Write Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue" + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 460 + }, + "id": 112, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "avg by(__name__) ({__name__=~\"volume_nfs_(.*)_latency\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{__name__}}", + "refId": "Read Throughput" + } + ], + "title": "Volume Latency by Op Type", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "volume_nfs_(.*)_latency", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue" + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 460 + }, + "id": 114, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (__name__) ({__name__=~\"volume_nfs_(.*)_ops\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, + "interval": "", + "legendFormat": "{{__name__}}", + "refId": "Read Throughput" + } + ], + "title": "Volume IOPs per Type", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "volume_nfs_(.*)_ops", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + } + ], + "title": "Performance", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 142, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This panel displays the top k volumes with the highest I/O density within the selected time range. I/O density is calculated as the average number of I/O operations (IOPS) per terabyte of logical used space. Volumes with high I/O density are typically more active and may be critical for performance-sensitive applications.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 139, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "(\n volume_total_ops{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\n / on (datacenter, cluster, svm, volume)\n clamp_min(\n volume_space_logical_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"} / 1099511627776,\n 1\n )\n)\nand on (datacenter, cluster, svm, volume)\ntopk(\n $TopResources,\n (\n avg_over_time(\n volume_total_ops{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()\n ) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}\n /\n clamp_min(\n avg_over_time(\n volume_space_logical_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()\n ) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"} / 1099511627776,\n 1\n )\n )\n unless\n on(datacenter, cluster, svm, volume)\n volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", root_volume=\"Yes\",style!=\"flexgroup_constituent\"}\n)", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by IO Density (IOPs/TiB)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This panel displays the bottom k volumes with the lowest I/O density over the selected time range, excluding any volumes with average IOPS below 50. I/O density is calculated as the average IOPS per terabyte of logical used space. These volumes are active but have relatively low I/O activity compared to their used capacity.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 140, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "(\n volume_total_ops{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\n / on (datacenter, cluster, svm, volume)\n clamp_min(\n volume_space_logical_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"} / 1099511627776,\n 1\n )\n)\nand on(datacenter, cluster, svm, volume)\nbottomk(\n $TopResources,\n (\n avg_over_time(\n volume_total_ops{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()\n ) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"}\n /\n clamp_min(\n avg_over_time(\n volume_space_logical_used{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()\n ) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"} / 1099511627776,\n 1\n )\n )\n and\n on(datacenter, cluster, svm, volume)\n (\n volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\", root_volume!=\"Yes\",style!=\"flexgroup_constituent\"}\n * on(datacenter, cluster, svm, volume)\n avg_over_time(\n volume_total_ops{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()\n ) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"} > 50\n )\n)", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Bottom $TopResources Volumes by IO Density (IOPs/TiB)", + "type": "timeseries" + } + ], + "title": "I/O Density", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 66, + "panels": [ + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 91, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This panel requires Workload/QOS counter templates. They are disabled by default because they typically slow down data collection due to a high number of metrics. Two actions are required to use this panel:
\n1. If you are using ZapiPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/zapiperf/default.yaml.
\n2. If you are using RestPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/restperf/default.yaml.
\n\nRefer ONTAP Performance - Resolution Guide https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/ONTAP_9_Performance_-_Resolution_Guide", + "mode": "markdown" + }, + "pluginVersion": "11.3.0", + "title": "Important Information on Top Volume QoS Resource Latency Graphs", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the average response time for read requests that were initiated by the workload.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 387 + }, + "id": 72, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by QoS Volume Read Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the average response time for write requests that were initiated by the workload.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 387 + }, + "id": 73, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by QoS Volume Write Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the rate of this workload's read operations that completed during the measurement interval.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 397 + }, + "id": 76, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by QoS Volume Read IOPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the workload's write operations that completed during the measurement interval; measured per second.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 397 + }, + "id": 77, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by QoS Volume Write IOPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the amount of data read per second from the filer by the workload.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 407 + }, + "id": 74, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by QoS Volume Read Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the amount of data written per second to the filer by the workload.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 407 + }, + "id": 75, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by QoS Volume Write Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the percentage of reads, performed on behalf of the workload, that were sequential.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 417 + }, + "id": 79, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_sequential_reads{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_sequential_reads{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by QoS Volume Sequential Reads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is the percentage of writes, performed on behalf of the workload, that were sequential. This counter is only available on platforms with more than 4GB of NVRAM.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 417 + }, + "id": 80, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_sequential_writes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_sequential_writes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by QoS Volume Sequential Writes", + "type": "timeseries" + } + ], + "title": "QoS", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 46, + "panels": [ + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 103, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This panel requires Workload/QOS counter templates. They are disabled by default because they typically slow down data collection due to a high number of metrics. Two actions are required to use this panel:
\n1. If you are using ZapiPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/zapiperf/default.yaml.
\n2. If you are using RestPerf Collector, then On Harvest, You must enable the Workload/QOS counters in $HARVEST/conf/restperf/default.yaml.
\n\nRefer ONTAP Performance - Resolution Guide https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/ONTAP_9_Performance_-_Resolution_Guide", + "mode": "markdown" + }, + "pluginVersion": "11.3.0", + "title": "Important Information on Top Volume QoS Resource Latency Graphs", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "`Note:` Typically these latencies only apply to SAN not NAS.\n\nRepresents the wait time of I/O requests by the external networking protocols on the cluster. The wait time is time spent waiting for transfer ready transactions to finish before the cluster can respond to an I/O request. If the network component is in contention, it means high wait time at the protocol layer is impacting the latency of one or more workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 48, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"network\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"network\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Network", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Represents the throughput maximum (ceiling) setting of the storage Quality of Service (QoS) policy group assigned to the workload. If the policy group component is in contention, it means all workloads in the policy group are being throttled by the set throughput limit, which is impacting the latency of one or more of those workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 50, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"throttle\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"throttle\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Throttle", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Represents the latency to a workload that is being caused by QoS throughput floor (expected) setting assigned to other workloads. If the QoS floor set on certain workloads use the majority of the bandwidth to guarantee the promised throughput, other workloads will be throttled and see more latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 88, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"qos_min\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"qos_min\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Qos Min", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Represents delays caused by the cluster switches, cables, and adapters which physically connect clustered nodes. \n\nIf the cluster interconnect component is in contention, it means high wait time for I/O requests at the cluster interconnect is impacting the latency of one or more workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 34 + }, + "id": 54, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cluster\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cluster\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Cluster", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Represents the delays in the data/WAFL layer of ONTAP.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 34 + }, + "id": 56, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"backend\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"backend\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Backend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Represents delays due to buffered write flushes, called consistency points (cp).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 34 + }, + "id": 89, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cp\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cp\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from CP (Consistency Point)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Represents delays due to operations suspending on a delay mechanism. Typically this is diagnosed by NetApp Support.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 45 + }, + "id": 60, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"suspend\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"suspend\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Suspend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Represents the software component in the cluster involved with I/O processing between the cluster and the cloud tier on which user data is stored. If the cloud latency component is in contention, it means that a large amount of reads from volumes that are hosted on the cloud tier are impacting the latency of one or more workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 45 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cloud\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cloud\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Cloud", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Represents the delays in the network layer of ONTAP.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 45 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"frontend\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"frontend\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Frontend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Represents delays due to mirroring writes to the NVRAM/NVLOG memory and to the HA partner NVRAM/NVLOG memory.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 56 + }, + "id": 64, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"nvlog\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"nvlog\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from NVLog Mirroring", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Represents slowness due to attached hard drives or solid state drives.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 56 + }, + "id": 58, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"disk\",volume!=\"\"}\nand\ntopk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"disk\",volume!=\"\"}[3h] @ end()))", + "interval": "", + "legendFormat": "{{svm}} / {{workload}} / {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Latency from Disk", + "type": "timeseries" + } + ], + "title": "QoS Resource Latency", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 70, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Size that is physically used in the volume, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Used", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 202 + }, + "id": 83, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_space_physical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_space_physical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "B" + } + ], + "title": "Top $TopResources Volumes Per Physical Space Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Sum of (physical-used, shared_refs, compression_saved_in_plane0, vbn_zero, future_blk_cnt), in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Used", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 202 + }, + "id": 84, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_space_logical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_space_logical_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "C" + } + ], + "title": "Top $TopResources Volumes Per Logical Space Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The virtual space used (includes volume reserves) before storage efficiency, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Used", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 214 + }, + "id": 85, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes Per Volume Size Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Physical size of the volume, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 214 + }, + "id": 86, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "D" + } + ], + "title": "Top $TopResources Volumes Per Volume Total Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Available space for Snapshot copies from snap-reserve, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 226 + }, + "id": 87, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_snapshots_size_available{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_snapshots_size_available{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "D" + } + ], + "title": "Top $TopResources Volumes Per Snapshot Size Available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Size in the volume that has been set aside as a reserve for Snapshot copy usage, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 226 + }, + "id": 128, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_snapshot_reserve_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_snapshot_reserve_size{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "B" + } + ], + "title": "Top $TopResources Volumes Per Snapshot Reserve Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Size available for Snapshot copies within the Snapshot copy reserve, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 238 + }, + "id": 129, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_snapshot_reserve_available{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_snapshot_reserve_available{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "C" + } + ], + "title": "Top $TopResources Volumes Per Snapshot Reserve Available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The total space used by Snapshot copies in the volume, in bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 238 + }, + "id": 130, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_snapshots_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_snapshots_size_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes Per Snapshot Size Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The size that is physically used in the performance tier of the volume and has a cold temperature. This parameter is only supported if the volume is in an aggregate that is either attached to an object store or could be attached to an object store.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 250 + }, + "id": 150, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_space_performance_tier_inactive_user_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_space_performance_tier_inactive_user_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Inactive Data", + "type": "timeseries" + } + ], + "title": "Capacity", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 92, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency per GET operation to object storage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 500 + }, + "id": 93, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "fabricpool_cloud_bin_op_latency_average{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\", metric=\"GET\"}\nand\ntopk($TopResources, avg_over_time(fabricpool_cloud_bin_op_latency_average{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\", metric=\"GET\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}} - {{cloud_target}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Object Storage GET Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total GET request count, per volume, in the collection interval to object storage. Not scaled per second, but a cumulative count (delta) since the previous poll.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Request Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 500 + }, + "id": 94, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "fabricpool_cloud_bin_operation{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\", metric=\"GET\"}\nand\ntopk($TopResources, avg_over_time(fabricpool_cloud_bin_operation{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\", metric=\"GET\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}} - {{cloud_target}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Object Storage GET Request Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average latency per PUT operation to object storage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 512 + }, + "id": 95, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "fabricpool_cloud_bin_op_latency_average{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\", metric=\"PUT\"}\nand\ntopk($TopResources, avg_over_time(fabricpool_cloud_bin_op_latency_average{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\", metric=\"PUT\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}} - {{cloud_target}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Object Storage PUT Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total PUT request count, per volume, in the collection interval to object storage. Not scaled per second, but a cumulative count (delta) since the previous poll.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Request Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 512 + }, + "id": 96, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "fabricpool_cloud_bin_operation{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\", metric=\"PUT\"}\nand\ntopk($TopResources, avg_over_time(fabricpool_cloud_bin_operation{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\", metric=\"PUT\"}[3h] @ end()))", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}} - {{cloud_target}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Object Storage PUT Request Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Average request count, per volume, in the chosen time window, to object storage. Not scaled per second, but a cumulative count (delta) since the previous poll.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "volume" + }, + "properties": [ + { + "id": "displayName", + "value": "Volume" + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "/d/cdot-volume/ontap-volume?orgId=1&${Datacenter:queryparam}&${Cluster:queryparam}&${SVM:queryparam}&${__url_time_range}&var-Volume=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 524 + }, + "id": 90, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "sum by (volume) (fabricpool_cloud_bin_operation{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}) \n and \ntopk($TopResources, sum by (volume) (avg_over_time(fabricpool_cloud_bin_operation{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[3h] @ end())))", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Object Storage Requests", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Avg", + "svm": "SVM", + "type": "{{svm}} / {{volume}}" + } + } + }, + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Avg", + "volume" + ] + } + } + } + ], + "type": "table" + } + ], + "title": "Object Storage", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 99, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Footprint of blocks written to the volume in bytes for the performance tier (bin 0).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 501 + }, + "id": 119, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_performance_tier_footprint{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_performance_tier_footprint{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}} ", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Performance Tier Footprint", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Footprint of blocks written to the volume in performance tier (bin 0) as a percentage of aggregate size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 501 + }, + "id": 120, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_performance_tier_footprint_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_performance_tier_footprint_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}} ", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Performance Tier Footprint %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Footprint of blocks written to the volume in bytes for capacity tier (bin 1).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 513 + }, + "id": 121, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_capacity_tier_footprint{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_capacity_tier_footprint{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}} ", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Capacity Tier Footprint", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Footprint of blocks written to the volume in capacity tier (bin 1) as a percentage of aggregate size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 513 + }, + "id": 122, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_capacity_tier_footprint_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_capacity_tier_footprint_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\", volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}} ", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Capacity Tier Footprint %", + "type": "timeseries" + } + ], + "title": "FabricPool", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 98, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of user-visible files (inodes) used. This field is valid only when the volume is online.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 304 + }, + "id": 100, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_inode_files_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_inode_files_used{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "hide": false, + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Inode Files Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total user-visible file (inode) count, i.e., current maximum number of user-visible files (inodes) that this volume can currently hold.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 304 + }, + "id": 102, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_inode_files_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_inode_files_total{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Inode Files Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "volume_inode_files_used / volume_inode_total.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 312 + }, + "id": 101, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": false, + "expr": "volume_inode_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}\nand on (datacenter, cluster, svm, volume)\ntopk($TopResources, avg_over_time(volume_inode_used_percent{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",style!=\"flexgroup_constituent\"}[3h] @ end()) * on (datacenter, cluster, svm, volume) group_left(node) volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",tags=~\".*$Tag.*\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Inode Files Used Percentage", + "type": "timeseries" + } + ], + "title": "Inode", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 2, + "includeAll": false, + "label": "Data Source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": [ + "DC-01" + ], + "value": [ + "DC-01" + ] + }, + "datasource": "prometheus", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\"}, datacenter)", + "includeAll": false, + "multi": true, + "name": "Datacenter", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\"}, datacenter)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "includeAll": true, + "multi": true, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "prometheus", + "definition": "label_values(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)", + "includeAll": true, + "multi": true, + "name": "SVM", + "options": [], + "query": { + "query": "label_values(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, svm)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "prometheus", + "definition": "label_values(volume_tags{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"},tag)", + "includeAll": true, + "multi": true, + "name": "Tag", + "options": [], + "query": { + "query": "label_values(volume_tags{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"},tag)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "text": [ + "vol1" + ], + "value": [ + "vol1" + ] + }, + "datasource": "prometheus", + "definition": "label_values(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",style!=\"flexgroup_constituent\",tags=~\".*$Tag.*\"}, volume)", + "includeAll": true, + "multi": true, + "name": "Volume", + "options": [], + "query": { + "query": "label_values(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",style!=\"flexgroup_constituent\",tags=~\".*$Tag.*\"}, volume)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "5", + "value": "5" + }, + "includeAll": false, + "name": "TopResources", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "2", + "value": "2" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": false, + "text": "4", + "value": "4" + }, + { + "selected": true, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "6", + "value": "6" + }, + { + "selected": false, + "text": "8", + "value": "8" + }, + { + "selected": false, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "15", + "value": "15" + }, + { + "selected": false, + "text": "25", + "value": "25" + }, + { + "selected": false, + "text": "50", + "value": "50" + }, + { + "selected": false, + "text": "100", + "value": "100" + }, + { + "selected": false, + "text": "250", + "value": "250" + }, + { + "selected": false, + "text": "500", + "value": "500" + } + ], + "query": "1,2,3,4,5,6,8,10,15,25,50,100,250,500", + "type": "custom" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "FSxN: Volume", + "uid": "cdot-volume", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/dashboards.yaml b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/dashboards.yaml new file mode 100644 index 00000000..2307509e --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/dashboards/dashboards.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /etc/grafana/provisioning/dashboards \ No newline at end of file diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/fsx_dashboards.zip b/Monitoring/monitor_fsxn_with_harvest_on_ec2/fsx_dashboards.zip new file mode 100644 index 0000000000000000000000000000000000000000..a9df99f63535a86a2c87798563e7abc27ed8c6ea GIT binary patch literal 49461 zcmagE1B@ua*0wvgy=V5=wr$(CZQHhO+qP}b?6Ga%Ip@2{&6oVSe?LhfonDo$>h87P z)g>nh3<3r4pMyidP~*Ri|F?h!_zPg9?`URVtM6dsNUN*_2>=Y=F>GqnGi>VY1`PlR zatRCo@ZUo2*9gk*Wf1>0!bgc-_uB-3WdQ&n{NF}I6y0QW1yyyGoy@JwJ@lQ-ZEa{Q z9Bpm>|BPR0TH9{2A${lQ;zNWAMoN3QY*}UFTnn;v1AumzyDYlcufoD;U`_^+D@ZET zZx6g|+KP3{CtWtRo78qT^Ae$jV#4LRmlnr@^CKY51e0Vy)eM~Ui~b$p7fBwee0ioI z6QwrJP%P1PAcBL>oKG_=5TYS0b5BC)o|#JHJeVs!VPZKHO8B!!R3(lT%4x@791@wr z&UWC4`4Y7tcI((5#N=hnKpYDe+J9#k<|`uu7r19A>KB)|6yds(hw#bFW{B79aAHn? zJokN!Azl9GFC194bg_`=17nZKLKGb+bpc5@|NGapKCg@g5310}mz-60OZ|^6+{Bqa z3`GJHBPYYdm5xAI`EwqOsgiKe7j#(Hi1axw=&EDUChVVPqX$+2I_XAAolAaMuF08l$ zwNt26EYlqMJ@}}99yvM!(#^uepMNms5x{9Fys#)CoTCL#VcwzaZN`Wj6X7X9x0BMg z2a44?1cLtT^?CSh?W;XCq@$3ZvL8}gQfE)#tEebU9AB1nI`g^5Er>yZ zmes0(GU5XtqI^2I+)<>-gIQEF75Sq9)*$&S0P0W4ApU8f;1ajvbf8qr8Q2AOs?U?7 zI}*p+YyF#4bImH)1_9oU;(#bbAnhnaI7vQN_Cg9<#|TBmgYG4g$U;RwB{tJ^kIhVR z;LmCHsWb!{5#+yOhe96^Ae7H&xp==orm9h-En+0R%x=eI9`*yP@W}4NJRNp#*j^{X z3gJo8O`xbpE$A4g8YL9BZ(#y7-h4|c9nNLQw0D%{FB(Ivu5;n6e|zflZ#VYcxRwz~ zzPqJ=n$S=#jJ%#LMuVnX(S#X&lCm*x zqeL?}exJc~3O*K9J<-g+W{FgcK-i{mRecSR+1a^WQHVR$&!9m~nPo}6%z(s%F?LmD zR6;x@f6Y6*UdOicC!u@*p^#H044=aCBxL_U@=^#w2-43LfWbvn%|_)QtVeXasm8YD z8VNY#y5REI)!$yN;0A@yslX+miE>_qph<<(AuVl*!D+HlnG-5#q#7Lin8loF2PdLR z8+D_9!+rBExvmV_DGlf@ABKMDTrxiz_M<4NO3TxHlV3`CWUM15t0l&KdTgtW4Ru54 zdk&#a$@HfCi&3f_&li?8Ht>wYZHf2EUX#OOfal$k zG%)4Z6Q3yMH00fWaw9XE)+#fWi0kYbCj_7|)gIC4qw2zP`4}yjnt1uv)2zzssG~)Q zy@csvL3_iLY;kw&#Nr{8H;+X*$4A^6Y!~zg-PFfykuU8Ve30#C#Iy9MYD1z55xxK7 znjn7!b!jR>{dnVL=Bju(?ON2oNDfoj9PWd}ewXMvbI3cf%@*lJdY?N}$W`yiHM+B7 z$YZBRa?3Sbw@s32V(Cm{FqMiT@Skm3aZZeEj`t71v;n>&6lA@FZ{l=;HG~gqGq;nb zo~xs6@HhGU@CU`x4a&{cT)*lH$>RhD?Q){7=xjtoIoAeCx;Hf`Mml+OTT z$buZnZ-6p*W}9^vWxa4lXC{p3!o7b?9RB@x5Hcc#;^}}S9W&*~kJKW(C^B1n zj*E>JA>vszRn>+n0(W?evmZl@%L|q>D?Y1gUM!s4fXL_+3X3~mPkG9C+C_1s&VF0^ z=Gr)}BC?!EF%TM6`h{k*IbYg0P1nG_kBV?iGGD;3KdQE*QyULadu4;&Y;$ZoxMr8| zOgN`}fe}*e{wEjT+ncG+nw*ns*-YPcypB8Z(7p8!u#2uTb($i^Xgu%4O<$#4FKhlY zgINJx)@Tql5$9f>U8s3;G}Ich)CF;LCyQWM$?ET0?cTnSv4-Tg;f~gUX9o z$m(1@J3ckIoA!xhO`E(|Zj}TR#^er@!6e2?ajc%Zm4;x2ib59eU$&q7?y7Ww7nR z<=hQDlH|PF_NEhd0@Xl9u;SJsTxi(kMV?M3WKiJsLVy>LJ(1XQ> z>f?!0c!yXH`*gIo&_RGtAyj2d3-zNdVprE!s93Be{_Gs~ws^LD*tw{Vz#J$6KL?JY zgbcn*Crgb^0`tf_!n!<>yz_)$jhA^;f`)n@B+fpjOPqlvptkK;QGR$9!KaDir-Uam zda?^NtVX`rqPy6e-M5U!;>8AXKgFY@XuS@T*@T@UOBveWQ;y|=Lii#yyUf{h2DxP? zsyW?yR`>HbsvcaZ#xfiw!+}b8xerAJ@VT5QZ7H3@5SM%rW3PRGG3g?^q>M5ZK?uc+ z4GtA2x8r?@9{-Bt5RZL_pBqV)P5v!LkiKSf+Gng>*vdE6ylK}}5;9c6NmY%j^G%a& zm+@qkL8xd?t&pxk2U-03Dx@mbnibF{4)(zu6F1$zqlXrudW=NoXF)wAy9aR{mN%f^ zeY_0EsM^n8wA(Vp*#o}cf2w30Af1W?XYeBA0WNNIkyM=RGl_W$=mhpGE2|p@V{!uO z%|LaI&eEh{H<)vzv>@_a#$@&vNg+P7w)E&?{Y00AL-MQo2UCHF~RdB^Vjoq zB*+spW7mojqr1QaFSxtav#Owx%flIMOXvWN1i9pptq%KSJ1E1&j?mNs6;{eE1(sPt zXwN}f-(cDO=*&YxD#~h1t5W1TSiLCZbm2P#LFu6qljuYiccpr9EC0eRp@zq}7NTmY@t`*b z>dkLxsImN@Pas*O$biL=;g^BlQkjNk5RgAA$o`XC=5|N|SS~<~g%HIB7HvnxD{spD zP-Mk}Q5IoCKdUy?G^!Ml$AQpikuH^Y2enzGG#vsN_w0QgntD-jNYLxT#(=cT|$@1(%{unYNRIpxmA!Upqh~!GEM4JUOKr1)@y{L z21<3_oDr;O&N=p%z9~0(vRi>TWf~%#>|_~pBwpP9R**hfRsw)%TKq(A_+AoQv5;#d zO}^L!qYehVRBI+`;*-9+28J~FU9*MP{#3d{OQNttf@Qom&>JVrxm$wxxQbPrbJ%KW zZgM?C9#w+j(hH+=g5qrpn#plNTe?>|%g(GbH|=D!QmD1lv#wK>p@g7OW1f6DVr4yC zUOl>{fT*$+y^=Z}zTT20r01^5l2A_S(C%fB{%kjn+K-S<-F?$&5)|@9%Jbig9^X!c zN!r51`x)-ZNMh>QtYaZEFCn9j(C%z>=apk>$4Wca?fCFYVyxG_zp%qCXVFBo4xFcf z(hCqD*URzOX#XkW8z-Rj=0-@Y0Jqg0*OZ~f0}W10Z5w!=O5E1YNYwQ;$`TG%S4#nC zgUI5<7%RUn9;QN^tkH6>&&v+FK!m39N}rl#P+K2JLwYUXOl6pc3}O?WS-Bj(8Tm2- zKXd>Z2=yYJ@LU3sP9@T&f6Czgp?@AIKQy_w3RY7ae5ksKY4>s5ky9gLBvN%B?#%*A zIx!}!aiXM})F=!;OaKN5y*Mo55MCHR0wb*AZ>Io&e*4~N3vD>bg%m%6A4T!KiYg-f za!jWW_?!zMAszUGS(Fi1c##!;Hj58^5ufmut_dlG91wWCzE@%}N^bF>N4D(12!y9z z8`Am{3!xt)A65V>h(#Qh_3uX8Wdqk4u~V4!{hEnxy%(|PFlvZ4@HOJGzO$=$l5wX6 zS+H7p$dN$ZLXi6ahz62Yg;a~;wbDoa9~y4qwj_CcEe-895H(@cM??e?W=+qpx!!a} zi;8>YA>U(#NRwM8G5dJE?Z~}QD5HJ`DrarNrDPHPdLbe($-K$Pm)-+Kc(!N+W>}`ZH3xwts?pIw)PnTp{z=6hK8rc9w&LE`4jloNM`X%?q&=*q#sOYO<)VxC4H@^N zzR-ZAU=1Rfh_ES(Ii!lA#uP3Hv@cG)AI4hTpH%-;kk}w|1U>nZ-LU)Pd@3frkroIG zMbc|$TnJ{%Ea`%#hRNdZ3#!o^rBLl@Oc? z&Z)jvv7A&#|q#RNDyGnC2fm%B=#7)HDBV9Ac7< zl#0iN8a^r3S=Cy;sgg6)UZA_nMNJ@k8B8+na-~v5evXy2N~%UY(#tKnAh>HaB%K$-vJmpM{i|zg zN+#MZgpsGOlR=K9w}ApLzc=LmW#*P;v{^z;MX&In<}gUJx0B0K9LnG#bnxxf6eF}) zt$rri-FYBqidL4hKBNf~^w5heOcIT<(AOtzb3#9ioy`xakWXg}91RXnwle?9*=&q> z4`b4^<&9m<-AKw_R|D9$oKC=OqTcV3?mf$k(rZ>(8^*@(QCj>58lC87xGJOVK<=;Yd>VoBIlAFgVv zNIzklcNhzta>s#;k)+s9MzYD=G(TUs_iTEAisXR#=_RR;_O9NP8SnX0EHo<09%Z23 z3qlo&hETI)PJ_r|i)R~=p=hSVEI&)PD3fK(cd4DP(o|W`FbRG>T570;z^Fapus{@6 zX-2t4gB~wrzgjDlL*3{$U-p|6zkg(PL~A?W0LB$sFMy`B)uQk?2^BK7a}^LuUBj5D%#4pY;)QgkRTp&JYUtasf{=SN#WykM^{A2X)Crwk zTKRA~xVZ0CF|(o-Dit@0HtYidR1WaZ{Y6o*2<1t*SDBzVWTeD3Y6{qL71x3LB~xTZw#rpv+X>n(>PDQEPMUZGV@8zN?*?-^K4B zK>-4=e>y=2l3VH~90XDyF^qu`V(9^#}mgunQba7<8Bz1ClH7a z1cRs-1|SH6LC}ZLU+@Gb2*MmZ8A>lCC)= z-m)=L_pB*5kgEdjtz}_;*~mm)xnsA4L zj=whs{pFutV85}Lo4KY%|M&|5UX zJ!6m9LmS&5VVo#l?y?o3KUdD_5OTI#fg`A_(+u#)q%Gn_vEKVLDa$JSO6}YkyU=sk zCF1(8jd*W=fOu|#k7&N<#vYodwl;0ld~Sh{DibWoD@dHNKHxv;7F)=25fwC`LNd-{ z2G6VWBu6#RA7+bzLF-lswWgw$4?G)((F1oE$>CQC_0}F+rX;S^xiD>_c~bYAlFFl;Z=rf!v-qn1bKJB7DscUip4CMgid9SWvnS z;xZ9l8R&;KA)1>qZ?Y9pvwQ|1ZXoy3wAfk7XK}=-?J&Wuv4*q?n`sWjM(XJVwOn-e zv-xA&8ms~8DO|Ga{D>8?kx+Gm0{HcjVZ@N&i{K&l;rWSMh5>*9Ms-+$&_x2#&5~Av z>m_yiX#%9-BJ>$4LFlny^$o>0fJnGSX>iLwsXq&%#S8ttf3F$)H_-^pLJIf5p;jrP z?1&N!pnTW9aKp59OrDyw-7s5gbL+nhBI_f?zk19!@jnVI&?^YGC!S0M+_y_|(mBpZTb*?87?^WyO)!2L09>6kK1eECB)lmBe700@svST1AP>VVH2 zD8eYK5d6UHjlTJDRU^<<>0xC)f zqKyH8t|!YlMiYw}{rmJFbe|;sH2sQY%wC*X2P(^}uhLKgi}m;8hPDe4>B)k4TciZx zCc!M!7w88LC>wtk2$<8nKD7?x7DI27QYg^Z*x^_Dik9G#32!=#FkGEZKbgXbU@EH7 zoXK3NM_Hleyht*>15@D6I%W`D>ZTrsQqn;Kol@~WO^gKt-T0My5exI1z5^Ofg7Y`r zg9M3z_>1Mn{r(ki2}ZeUCTIe!7f$;%C68aHrLFe|IKJK5OiSUfb^X)!BU#2;6??06 zhCN8-GLQ5fDFrY}274CTwCKD^{k4e!XrRT=2oqLn=)49TA$_k)AV2LRH(45^p2i!$ zPuVf6S4RGc@Pi8Cow$)4BgLYy95=y(p}fYN$en@%;Rf)y%;8=3{TfpihzA5SiRt`f zJgME88iTLwC(P&@MFz{JOLH^kU7lw1c<} zi9h*}c7}9xDy_?N2Waz8)pal$yah)o9q&aD(CZr52N6G9fvZPCb1!P_Y0hMghn@s@ zzAx5Q~1bZtruWz@k zacAPP`w8-smA>-jXKYnKNh(A2mhH-f+u2P8)03osq=tC;ftQ5Hi_MUs^TCU^jT`?0@v-fgS_iIIm`;*sMo(9muF)I+kKe`wGTfGIfSpRtt zuNWt{6EW2<4@Q==D=xI?I8(#YFp*Ow*;20ui>>;c?K_|~?!`vW766y+?hOCBR)`Sh?Nx1|pQ}~6 zEnSeoLd|VEnP9n*mDhcB^%>s!KBSlF}1*uE(;ae@eR)DdaYv=F!b-MXBR`?y44}`J3nm?B14V`pL)_A>6Xe zl5n!ujzK4_OT?{ubdbc!9ucFix(L7WZYwjP>;H(p;cgGq26hYVki8i4{uE|z;VRo) zt0MAGa@~~={w%LQoy8KhL>*)h>r*W5_s6`_%JfKKzr>qcYSFs@j@Y9V0*|=rT5CH(v84f71u(9sQDB9tsJT5mY< zcp>WCzJm0PF18jUE(w+MZ%eg?fBr)h-REd$~cC*r3=#kNKgTS0HA zxKC9cdw=Hr0XM|HX0z!7o^<}4aeRiY?N{Es2Y!8*eM%NyJ?j46Y6x6~+q%9krh#|w z^LD+46-tnC`^`7*4L6PHi0IkC-hHnf5MJNPINY#g2im#qjsyYO(k!r&tjM{pW46se zuexwcZDPH=&FtZDp1bV%l9GB>b!O5CkM1d|k2MzYuf1GOq3QfH1p&T}7M3e3vX-Xq z+TiAzQn8r`>_41p_xB{U`5^FQ{by+P>igYdYR2HZ;M&g0#2lf1tK$6%`y8@Cn%ZL(}F8s0_W+AwviFTYHf-ju&5S_%lc1x{LDE9+aL(TC?kZ3Tf_}oZ0;p zpUi|@Hz)dX+^_4^53lmM!L|M&SrFfL1$*`)ouV8d6OW`Y zw1{-_iX=LC;PR1l_iH5(WcrszV15RaBBBML)^!~Ql<&SqlQnhY)ovslTnLy6vl$*JWwKoD_Gd4O&#x^6riLJHuRRVz%PR6$xv1AIxX*p+^?p;v$$4i( z<+E_o&u~ll=H8LrPKVd?+a7phXrUhIqBz_z^_smzUg?T^(Ll3DEKdQp4yWQZ)YcO< zAXA%7j(E*0cuGcR?(3-L=J7)hCkS8ov}$neVv1Qq!<^yB-XmJn$>nrThs&GK-k+Is zOZm2gfCPb2R0(Z~B+bq4Bkarl*oQL%R^DmdsTU`AL{HCFBExmy#xvI~^%%4*+e(oXvfW?cyDESol;+GXoY&CfW>Rk z7J3gnQ!kR_T$q+_#y_DqG;eM{;AGd=H6^-gY4A>Pg01z#YHWk5HQ**K{6olvs|v6) z_lAz|mOB8kR(#5ODUHRZcRd@8a3Urp_gqE9%Aa1rvcOcvXdeZHWOvl^kMxW$= z8t+WoVVk}?WUr0Bv*Vu7SL3R98@`dkhu$gg_(0iy9`(5JeilpKbEigxe@L2W?M!xz zCJFAVgL@QA*sUyJxmMBhWCU}gwPN`*E>{~>ZP&&cf_;Hw-Jwz-0zdP_y+*7xBiiU5 zLn2@D+}(t^KX@5OVbe5}!5019_gyhA0#0AtAQ|k1?5;d?*ZQ($^jWaSlGOC1hamq`t|*ReF%gT6^SsPhSzZPO2!F_pe_Fr>j#KhL7#ZSTpIpI{vp&8) zfyNl56pQ$#uU+y=PnrMw%@Vq={%SBys+cxlolc~_7 zWo<~A)8P`?0ZeQnM0p8xt@-=b^iuk1iQR>%!IO4H&s)*$B3!TTk#o@rII}5C{x@+) z=xOR8_86bkF~B5JAt-mwv03CTsG=)s;ni!;TI3(jPhu>OEE* zdQ>v-dcqvHm|tip@bCWqqQ$b|vNcFOckh9r7NK$@O zol!pUc}BAtnj)6b897+uuBhY&{nG@|aV%Ol4lt2W@|D@_|%#0vd<=em~1qbK=5hsX8fzJ-ovWj1K=V{wI zsa-d)-ShMX`o7(>3>Qe~fI*~iNM}|vnD^(yGPUuu^H{=~0lNhVijo!eLZ9wEEv5nG z=p~Xl$4{IWk^wMj;E)Rd&6)l@p)%G7r8g|$w*+fXK+nMbB7;$)Lji2P1o!srMd6BH zD&mIImgPi(2N_7d>rRnc6dUX+5r&xq!c z^yB#~#CRUWtA3wp!eN-d^gLBsWX%_qN(uv)RHo_j&U!JX}o#5_-o){>R~YLWL5rT5x_eqZX; zniPH^PbpK;DMl=jUn>Wj9V)5|TiStkoN0_2HOf_tK*BG09;{K%? z^;K%WzIJxL)V#_|%lYjQr#K+w%s1aU&{OGIqoV?cVm|I|YEZu7i~}7Io{7rg1WNp= z_Xn5~vIiOMde0;_Ir@!OmHC*q2=E(bEO2_h$eWBhZqeR&WKVJ&zSMIPgPnlZGrHe! z>cnud?M;sm?A^Pkk^C_0A73@CA zSLP000h)H6Ft3!2m&HOTwv_d1|Ji5@&~qZLS{%nZW61rAwW3z8IZ;OUBKoDxaWDs} zA-PW9Gw_ea#+Z)XuXjA_UGjAk$Uhz|4P9|J zJGNeMY4$9Phx!>;YO-E(f65YQR@J?(dCNO!eh=^f+FZg;pjL~%M53LHXpP<`aM#ga z6{9KY2-WFK9N8j)g=a-#QRvpim~4sez~(8Js~@y=2J`Oj2#xM!(Vo?5`vChfeVf({ z-cG!)cvEU0C04J*{w{5gET695-Xd|+@)`z5>}^&2xB}2h{1*P_qbQCYJpGg%e*yMx zv-_vbk@WRcCsQ|$BDSC!$lcT9Xw+@md57>;k@n8m)&ki6hu~Gtr2LG^!Sv@Xx`wL* z5s@`H!&BR2ihOsf&QGD`ia2135(vHtu)A&j)28!+8Iv>OB`0%7wf=9ECT`kefkmXJ z6Q_@QV_T$MKHJ6#s;5k5H9}0=^_v402B>lEoy2f93JVwF7M%~PR5rA&omCRY#|rKq zrC`xa(To^oX6=mG+~dGkLeX;3@+r;+Q?<^mw94+ir{I>=>Vv%_vN^Xo_tu9wbdwLl zbt=&?>2NIS&oUaBk44)L@)P(uyz$rGNy`hklhLzPGqL){Ptj}9tKCg%wcMwAn`0JJ zE)(y<=PLE;C!oz0_*ZLn9_c{!SJ69G`Fmv9lkPPVEqO$Y5j)j}SbXAR+d-uY#A*>G zI(c$JB^gWYVZ?pAKCP(xT5Jr*?{6lg1%IJvO)0fD_U)!b!KZ{jFMRPHJ%gs-L(IL& z3CTSRLd}biAy~xH4L=#2z*W3q;K+qZUX?>mja6i*?W2H36d*cEkX=N|A^OsQv^v8@F@ zE>du_C@~ifhM-N(sKq7auf#-n7`QQ2XoQr~hEAMJ$wjthj50IhVuC02ayun%?6o!#j&mlmrs{m zh_&zfdT5@M){hwbew7ckQ8*>e=ng>NS|%2wllz<~*itHGfEpkBpzJC!NI6udpJ)l0 zx(1Op6J$zN`=3-8(s+R}5zMx}K3y(jfu-SpVqvV8P}%)CWc8aFyiLM>-Mw-Ar`whD zH}uZ*xD8OzNT2;cjqH;y?xpoNAZLD{ zSKUa}!{znXNNDCzBDx?fW>q(wpUSKvo?dHF6+z$X#ETV$!{)W-@UZxLve-ORl=N+1 zb@Ey6-srY>S#_@>tGCQOOKMb@-sCGuOwe@0*ZeE$lC zzGt{F8k^Ljf^$!ITqdMtbWdlUnEV8|^Q_^fL_p~y^=x#8vvd$gHy$IFM)V*Ph!syG zcvMObCU`UwCKrhPM5T5@^_7CwZkjWu`fYD3pP1)cdLw$s;HDZ?ue3Mt7CG0bC9Sku zR6B~VN?{2;GUXXG?JIZn>Qn0UO3V_2p5-goe&SYp1gFncpjUgC_u4#Gq(UtOih#g; zov~6fRH{DefEF54Xq0I5u{ZTkDxnmQsuA6jCi1dV3w|5jlZq-F%GyXJ};#R;>NUrCU~B5?{u*%tYMn z`Q5DsK5okz6!*;E`CQnP%J&$X&3Yr`bxebNOCO)D`qEgP{p&4tm>91PYjerKxjtm> zt_NNbIbRr|37_r2lKW-E{6%K1gswxIRPrYO$UTo%j3yfHy_;?? zy4HTO`nU8Qc!)T`w5|o8YCdX{0Ai@ne>&eT@PqTLSL3_=0?<#Zs7#nuqTmqPd^CKI zH|RuV!oYe*WIky~^(z5bqKCh8KIlVXA6SN05&w$i!9gI`)3ogv0Se9gQQ}zD5<^Po z&Y$5g+{=Ry zxnD0L(*#YRR~?a>g6pQ@i*u~j6NiFUYBG#C$E%0`sqnkQ3M5)%wCNwnKl)dogYf#G zgh~|R)7(v_Y5``-xGGNJ^Qu`ciB0Tii*$m`Hh`8m*=8A-Jhn+i>(kuM_Mzp5MMv$X z>ieg*&JXz9KBy>h%IWEmb!0`=4n%R(b$8+rx7q#5Q#86OeQcG5iL;ZcqfS#WxMs9yjG#q{}2u^ zZ!8QNMiWjGJy2OHc{LfweA@8*N8vx`VDl!yg1QRTe1=jd_a+y_yfh)xXtrL`Hm(AY zk?Bp&N$5%FO`c;h7!!HuOQbR|5-}d|F%*BmgqV$79C?W*h$f8Mdrjm^m^e4Z(qYRm zOo;b z)pCO2)omq6AV?srAAkE@qp|I7ZL3U>pY+<;`}-Gouk`C`R9m2s@BfE<2z3$f*B%!0 zDJ>@(!WZIK{g!UyX>33(;!ktT&;FcwHBjGxiM#;6Kz{&AzFtjGP{4nPG?F00(58Zn z(w+|3rs>y{h|sIMqbqBXZGiz>)glk*7r6wt28YSn{fCqn<-T$5omV+TJ5XV@G9PrM zObWFcj(W}anB(1yO~)P!p0lEQ*h#Y@$Yi2-hk1l~I4P5yTQoV%SJ-2MrxGA}r!` zNB9W3;>CQNr!LM@JWC26rezF$+t~X=;$TA8;It8nDgp7b(gNF-?}Eny05rB#L$(h5 zY{>PK%lb^~DPu-Xe*8t$=g!PGe5X7U51q`7`K<-DdRFnYa7$m=`Ki zh;JY`U%D~{nNyLWmCz!`2JW^}IU9joM|y%Pb$B{k0>Znv>PL{dM$e!6Jy}p<;6pmQ z%-Qc-XszfWhU9Yehxrah4R{Hm)Ay2?MVJ0E)ZFjynQO`Xr3Lg&h)Age#=bC6eWsd| z*y)7?kyyW83s_qFi^B{|=>U-KEs=^LnK_>0Dc6|!=X;_@UlG0Mo?lP9pLZrsn!T?> zZ;s1f3SAa3lUU<*EVW$hY<&tY_TpZ%&NEto#R{xXVGJxlEkF%w9l^Mbo=+PC16|wX zf~WlO?0~8nSypijEak%z!uH0?MXf2H$`05@HH1yiBhO@A+gQE;`sTy%GVi*F7mW4Q zX(dx5rb$yCGkMHel!C}JY1Wo&ruy!-W5`*;cZTiu-EFrVwTrHezG-;LyVO-$g00}wpX zOeQG~Rh}-3|6v*_LU(PDcOY|;a2$bJh(Bj`zik0J9kpyR_rHly{vN7(<9J$13jDW* zu~gxnDq#jk`Zv2@{?swT!tnP6pjLRYl*#<6T?a91>^ zCSfo2b$a7qnrdIoq0(UC&wQ6`GdDGVfp1Pu_Nb5-*gLiyO4?1N?3j7cZ5{qc``~d_ zr`mtNPoGcizr-O!MZ-!x@6fG=+Y@`9Z2@32EPR`=>qUxJcNI>YGrskE;m!5q8Mm3y z39c4)7mVyl8UNrT2xo=Fbu1##U5%Khl+m3sC@_D1a)@@4qx9#Y8^_5&n!Cyi?|N+> zuGfnXXcoJZwPJO==xe^$h3U!`N+2}TkgnjqZE`rbj=5;~p%h-lPA9XoBy z*C~fJ*WN7e&&S2#BS5B;(|_<{xU5DvL9q3{gQd?tYI%gx{(59kDe8D`3I6qbx!JJ> zAXlH?+eho*DH}5|g!D1-Elzj!xqx#ZnIQdBzCL!h#l1ISezPVYz^)#4EL{zKE5MzF zu8@kzNpL9huz42XY;TV=c_Mko-O#QmT7u2+MmpUu=zF22-AXOpqTNKBIa4il0Lb%q zNuIiLi4!Zj7D%>SBC@7@D_f9jmZUr2u@4M1D#2K%Kj%7iX76FUxnF;DHyLvP{IEHh zC7E2!CnjiK{m!TIS|OYJf;6L#)&bV;_{!x{z%MoYC0MFt!NkcK_Mp=mW>pqqGi zl!ddS^0*aJ{SWex;WaI(HP=rXi1n&GiXOXU?G`$t9xBs1>v!2}k!!y%73PqRLw%V= z>|4f4or*%0H33b#Pw+hAhOnl4pMs={D@hfvsA_1rVhdy^#4iO%NG^!K82>E!w}f{sX{dh8MatfJ^JtwX)QV3OSA%4%K`f*?=nnvQP0E1u?q$OCK;|FYiEmi}7v;zek zS!3W;)oE>lK)eMaZ%RRqrTN<}|CDQz9c#{OEJxo)TMiXODyw#tNXm|&jKTV1YN)jGd zd$b2I`zVE4ut?AYqYOt8ewH`814l&W1kUsGf&KXYdU3R|b%>#!s2=7QsWhf~sbkS0 zhDyUfgGdt0Q&|h~S>)sa=_7qn3@;gyjFl9-Df>}cn(9%e1+n7f2}pb-*-J-Y*!eC< z-m)CAnCZB-Yls%^i;jDQOEQ(m71#P#dKpe3nA!Ovre*VgEXvRF$# zxGIYQdL%dKZf_&M{#~54=)tK`r?M&IRAS&b>DI z*^$yVh2Q>7)8$XLdmgO;0EO4WK%%21N7x$_eNrVK7<>aAY z>Tt7NS~Z*U6NuGwdnBb}iB96Ap7DomL&kB!8%2Iip=i2nCc{oq9*G957zLnj7!3v{ zH){3Q^`HmgM~k7JT{jzpWTaA)TA+D#{<_Cq>2G*l5-g%vsaZ6JeR)< zF{j_mT>3<@g%JTw&EP=$7nKOhH`@_w%A(5)P<9QdZ5ME9*H}+OWA1m$ ziva|`q3958>Bvo?NKjgQO>lVM2IO|W68@h@)Tdw|p}f&PnVHqVi6To&O-0Hz!&PKU zy7K6h27ImDUm=i)Xow~$79}2Kt~Uj{b}4ZCC$a{I2O9mN{{+%HSX>;@3(3m@f~Fyu zHJ(S>c$en03q~{F`SXu|%GL}*bEgL~K%P{7))qE>R10=7hK*4tp9$yO;1C|dAc)%H z<(UAB4%8aK=TMUpkXQ%}9%)+J-#iBVDLX~5xVv60S|Xj#olqOcXcx*7-WZR!shC)&;i&IXkbS4XdAt$uA16Xmn!SAqtKndk=sbw>OR$0>(TKG|NZ9{ z1_gOL9O^~F8e9q5zkWCZ|MCo{H44}j96ct7&l!>_S+57)o73%q+!F|4M65eZoIvnPgoGH8@Gr^JNn#Wo zW?S3pfkh34Oz|l3D3f1^;!pYkfZ1z82@ELSonq%&%mS5S6h&opa>~9unB&Qee|%%3 z`P2tM*|1o7;q=77HDy}Dn7+JP7}Kth3=w}3CMFso7-678yYihfmAFOKX1ejMDydcE zU?e3LeiVKz)`ZEhCU8TQ?AD%o+6Ufrj~{?D54&u^=-*1eZG*Z#o2AD%l>>9J$50Kh zq5wm_R{M71MP60HzGQ5AU9%sYPP!4YW67+Ke1jqELn=o6<#$L4LJ3AW%zicTlucm6 zcVH=5iH4NpKI90;gaiUKLbOG>|A(=+jL9r$+JtYM!QI{6-QC^Y8Jxi#2G_ye-Q68- z+}+(7+#Lp%=gpUVd6LcU=E|wlf4aN6yOKVqy6S3q%QbtFIPAzCNE%a{!3NkRnCg8d zY;P96S<-(BX^GzTu~OCkQVXbXz{mJ-gweTm;D_K z5kW|7q~2%Qm-n)Y1|$Ml-(f-JKqn)+EU_KtQ zQQK8@Z|DoBF;^k0O{XuSZ08((9&m8$)v3g7lWB?T(d&}~s-g2`8a4@n=dX*ZewLw{ zlqI1tqcJBziUTo5hQgu;sZ`fmZ8VQ6>(F(ERMvcxR44BB5LaprJ8k`3I<`6$31_wm zy51j-|IOIal+?HvYMd-?8BVYM@eTkxQAZ<#CjCZ~j1-c@H&UjOi0HoUMcDJMD6%g{ zNp1MMGc4w`tEy|-4g6mu{#LL(=I2@f^sBV)Xy{^fwBwDdX3~$e0S~|%E3zoZ-gX>3 z|2Yf}_>7aYoy{mv2}aB#`6lukG&IC}=-*K1Q0=cgDR9g*VmHX1HMXLr?x*dig)jDx z@m&+^m{0cZOlbiIzBV8!i4!JPCEDJeoKj6tWh%%p$gP5T8|wdg;J;Gi@?7&djSd|F1)M4v4cHlq@$m`N`?gP+`wU3L>sItuRYUa=Lml#pM_l zSK{4yV{PHleQ}fF&k+oZuSz%UL^3eG%q=21Uwq3CSa7pvK}jl(waW7aGp|)pXc$@?r|++DCe* zyMFitysViPdNMtW)&BZ@3ZG?^>L2INlte;AmlF3~`YylOfFWVjE|ZI_>!&A>kZdB6 zG;KLX1vva`CMA&D>S09e@$ew*jB#$TKW*zQyJa4RV?G(?2F!b>$$`Yt0UUXn{$OqB0vV!Lw!ubB9OsvxoEW4o495 zh0zv?jkNY_T%cjJ45TZjmmeL(!DXxYS{Dp57fjeB;w&sequ26G=}tkH=`%k=p5`7F~KV38}MS~Y0QT_ zigLu6h-wW2>APq&!9&6CRk0Sr?<7W>VIx1jEcn(ZH2{oo&V3`nI>9<_KS$KKLO#%`4U{)!#{s-yBoE^0c=d*K(if87 zz@(7Ba1R5I_@|;d6=h8sMxIXlz)SofD~+plPz=~t@{{dWL%^Y(CYNLqk(ctN@(EgY) zIhjBqYl5l@gv4Tan}I`2^oA!Gx{=_C3P^ma5dt5V#aSZ@oNRDD^^cHz((UJ}J~*5% z&bqDFx|PiV5F;6LGki5E|3^2B8GQS)=e3 z-%fOBMPrf7&uA|cdaa(s4Z?bvoEKjPpUrzN_dtGW#&Y=Bww_QrX!rm!RDKQkc_v6v zzQmM$rIP)qfOEKKEUiCnU7Weob2{B9G?Xo@Xl_7vY&efLws6^yoVJF|_8DMJ^G2`~ z;Q)nS06L`siGhrIx<`aZIeV8`XW0w98O6Vnah_SR5$Mt)=!|)Y4Gc?|tySDUnMyiZ z;sk_vnLk_d;N)9p(_y`HX9#mJ--r~jD@YbcDO$=tsql0y^;Y{q<1+k9CQ^u8zs?RM z8J#i7AG^)7lDA?un8DLUIJ3vRl2?mdXG@!h`7ee6|)sAnDpjW?gHtNEoA|C;Pd z2OQ3~Mpp$|9Z0iEoi&YXru9!Bi3F{PANa2R3|&5JZh9Bx0AoFLTV{GjYJW8&f`74X zL43jJ4-@HL=&Zq>>yh(#>HXkqJ`nq($2se6$PW7oIRJU$A`mCIcpE2}yWEFEhTM%? z#>A9=!Wf8Xp7Oev1NVDO@t#`8nRgENpG{rOFZVTzurW(Rulru5Vb^=1>CY7qAXUq@ z{H7hZACTPj+|f%|N~`jbQ0LRHrxPf2QJDfQnJ{i&*o0pKafPN}mjF7fEWM3xxEGHw z)Si(mEMWuYAHmS0G3wDDun`fr4D*U6|6rkM8k7$@9t@f%3mIUE7fy9egl{Hdx~MkE z{5qbNp_SfWd+6Y6&tyS^gWCpMubXmCS}i%Y4?<=xAP;^c?|T(vVw{$M#kh zl@-7A6*@cc0Z$UOv#m471Ok2nHlK?IR3#S2*>u$(mB?lc`Xu(gP0cW+G^= zsgK;<$iOJ9(@w{r2whx9Sx!A&1C3gy@VGYmJQ^3h`1Y_)H=#Xpezmx8Mn$D*GmSy( zybHlN4Jg7ksBCt%mAQ^>qjWKHyp-6_Ut+(a^Gm<__mL=!0Rysu3DWw9>rzPCw3wq@ z4lKZ3d0-X-!iv29-|Y>^ewS7PSL`=FoCtYHdSGH!ReNBrS-@+f=prnzS0#Wu zeeYu6?I&U_mj(|D3AoiS$Q#nwCcGxRzUxiy*yHB|Hz4niH-H@=tcWo z8Gl!!!LR6RiwrJaP6-b}V7?W^cBJAOj$u{^0#8!OncVHPgawsq}{+S$npE@JP zB4W8AApV_8KGJ34XUU`Y80W=!BL0ne4gb7$Gtf}lq110AY;C>g8m+X>@B91v zT?&Sp3Q~R;B5bEad$jiDbaH%(X&-vb=|qqL4}ViF(+lhF2U#aSj`kZsn>liHRF8a?$!H(XQi{XA(wYm z?NRjpCBpMnMwAMJNt892Bi0(HyFoH?2?|7=zH#;QK$C}{hw$X>PaKBaqcEywF*O!0 z^1l;SP$ds>Xw_RK(G}kZm#iDhaLV|+B1CFG-SQ>on*0kD22Z=XPJCZpJ}(thdp{mp zqyC7oX#bpSj^V4D(rZ*(aBoZ<*|#y6I*`o7IWO##hNYX`Y?m&+zZ)tXurERVDESS; z?q})2m2Ne@NCh0jNYh3}+FMYTTzlo+gI;hRbVYG(OlhdO@?$#S(hb zVo@=NOoTB{1I^bJXOMOwQg9`P7!~l>q9^EZI=#Ih@gIx!9#GtDoW*#noLsyyLJ{Yy z&={O0-dv*od!4TL;5BB)Dm*_FI1qQSm&)M zjX*;yRKp%woA+vzKhA8Y&^7SAvCYgQPL@)Z^;Jz*7_sLpPRDREon+?@;LW-I)3 zr_f;&qjiV3uBo>lF)}QCIufCjMBaXF%_=*K=yaBid7l(d4jgW?7)|^Ugx6cLSag$`wh+OQ-ZZWJo9py_?)6#FU zocEV6OwPX&-kg#?=keVhmi~dYYLm%AOWQ_A)n4Z`HU(hPvmk`p-<;6+9?%##a%o`J zguH-aB&`32!QT1&{omP0ncWGq@!u?qGv5!+e`h28|2&Nhp2l{z|0g;{@|bRf0V25I zXTFeDIrPi%O>*brD+OB=GE5|{_`9d0k&P|iJA8_-BRwBD41V~+;2;Sq4-09RQfaL1L&_cBiHfIr|wk60r#zz#|Ds)#Ss z4JxKV5ETRe`;Y!p?_Vkj9>X#LykMMS*rEcFMPi|t0>EF%Lnfq%${1_cd5B0Z)tJ=3 zAjIpGhf#Iul81-|62t_^Eg;kcsuZ8lz-KjC9Fg7x(uQa0;oZ?ZuQr$%XbSjK{vP2o zoZv^1Kc~BRf+$bfmv984jmpL<-hsAfhSaFQbfY7su}1g*5Hn316z7f81q~1(OT(m$ zfISdClpoV0N7VmLiU_EGoSLn0H1oKTmOSR75V9=$-+=lHoR8f;{-8^rYl*&@dyNX3~sNR?%|PR?;{ zUQ|i3fD&W0OB1aik$CO^F<1S}O-{8kgZpP71nd`R*|y6>Gb&TI7YJwIkWM>2Z&wKN zvKLi3(YxZ9uZefQ1V?%324~~S;7er~p~Autvud4Hg%LS*#qcj>*PMOVK~;CX1<(8- zxxS=-q@bXF1!?x6L$jS>_AO{JB+y{`Hi6^wrD&o>18V5BFoMhHBgcoKm{0ntTX1iF z9rrW@00dX?pxFoNi zlC$!VH_UBDrTBvIYtVzRzT_iqsrr@-@+kBAV4(71gp}!5Ko}sjMj9s335uZ328*&LR zz+exaxT@RDejn_onbv)>J#_b75Lyu0iqK&mgoWE8_4-hvzy)d5tEFnsgeS?1Y@)I16iq3pt`efPJY2EG`_|})vi{KlbfWCvX*YS=C3}@Z7W zSR5AQh61S!a^GV7k7S4Phbl9DM#u&u2{fcB94SovQ~D!(smgUDgsn=Yvmw8#p!lV}6Xlr&)c5?TeQv^N%f;EX_va;L z0&Z>UFy$#LC5Q`|4kbs11>seaWSA_zfQYffkE=C3YAKbpTQ)d^w-H!>DzVK#TH};i zOva6M7+ z*E3lawJ$^n-%8W&KNFQ5D>s)|&xMYcQQ3)C^CXPT-G1CGogLKNPR=#g89=F?CTdYZ zo7DYyiAdS5RP*d=@BD;x^yr1Ks4-)9NUl-^<{*LnpXTL;z3#n~)KD zwM;nw&T_bS{O-x2+-AOHjMxTj2+}4UAx2zPta@X0uT@7ekav7OFX_X1_u-4D>?GCx^x?Jh5~Eb?tIA{6NhgmT05H~G-9o1JilgNra5m@{Q@2;IMSCA6T*1m zM2lK*Y=i#kd!#~CVg7MzZ1XOPs2q`A>!;EqLxC_PGWvuCXYef+c|NT^iPGMJoH9jl zS%5Yb416g0GPqq?58OFo#eiCdx=M+EL-eE(eNwd!L%+foSo<=%*1c=&g<7CcY;2>n zGX`^_pj+E=kaFEdRLZ??eatjP(_hIys!Ksea6ys$TXLyXICY7nfRTyLdDz4-JucXT};}(E7BWZ(*F7hUEx;7QUAlG zN!3xdhP&ab;A*;rJib?AM@Y?I{)HI^$(}9n& z=qi__OPfS4cBpq)U^GNP-|910$MpV5q`7qE@OCmHPqx<2cZs83#7d3T-ch(L+cba) zeQ-QjXvwNOPz(Ho?$^nrbaZm2c4IqLce}eS2jn1jM8xN_sevvq;dOocOy|~LJd0OVm)lvo@iI+{ z*ciwUD%u*DZ+bHo9D|fd8((Mm8|1A)>9j7)of=yZHO@}23$hnF<%aS9Wt)SlqBlwwZhyL`r zZjz#jqPCHx>|)d(hHSRE_6hg5Wl~yD56Dq+Ag(madSOXk^WpU8swKs-S3vLaw2zw@ zmGAJ>8*Ytz64SaTGsR)!f5iF4GoNLyp{w}DC$9ZvpdcM!BB#0Bz6e-{jZVMpnFVh0 z;>;KkV@$3S(Mv?fii8xrUfPIH#z&>_^^M{o^55CYW>JSUc&hc}q_Uk8%MBSZ%j9c& zWm>xQDGPCqwF{J?5=>7eCr+>AAd9!;{w-vX%3R8Q7ItyYR(deQUZ8KVl>eqnd>bkH z)OJi6QztiCQ@I)o3DX$I7XwYx{%A;|4Zm}aXgx`IrQ0FW9+Ji4%WyVa+M zA(O|!qf$tw;F>*kWBTUh_vsRvC6>AwBoYn+huEJg1=S>Ys0<4ng4>6L!J)Fu%L@qP z1%>!wtILhX$E;Mxq@E!xT7&B54_UfQL_1iWdCUlb^poF7xZbkSs`1Nn*GnW**A@9T%rsW~1Ru zj5wOLC2m3WR7K@cDp8NP)GI+>dUfBH-c;6+^2$a$Akz!FzwCH>e_m-v5gFRq3N6ss zkTxbUj3_2CIZD(QTZ%|-NO&tac?$kT|JtK_UrDoyr&xKQh^=%(nvrHFVBRF^iGqtI zw?L&w&vc5wFoQIaEyZcb3|;qO=x0ZIl?mt+#?SO7o>mrSuj^BVV$nn5D-E1lS-Tw$ zBVbH4h$-YB8)#0G4^33j5)N&>vd&yM<#x&K#0B}2Uvh$#9_$;Wj4kH#*guOXviCIY z$TH~Hw$)AJlSdj^JM%Z&4l#W4|@^p9{0DtveKKjAj2DQ1$E=nT_ZG!SRkUmu&7 ziXCKPA*~x2i~nd3l&-1qr!26lyR{#dFS_&_n6lKUo4Xz4&sKIWmbiv#*t>5 zhDtbE44J}n6r6x~1~_0rcr!4gK#)iwc&j4wtCNheZhX$}Wo`B@nBXZlFC;4OlPS6$ zZ9b!0hWSdRboo~eE~A5)lL!M*jeFY8dVQadmu9YizrH?Kf{F`J!i)&mp6#7TQr7KI zW5k|pD;sxD4blpYG;#a|g3_fPNB`SzSa5}@_*ABkfJC$oRu$^zDQR;!fiUp$i1b}e z#TVH+kmw==cKXSwKKVviw3*FQP7(v}u$dVS8@ss2X^2X3^wY=$?#3;tH3O=aLJkcN zf0v-s5@d!C*Sz%Y+aP%kPJU$esFJ4W!FxYiOW{#y8>=pTRNi1TYk=;|EO!r3hLm8c ztva8&IE%XeE7^g%OWasq34$p#%JD?=5>I(-R)}5DT3b%?xCLod=v*UIb3OJ^eh{lm z&7lSwTOAoG-UBCA`D@Nyo**J}i`cmFYeYu2v7yH;l^TT(A zbrs24xQY2(%(jx48$)Sx zUf+NVXWCnHNAm06k3|~kyZ-|FZ%4~d4`m8DLI8lN1QdYvzj3t4sVn^7iEZ@Hoi?~q zK6};Q5wPrkX?n@l({sCJEk0e5DOm8ROJBrno@S@JloO#OaFsxZVA5VyJ%97sJdI&s zZhv!V7r>I^tCOdUitfAlJNF<$0at#1QU=3abg%&f3!9_OqpGOD?j4^RfyTfc@28Tq$ zdZ}(i61th|>6gqmimu3A%SLuCqr+b2q@f-=^q+Rtw4q#oj~@3s+^U(~iR%!bct7+L z8|?lC2fK)tR-!2~RkZVK_I65O+(d&qJ`C}mxr(kkNc`9W#Ci^W{d=vNaxryoM3;r>xa>J1>Z%^e4>Yc*YPP|^} zP5hgy#rwQ;JwYuh;GZTzwzkU^hOT52{0zq{3ewCF!agH2%d`H%5BfV~Z@<$uU6G%h zPfd%5()vFuMw%?+7DunOmAAtg@h4U2TcVIx1IkDbz#JWf=pJZlHvdpy39C>`JXzQC zH^c_#kPaR}%n7k54#Ra0Dr*!-<3r8(m?TH?KG~jUym`vAQ4Ga!TVQ%AHfZ*cP2uYA zc6cV7$d{md6m+XKB~9sRgarFxjjByg7OJAiNuB(K3Ur|wl`?}Mxa%Gy={a3v-PRUg z=_Vy0Kz&GI&#>7h>=lQvHjUuVNVCke;Y2CA5-j;A*eWXW4@a|bB7UUk_}BPNYp4YR7yCcJ0$v(?=6~N zL~qGEWEI~{BnOPs1uuCt=&o;FtUPS1zJaY>&71O~vEPh;S^aFDZ>dDP=f(S0h1*c# zB(i?zrT1&<+(vsfPfl8Ugj_|RCP{a#dkBAfXeRRdXzN4z)}+2PDypm1ofZXypXrMx zXc~q=#7T1Ty;CYAY_$vDo6SX5o(Nu=E{s1=7Uc{2ACO>+lEqj^mSwRqf_AD=4~n7;s6l~A36@JbASD%A6X7zvuvxk zi}4f^WtvjE$L5Vd%`0* zIVkRYUZZD7xryok(&{-)1Bz{q0(Ql=J+){EzO!~R6rbe9m^tFn_2V0R-wz7%k^ zng7P%UtNvppibXyCBCl?d!}kxA=F|YlISdBRpxEX!L`&gx)B6^DN(?DhKYFGe@dR3 zkbWIJ;NivjF!zCgK#?U|9{W%7^Aha_EP4_3-g>P=VIKbMhwQyI!+_lNCByI>jlPNl?Ilfk89zVXA_yR2G>2=E*)qqLi&@LTGW7}$WC=3Y_z6wdh=A8v54dQ+!F3LTgyg<=V;Y4ri}Ris zs>?pswyzsCiciJ{jj9?NH%fR^@&w7m`#n`76rT9HMxcoI#5hXN5N>cd=h>x0)Y1N$ z3^K;0tA~AK_E!VaffH4H4{6kGZjpok3^lPMC01flbkz~b9L%8!5Z)0^9+H+&Gb@h5QDyE z{7gC0+S8W!q_TXHEZ>q;wl8JO&gX;Mupzi3$Ty7STz&>LzN z$X=C4`Wcea&m1;I?3;w!X(trK)9i^E*NwKQ7&~-}5j2T=wW{hDWw~<*%}0)Yttdh| z(;+C+Jg^cRimMAui)}KfY!?rgJ|faiI&Ur3nXC!i&(q!|TX6sl%qM@r?!n#-EaSN#AG3l~=TID&8@+54;;3OEQUD(R=F)*m1C7ZIm;+0EO$^jBqz ztl6?m$XD6KI`LVTZK>4dQS1?W?x{y%50UO}Tt^UUid7xQ0|caCK52LiP>1Ra;$nlQ z(_igL;$L9%@G_71XaC)zH}!{iO-GqN|8}L67K`ZXZVSzEwr8y%s>1Id&o(**z7k20 zX9UxDv$OC5v!4Atj8fXN%bN>z`+iz(5a@a!r45)y-M=YTQKV_!!uC`dJv?{Nl$>KK z16jrpn7wp+{blfJWP7TYHct|xklg5>TIHiA=W>x9t0iT|{gUc#opOoM%V#L-L9@48 z(jIDcil--o7$*qeotrxt)is=QYlny)RSw`!pk~VY2Nk4sFvG8H)cYu~<25Yx1Jw$f zNE+m$I=#%nKZV2Ay*E$iw+<}z&MQW5<=o#3Z_Ly<(U)7zykRjT2f}K&(?tXI-(!bx z<9(2$UswpLm*lZI)kB<XGLbnjjguB%Ygb%Mo{AQy)3IEY}V5^}kyT(4ZE%tJkpgxMa|a zhWt89>a|||1J{IixuAOqCxSxyawDK+)ug;+yWKalz)nRkHUd`U@G~&a&h5!%3U8~osTqb_E^Z!f?VaztI#`^ zZMT{;Ggp+N>hZiipb>H!hs$QSDxi@ewV6Nb@!f%kjKGkGcDbg=M%BL`5{CD+DYM_e zL*DULUdV`ZY|YOZOUMW;z}J!o5u5`Qw3E)!10Mw1_VOP)^6DA?>>2jO@rWBBC>t37 z+4&==JAX?m17$UHe46-v(2N=8!DTBA*LWirONj=eXiW) z2*#5=A?MDz%w7yR_xx6%UG=iccDo%!;(CJHjDlwJl?WQ=0DM3oJ&G0M5uEL0u%Z2U zFzj_n3kkhRh6If^{wHYTZ8O+(JYiBJG7~YyV}Z_@I7{4DP{l?VA(As48Q~p;APAH zeuIM;nMVwqLn8r7x_gxXTCGjj6o|%W=%t#pEhk-NQo5Ep&jk+5sD|_OgP0CCgPAuO zEYvxFKOUIHV#y2O3r8~kWc#GFn8dK~iriLY*CUktB*mB=7md+#zKP?){eiY2;iKjA z&dvc=5reEv6?$Pl1$U)T87O-UChgg(fltwrx5gG?mEMp9itsG0%>@r<-2lT*J-u|1 zodMV83#qRiQiE0+j5Ea4IDUFB`p%%<*F6zNTg%<^IGm#aRWuwB=M~ZCKcb2NnJlsq zRj9u67lX@BRCINRY@VY+K_1UeFI8VY*tUd%UAe`B?>M>eM~FP_Yn3p(O`sj?Qe^~Y zimR?33)+^;5n6Uhuw%2}!d4Gvc6lXr4_y0FFcKwltNxIfZlTn!`cRQ#L@7@+!-jFH zk25R=V>slpqm+6jRD}jU8FM^4XsRn{?HLAR*%d=0U;a4R2}#pIZMo01PhMu%t=z!r z37?mI6{YmTZ5#>I3>OuE7Vdx_WtHw+j)|&sB3K&RV|9 z2=UmNwIj*->p|vD){8hIqF8yT_Z~aH_ZnJ5+b{A*y8TDi$wX_j!-Uj+C8YCTgN4lv zU2NC;z9t?*6$*Fs79uW-wQ+53Y0()ddkAOGVb~_u5Qq=4;OZEu6CcB(J1D(3hf?Mb za1#Ufer8^9$fA`ACZ@H{>;onEqvmxq>0C_YH899AH7JLlH#UkZDiOUp%GX`JN@w6t zRXQirpQ2ODcd^5Bw-K0Ki1?B-F(lf`zuF=v%YqIBh8azc8rU6W*}{gqnSW-1Tsa$R zUIMXt#&`y4HxT)}u_UN9YPZ817V37h%;YUC2gu#eAUv`VT@TLYl*=X;n19)6GBb#Q z@`1Wg{D5;EkZn^7dibNNZFOGK}PqCNHs}tYT=(CtYU0fZ)&X;_WD<=922s!vCo?(RN%;$ z;IC#^mC~o>8(DoAj)GYVyNf7}!+?Iue$f0zTb6m7j`iUpRT3@vm5ac6`#9aLHrZ6# zz0#a*7+>?ijKu-LQ7B@xs8la6$PCK$xyjqxTno$V^TOEmSPR5=bao0aBm#cn0}4Ie65Z6A~2yzDMo(c3Qjz71m8$(`4Hm$3;Rf{7RN*;lp zVU`2LC27QW5j1<@0{|if<&Qw)qXX^&?zg)8gCvSLEBg3KH-Cx+1hzUr5R~c!5q*vS zFwwq_@n>jDORoTc7$5=&RsjKsxRS5pAvJ_sDb|+-Iw=Cuilkj^n5^~=I5>zf!$vi?Qz=Jvlze}jK|I=fz83*1FmG#vJsAfZ5EQKx|YN%96| zLNO}pXaYc3p@hV>ofP;jBicf6(%>-3sv|h;?wWTh}h| zT`aS}Vc)TFDgTBOaUski<>fmefWYH3;RNosW%tYQqT#IoXpY=P!OFmuexM6f2g%sM z$8K0&J-Lxc1?Gi-9q!jwOx=2z_yl#?>__mcy*?#j*JIcvri!s*wvO>C?kpQ+b{!j8 z>!(z@;0*?biSd=Q>M7#lG9*^CR+?dq;)i+Ajc`ESBNU40&eJemYX+`Gk`lJ<@@_Qm z?14)yVj(3qh!rb$q^}Az)Cpp%TxgGy=89&vyPa^JZSkP#j`Zq1#(zgzAAvxatR6N- zp8Qlm7BwfSL-8i_n->8&3S-D4dfxwgvJ@?;8nYe9(*)rmIbGt`3Pnl#*4 zDa;{nWzR8#p~GozX0?;lZ&vuXd92)8-B2ijy^T0gDNk%mRL}eCP*|+stdIf2qcm6yu3Gc)-smkt%nDGV>L$pjb!!q05}^cMDfnq$l#JH8KofQrstPTUe0={9NCpEn~j zh~_X`T4*#K=}R|{qxV?II0Kn(1{Dis)c_W68&MgN9Yg(OPtW6CS(XtRBgPlrlRlj` z5V!^Cc1+AJ^CV*=)GtmJ>7DiCbB+Dn-&+&o4ZY7v!S2?THrS&gBSx@24w$@`s`u#g zM3q?I3lsgyNx!4M(E-|k)FvMyc~pZPH~Y7%P{M#r&52h(8+waua?fgDQ|DE$%j)`; zd;(3vfo_lngRWJa23!h9BZ`xJ5SxbbWM6|d^fI>1Okd|kJtkj0z+gN7uLO1?syZx@ z=ro0b=LQrvK@7ky8n9doVOQtu9RfH?0z?vT{)M@dtjF z4yavr1Cli)sa3*)u%?|?Obaw7uFfpO09Z%3gD!%WB`1QEH47@qR(kn20j%h=sEDdh z3IxzPF|7F~VqpNy0PFmIsJs-Z%;;X;s3x{oLAtz0*vC2E4TvoA?H^pQ4jKxO4$Ggr zbZPSyQm`c8>=B?jkql?8e7-tirPKF?Y`fGz)o&H}%+to4J0nW}(*hs5I>9dWtpZTh zztoAvzZ9S-{y)+xfe(wXcw4QRK#?DUFIfg82b8b4pH%dxQJTsHUC#XBSKZB`to9H8 z(m`GjMUb5FF+k2bAs>XJ|63Cgg2S8`h=n*NZHrkogw71W(!2;@2{Nmq@GDSE+&rIT62ItywBJk$OR3EOf0-^+06L%|3l2a7P>Y{52BS&q zrJ;b4qU|V)My-<6QM`wNkb;fF_-?=oyF_$gYe;*1lMrjX#Q@pzS!m~xSL*Rj*ish=uBtlViZVT*1mzlB z4kC?)5-bgdNuBIJGMQp5s84NN2)NM5BlXB8$Ok9_l?s6&^j8b@{yTd+y-#g^g=p(_ z_HU%s<7ng1FZ{7Du(HQtQl_Bc-T zsFTjsz=0?{nfhTmJCSp5P_Pn=?t${9@NPI*R$M~}CdA+fb0F?bboj@kOVI&MS~8rw z2;p9zeh`o*lrkyH_M<^RI9`qO5c!H~2N_Mt3`*b~)d(tGD-Z^d3Sa=y8|DUg@C5G8 zla%le6~K7GiYi_e-ZK4f*GLRIi^esBi?s)wVYtc)B5MgELU+>rECMvN zU^d7Yy6m#5azUr(3Wj^_>k#dj_`5V}>r!rRBJ_MDFJZr}Y~^NlPJ__yn0vU0fI;SJ zePVmT z6cV>TC32G8fGOd~o%#o+@eeETg0kO4QPNr2M~5S$4Fxdq6d!x7m99;$H3(x7V+Z4| zQbz3D@q-2btRWs^7Bi1g$j}i1#24fZ^dUJC1?gS%PPuo%Dru&ip5kwW;}QUbIu3R|!f?O0;pp(Yqi z(Hef|U0l{)i(?T-s(}K~08{{_Z}o&g{6PWV>iMr92W!;ohUMl6a~Vs(+^BlrCZcxW z>fA0GOGnU7F6pBQ+tH`OGoi8eihQx7cZ0@}`_=i_f!o(R!B(LkRJt!B&e|NufsTVF z6~}=mk%NPngM61Es9=>abnwi}OUuuUmxri?)Ya9E)(6sH82W!1ASz~ws@wVh`$`?W3^6h40kjwM$ z4RO_thcZREGo2nTcmOX#LDkm}MZSm@vrDKnbS7G(Wv(#q813c?s5o>4kXdq=Fn}0l zGpk3lg}NayH1c~5QZ6M@@cJEFox0!u{=HrI+W(;n2o|;3=Mq-F}6%JM*@%l!~o&nQ%VFX1n!@v z=wbtKwCrTGxEu%{586LtCIR>SwqVj0TxDpV(RGpD^s30VuV&LIef02mx_e2KeRx4>d&;dm(bJZYHnJq{HZc#yw zq919BAUCADl4>ta&@Rv)qh5qS$)*E(>b--f45#P0gUm{s--3>X;-I#?c$_`lJ^N50 zs)__hG2$mTJ|>x{JTUn)Ir9ARWw1F8v_ zBk(U-BBN@N-{W+W$bJk9*GVBXSiSk%foy=b(EmlMyWBI;6{-2-MJ zMdwG(l`wbxzTBgJ1@h%;;>lJSEj?R*jvzrj56t4RczmB8K*?*Tyxh| zfC4dZPbK_EJlg`aD(Zy>1QLZ;nws*N7)wvV@U&ur3vD@|I z62AL4H^pVgvD&_b6Ep%pS<+E))Jmal1X&iQVYNtrhXn;AyiqbEZigDx!7PGR9LJagFKJ>lp8TH-rWrm@NYNH$1K+WHH7E(WzhQXA1f{C+$9+dJD*A`Gm9X#;1YysoFJukcx|%t_hwoQ=DId_(nsT&C;#jUG+M z^i4*lQ-%L1C$Ft87<)GQ*NaII1Rwt;Q^6GwH>*6d%NV*Z8li*$zfrCoCSgNwP)j^Swvy`ntd zJ`SWU7lVW^dn*neYGDGzW9~}x#Pid{^J?ODRbQu48>t$xsNykxZ3gZkvs{Hom+{t$ z{_OKaUPDo*?<*=9mgDrEiwa1ys!01HkI!rVS8eYcUCG<5fyUNE6Wg|JOl&(7+sR~N zOpG1dwylY6V`AI7dw$zPj6na zqxwmOxK;5LLyzVJKj#+M*J$q8aXsL z2{Eh*sv&5ODPZwc{TA%jH2teCs!)KGB1qU+aS4^IA!;fJs%MwM!H6@#=jqRx27f!P zreg0R+t$EttC%}1ZXQm(n73!%PZjv+`l%}QgLQBFxR_AyzMa&Ix zY2;tx55|z{2a&$MyGKiHg~)kRzG3WGNxtmGls6dwVqXXANz>4YQrpZ`=e68JH@m94 z%zl1bHoW*xe8Nl1e!W@@a9M*Ii7YWrvDfU?k#z??PW9=zOx%81y}fnal)Iy=D<)q@ z_>J_nx8Cql=kZi*h3jPl2B|fRFxe7AEte$%L6+fS3DN$ys*OO3S~J84i~jJs4idqx}Yi`22JIOOG9obK(P<8r6zxc5m{O$aWJ zl_reeA36a(JZCG@=D1t@{k3neMhwa^d>zSs=c$u6 zj2)x~zw_>hyHZaJ7WbU7J&J}dX12#uPP3TjHla8*j$pwVJE+00Ep9x+479kN)iVeU zk;9rTYv$=q7&^mMyakq$tC(f=E}~2(1=j33D3R+O1p zAlc2#_A*^@9B8h++Lyu8WF5Q7;oNAwHZ#oKxFx^&W4dahbw{8jt9F`ChxA?!acSZ1 zIRm3Mc~8{`rH0;ZZ!)VIbFkaO)MeV^lC2VVg84%eyXvb?HUSAhGsNAVcdMbaQ?r83 zd33VCvta6iQliUxE@evI_H?x)ZoN9yv9kIgEBzzP9f6~vP$1VNdO~8r-~ANpqLM%R zx4p@W`i=eZ7N-QZ1WDkz}5>dLfcDH&!5Vq z;ie^q{b{(IQ9Gd(5pDh}6UX07bz48_^ea^*W`x6}V7;_TS*MW~H?O>BH3W~4A4YHG z&eF2#Gpg;rokr}meC%Q)Jr2EwhAU;3!;f@++q!doh|i{7GOsf}Juf<2ROWx$(4;$T zOK?=s3Ma>Px@%9zf3#yJR(BBb&3>YIh~vl>!0mrt`)tSK^Tz$~7gt+i}s&z(d)cL-)lrO*=N4yid)yaeza)o zVn>@BM5rL`aYiX<$NtRVb+#IBD2Fz}$(!NuG5T@Ir08BiHUv;rnFF zic9#3W5Wj&p9-Ryp$$(H+8c3d+H^l%N;{>f(@xQX;nNAm> z3=J@y8T(?Y^u|IQK@_1(K3f-7eSatZ^?h(a2s&sc9eV2OyHDOd!BHF%!-mZYx2RQq z;UxAm2TyG?C_E?A6w{P(IQt}aCi{dEUTYJdA^T5FzqRoq*!PIV@y2P613V)`0ZzM55X zo4K#(IuF~mcQRN$_JMkDLUT}`xb&IXmrfy0v1TcZZ*pgX^k~Ain_Ou1BB91*S&YTZsVQqXH}T02dkK z6S!lHDe8h95Q5i`N5Y2ve7cv6prGjWFl+9@mef zycF7pXR|^{{0PYUUDQ1vxS+179F6=7k}b zZqoBTcE*F|B$$K^u}wAOY5x@3K{iu`^U-pdm)+y2km#;16mO{QDz|YG)-$ZUk8_DNK`~;9S{P zaRdU>Y{t7w8bxMSJrZ?Mk3WtIbH9`Fa@*_pty<;^k8v(Q23Eedf>wbp`JI-4Z1Y^gfh=_fpT4ZSF6?Nx zMNtq|0d}+h?v~RUv7VPm+KhLNA{f^hv{DlpbG5holJPq;!z7|q&cy?&+kdOaUJ3L{ zXA@(7$X)?X6tPW5#f;oVBR{)@XLH2eb?;WM1L_L@NnZAiEtVp!p&HCXqxF;yQ?7os z?izeIqNvitaDhIh$}uP@(BsP5W%elO(|mIsHLMN8nJKi`8HVI0uon%XUE0>txnK9I z66H-kAE5+E&W@)uqn(_FiQK$T|79ID6ZW{u5aTf#X>c z%VrvDtqn((9S{mO z9Ioz?TAsuZ7FO@+oAwL_J_K(VoM*%h3hMG>?{r6jos#LUx={(2; z8oJ>I^IiVF0M2$kev9gfw{`E4Mdobv3PZ4<>;kss0Zey$y4BWptyhq>wfU_0uUp`gjNCRLsF9oQ0&HFA& zeP3JIpzk$$xsGdA1|V_%a!W4~C48qMq9LoWp8?ZDtE&(ACPYD$Py#Ycb_pj$(Mg{0 zlDCUagl6f1N6w-&`5o9UDU~hwn@spX-It_9WoOCMg7#}86f_Do)Ol1!NGs_El=)c` zE8%Y`>dbOc%tmlRQG8U9s0&nw$Pf#DR%s&aW&`!hFjk`_E&1P6xP~Q2EsG#H*|K!8 zd@Z&q9MD=|*HsZY{Ui4PeaQ4&Dd(XNH);f5qlT(?C40rU^9F#+U`o~#-NbiF)>qpl zrTB=whjeFR%MpD@62Y>`v|$85v;7oM0e!fCPmU_;p{)KwBSHl!U?^4!)HtQ;Xx3I}6sj10LccVhohz0?7{R#+-%X%^Gwv)JwYH z)%2et@|-n-_XP0($N>PfV;BQck~R1kahr#mUKqm%S)G-Yi({+o+G#&Kz_|g(ctnZ{ zvcZZ^`x<7||78q0R}a!uZ6-u_orYV6bC*%V7&NIeur@xx1)531L;Ce>tBXn5U0Juu zG4{XSMg`E&Eg{EeGGn-ESndMw3;2oYi|loZ;V3#jEDY(ioR)R?#kHItld ziUc>UqN|jO6@inLp|}kP?htZeW(y$*Tb^cm77IVUQIlkS2Y8>8Kl`Bc!k|L3<+OXQ6y-nTon)kd&3hlf^yXNan zoF8*))VDh0v_1zxSCsCEenCcqrT|0p4-dc$2=5WYWuED4D5$10wA2Iq2q1s{6^}|! ze3h;U*Dw#K4d-%%>0i1>;S!T7Pijzu^aU-S97!hQ7liJw^SK#wS~!hs@%)s~b^Wi& zu_prHy-$>2PnYLfv7jSd|M z01=?~j%iDt${#WwEhB8sDPnFfZX~KJiJuqSa!|iL-qftGm%V;`kby+}QNbxk_0#TH zmW9bLjG7<@A+bRA+ygJBvWBP>4wT9Gvdi(cv-4z+5Y}**VyIH48`%w1SMm5P`?` zrqP07U#SElaBvY13vs1&Mmo48B12tYS%HB23HSFHzQV&|uobR^DGI-Ypp&>*uoJ@> z)7X@EdP(()xYt_*A_zg%$wFcKA&?0Y1-=XunM?>lBI<)7<|zvW4(9l`PyA@dXvke* zQtYJ&Y%x&U;qU0JR@ztk{qYg3!rv;*%2#Z%^-2lSsVlH(|(k`xhckk`a6Z-MU zDTO82HW2*zOfN)4xeg+0GnKfOvRbV|Tv13P3KPb*8Cr*PcdI`?B=xN5F|K2gP)mnT zQtMl)4Z4RDtf~Q|B?`YQ2sI$0sX(+!i-l%9JaH4}@I@5>PwvKJ^t{T}{}&^x3f9W~pK$ zC%`H%6f$Ez1u1K8ujSn#nqJN~(V2|4?IZv$=YlUS=hH-RJgTi@e(U2b1|5W#@0BLD zbv%c&sd-DLYrjyGD4xT=2bfKdmDNX0My8&6Mdwb{v@D7A2L0xj-jVsvwBVe9h_=MI zWH~B5SfB9*dfLan=k1$tk?Tvjy}sd=~k5r>EK^213oSks^{svs$>n zw_ZIB4br`NNC$(y4*%oOY5r-cr+uel$$y5|n_4{rzVwap2U4!`4RdXr>{?nY8#v*y z4(rWqowLQ=fy#tJx95$x^SSkf_Kw~=e^ZvL?q1YDn&n?+=So_eO4_ZjKU zcrUEcNEsJs3Q-a?1q7saH0<#9SH}4-hZ6VOSx$$8I}wh+Bmn&0Rwb$hn8qzR;-5nU zTFQtVA~Ub|Pa_ur4er7!;9r(xw+4?!!dKg_XZQ=J8m52cz}5U<&IFWas^o5AH*-Iv z{E&y;Qa9mn9ESW*d&RP|>dIgG$Y92RB>lu?AZ8z%tCF6;IYduAI zI+#eLK42FV@=f`5p6L>99@|GaA2gEc0#>cQXixp)LwhZpLH%{M5GZS{WYst9)eN5FMumWZhD+66c4P|*}>R2OnK=^Qdlb+h zzcilMK&;drc-bM8k+9$)z`%ZYHH4SQN0iG?D+t%p70L4&x#z~(Ao)gF{j$nsa0B0I z`AIU=^*DvXH(b*^ZRA?Ts7HEqJJVdaJ4ZxKv)salIutiw#*Z5l9oU`Uwa2>;XF$=n zz;X~ixrN6S`}>V`N?~e?^vM?96+*dJ3Qhy(P-CN;QqcL*de2`tXX^pjBPf2vr5Knt zp1y>3qmj^;uJ-3*3em)ng*O)Tfd7Y2+7QV$TA)R>on105@F)``Lu&qot+R5O%n)qyoa*>r8bbF zPpBbG58$FHZF+KSL2u1OPs<7;$bD2Ar3yB)Y{he>#x1pH#z|OYh?PT5#(kRBuR*nJ z%(@ay5k7G_9r)<1gNkrE8gR$hyG3TdON>@i$mPy^)@#2(6WNw6p(xhl%R z084u43&RM(uqnbMW`s%e8d?H$h+#Z#mKmr5AfsL+IRVq^SJM6tqTQz}XI2}~Hv^by zmX;dGS)jqwP0|na)$FpB<`NgteC6CeQ3WU0$VN;m&E?ootR}UsyTjv(t+)|muxnEL zNVSi`CHPp9($0o>vdq!_N;f4mDm2guey%~THqowUQEM2`Lw3uOzT)Aoro$6cdMTqxsiU^M)+4rN5(Lpc@v4)3pgOQRD|kBCkBcWQ zou*tXZjZFS)FZ*_nlhTB+VuMtqRE9bxlTSve}5Ngjant#aSW+CHQ2OQ+VlpP685hW zzgt)E{aJ;C)FxP}ngG_xB|bmV3B;z@(xgb9JEdPC_;^0nGa${#R4n_lbLK@8TJ9j& z(%@q4X6gBGvo=`|@9k-Rg}ZZMIZqwiRg8wiQiZw|`gcOxn9V+$>gMJy8@n45!=_zF z=oG`&7!d2dhiO^+x;J)@e7V)imN3a}srV6~7-6i@+-nOg(nkdji~Nsl$-!ptNCC=s zBtx%}z!8xVyIm0t@)G8uNV*1Ac}xCGd9W1P!9GWp5JOkc;?-<3uB~w)z!E0<2ZjD4 z_EQHh#Z!HheelDFJ8@bnRFH^U1)aBY@Oh%|bnxK^os$A-@%=bb7gnWO9I0zlZIa2O zWh)z!L%T{%2QxqNCiEVUv;D+Eg^brr|6g8%C!PEhgF(3(<@>sQKFL zz<&xfMk4`OfB9<*SE`+yLP6IY(^oCxa2SncXd}3B=z7BYM$v@HPs;P52v{hRtTEwA z&u6xPVoOR(X(tUO8Ey=@AG%8ho&NR5A6_JcTD7evie&FtY$1?%4$G0(#U~Z27QYq~ z{j!pdEXbV+rQ=pgc$}JyENz`$F_9??@d2#hF{c)P(2>x}pl87N(oyRs&CBI|?`>$! z-(_dN6+US!8@AOy+b!Ht9P{#$<=qClv_(1=zq!vwSs%-ra$pnb+lF9K)tGTF)JlDS z&kp&{LT^Fm6tFUgaXg%k5s%r1nL%JA5+*tf}{;0{@+I}n& zyU)4-FFul@pofNG(h6)f3a=!DQ>iS5A1_eq6HV`|iY)7s+aLF^rInbhFvd=sn$}<{+j#J2?)OM2B5ptDeL05&M{$P5xbt9S z>Z61nO)w*3GXF72sWZE3SKDBpkNo&U^Pxt(2f5L+F8(WS$p#C z&e^PSJ=tU6Deb)U3;)wvz6hYri#1F)@}j|4e&6JrujAs`YVitm^qfvaav~Gn2NUC> zdXTGz?j<@RMEKxg#BWBqyHw;Oek@$!Ij7YMy!o%Q+78w6n z5KH5gfr-%Kbx7Qesfp$yG3j9T9RA}Fzzx?(I6eb+5ZmII@$PifL{D9~<6A3T=+GUD zUZqMSE7~^3In;%9AhSQ=l-=_Q=GFSsOY5%NP=DH`+^8Fmd0ABpnva=Gd%wB<@oTqG zMVM`XCUv297~mf^s8)N!;)`Mo2^4ZQ5_@CD+IAdN0@0YQO&nV1)T@`P+}cmpc;$Q>k~^ls%)S=F7; zcXij@4A2%3T{2v7;0PgZ9U2h{qC?}7i)R4u0@?h+-f_PTf9yYb z1({s;fZ^GA8JXJB_oda?&=nk{u80{VHVb69$-Q(0R;*1{E?nr(B_s?4@J}=xFs!-N zW6qu{m72bgGT0N%UNf``zG04B^bSJk3)=&|4SqGF`GjTLc5Uxbk%5kO1^NWyBQ{2u zgA6?7u2+uw3jC?|LFoGxDd{Eb&=vRxM}g?Wj49QiM5|)ZoE42C7+zntaajpKNS zZ}G)8K^*3vY^KUIM@3ygIOpXKa$2xpCsA2Lk* zIcBg~YX|7F)Oq}ZY|H$OhNt@(eKS9p8?X|p-!YnR=t?NGxBN};0F0Z19Iz6d%#4FB z)4TE&p_24sys%KGUUsZLdCM(JZ+VP0a0}aAAzy8F$xGz7DZi3~FX5K&0?--fzP_?i z$wyC~zT;cPFAxv){!|<7V#4h3D_(D`O{Xyq9PY8%DqY1sbeE`Te)o?f0^GC%Q#Y{Y zEoOVKfJMCbwv`65&qgvId|?>u&0A>{*_q3F5cpy+5+8@oLLwGikFg zq2q2%OV@@mVgqFA~h$5oGwcYV&9)H z;ynej9p4@bWTOP0PwE-4EopCy^`a9W!A({n=NRNb05Wd7b{t#EZnSB9GbGOSmpI@W z^-IS6T1js1Ref1d{73pamzs_4rN4^hI5u7qI9*BBkmkFj6CMF1GPuuvEz8j^OZ8$-^DFmA}@sNW%qp#SvC30_VxyP?4 z?RsyRXE;ibPu%hu;@ztGA;EC{3Wd96ubGl|)p~LEX{O;*H_I@5`c{E9a$KXayTY^q z3%^b~>|n6}}gyn&`|m@_J{iX`J^~&m^O|yAb(oXOGlgzN#PY{<#a$Eb_VM-IJO$ao+RTqkovIlci_!?-@Ulh@{j5KnI82}*>; zH-DcBp?|WEBE?sQ#>*pY8T7gx(5WqDcl?$d*37Wr?|YC%cA>BlOs)jZe8v4pf}&h< zd_kc7Oyo>J@c2QaZ7Gpx>!h_nx%CISVfQox@lw7)c_wzQR5R>Z2#?l~lEH(!-UacH z>EnIx&gj4Zvt`F&+5Q2<=PO`i(JH~Qs`b}D_d982@5j$A4(reHzF)$%xv zUPHC+ccVyk!REYb*qjW0pNtc^|mBn)BDR$$=djKi4)=7d&S*w z*%meO`vB-t!E;E?S&!4&99sVARLBi3Lq8fJjFG{=qwvgXDG8|u08V|to1_Lzlh_ue zCU2CkYWtU=FOK0Cd;T5~uN+pxxb^Hs3QncikPddBPdDVvX|3_ATr|#2rv*((ZU!ZI z3FvnW6v>8I;Yx*5&3p`<*_6?qF1G2xRq)J0#EV0GiGC0}5KiSNP~g~C*SI@qv3u^r zIM@?)1?P|36QvqH4vBOXqFL~5b9)fI2XtH0JBPI=E_E{Kf3cZvt&66-*mlw$Pkr>U9iu_8Q_%ZPwf`$ zx*6BE%53F3@uQt%lw%^G!Kpy9dtIPfo3+oNUUele{AKHx>KNp*f|Zv+J#l6wz$&qP zw!YU_Wf8LKs~?#$iZvd$LDiL3^^dr&gcp^Z3=$cfN~9 zf&Q@Tu2*Xdv=@Ck6LYsl^)|mGwjs24$_9V^#r8OvQdo1_vzr}?I|z~mq9C0c-!rGT z%pol6_kE~W1*Vj~cMLOVUfyF~T_~8{Ypl$~Xz_p#lEDX-AyS4N01tFG-$la9x{Gh1 zz{=W7=36TM*?UxjtwWQ>ywEn|Bx;t$IAg=J1Mo~F_*~vnAeeqU z(WlUWOoJcQ#v|ltcrfeuoo#3_ZPxWW@nBgL4c$CBWbm>&M5cn~s;Yd4dZ{t~CdQy- z&6@zbQNx*fX2l!Rd#PFjRp$OlIg_0rwOr<2mjtIAv%^m9&{i|s=udY*vTP8R&+{~= z+@7PrqvU&AS?cqBAG>wn68B88)94F%-(g$*XR_Y)h)g?C^=-X;P^xC8=^NgJ@Vw{%8{ ze)pKMFdHOfN;K|DmgOHlvm|U1M^X1zz(} z*A9v0!Uh8-vSot-9VCo`57vvis%$2Jx1}2?g2>L{Pu2zX%$2euApdR;veG$PBG6+G0Q4Mlj05 zJn4?16U_?Hi-}J`eGh{ux?UvhYX&ks5+6zxILB?Np*tuS5S~RgsOcHBqh+_GT(H+7 zwVGW`?kvn2@PyHvnAv9X#SnbkVMehh@YY}<0&8TK5I!;j{D06hz=$1(Z9DugdJgyXs_kzyi9Kz9>1-8{&Ti22CdjZGgs_$d z97Oy?D{r&jBXH42cFk(Z(9xPK$ns9Ks(ogB0u1WJEa(BbX~o4LHMj+6gA@l8aD@4_ z7>$X2U8A}z!cBm=3}!Rq+*HCKO0g}HQ3|Cupo82NOK^msN9JDHIO2%jLTr!d0Nz5} zz3z6dm~bmGh}7t_>Cu`xWXtFYUPapzUFPI#oS!yO{%(4vbeIsf(bt6w+r zvEdK4ufJ-QXwY{WbVfr1d;r4Od*z83LR~VoLiM+5B&23&EXsD=*+2YMh4SD0q8N9b zIiw`RT@u&wy1hu+b+y~r+Z*`2pKT9Uf;;(GpE?~M*|e#DI8&;2E?`(5JYJt`oPUpU zY0cc&7{n^gO7yA30#LYSw zXIvSL0G`PN4%odZ!(k4j2Ez-mTlf@%o;R?Auju;U#);UBpp3?lmX4Q;RtT1M``KMz z(O|BV3D96tgqHQ)Xh?S5DA3yW9avTt+z0}i4Aw=G+{jLYOAIrrA2yDQH+KEvFCVdS8|a0?XK& z*&7IZ{xD7E;6Bqg*Z%SqtkA7R3ED#6OfQ7$0s`MY5cotF;0#pPnu zG0?*$ZDAJdsF#jIEO`-&+qdYpc#)CH2Fnb8F5crJ&6`~+|w5PWMK@S`JG7aLw{ z<+!Ltn&n))RGejhX|rj`=k~9sebMTsvN*&FsKfb`d&JjAT(A3kl_*{1GdZ9Nk#aP_ zlpzHU@0Jp*e9Zd$r)K*-Vpjux77t?&YHSf%SQr!@!0@F6U4vaS1iS(j@^3JpkkD(V z8B615?XkPKn)vr1cIlNaq3?FM<*!MnZmMq0g{|qp&?6}w501zCFpV(!v_+bh~ zn40_@b1cp+;=W+W`qM|(DABix)Icti(8QTXz_EIRF$ASrO#`R$Ie(|})8bA`+UaTe)zUcBgn(V zZy?++!FUU%ko=hIe$kE4JmC0R(h5Fr0^66OJeq_P!NxX;aN=C&NjJGf)@F)O zTC`@6v5m@f(`OZcVcC_SjzpbF8v`PYX(~a(-7oRVaD}mnP7!w+3Z}{bg9dT}X&`N( z{Y6k&a~sLTf6_pP=D##h{~Jf^KQvHIc{mdLY?F##L#H$EPqJQI6n!psE$1J{IK5%0z)<@Oa=c7*)C;L&qP!dqVha@#l9)a5 zPyK$!JKH{ZV6>g@6w34GW?GfR$5?cb=!-ShX9)Tn|M{i^!UWmD3fVSyz9`K5L+80q z`kI%wG9@QlB&mj-r!p<8*RJ`=q?_hZM~Aj#=w7k868B`qtz}zXSL(5<)M!AikQ4j< z(N$wvQKWcj2Z@U$?4*ojpjgRh4ws0U+qZ`kcTV1)%XV5j*ab9hYuvQ~7M?$wr=0W1 z8c3eDwrolGQ$)WtWWw^8l=JLb4mNjnPMr*DbUku!Jt4&bDB47p^_DkO!>PpB8r#R` zGMb(#U%o!V7X5UaMNt9IXxC}L;7FN3uSsS}VT1|${Mnke3(bVe5nfjsP%6h0D*PSR zoHi6o?4dI_x1Bb00@1qn_Bna`$i}-}R+jymUK4*c{=C{zSaRd(J9*()^T)71z8-Of zV8zcevGtea8tMGn5Z{TSUk@oB!M{O%gkboaR!%f7YJSn&sw%30kz&=5)_@2ZY7H-! z=x=exNrE7fCz7ulkgg%?DWZ6(V?c?RSoBY`9}oIN7J31I@f(l=HNVY(P85;T~0@Or9bzRNHh0 zN6fyZtN}xQ8|AVxgVE>I?^G;f`Xu5qYZY3LfPdo`yJlV16GrewuIYQfVZY&$Y0dtc z5`fOlCx(ZP11N44V(Z3_JqB^$ZjUx@+XY86CV96c04iK>hNhLIere{hVf!^{CARDN^i6n@yf0R;w&4N~Nb^Y*?yu!yLxaOwj zrWBPW>9Q{MkZOyZx7*jCPk49Ztqg|nbrw);3IOi+cek7rPlDg66Q0%5wnE1yLLEas#$EO2 zL-UwFu*pbn;AXAoC-ofZD=dCVhdKI*Fn-^WZp>{`{2dQ4yOxrt9JB^>8RRS{8072& z%x%fJ^&xnyIrFkn$io4HMZx1XU|je>`%iJLsoEw(N^yWEY!pbcJ@Al?A2QbB2Bkhx zw;r;d>s$6Box`wzx$EFNQm0sRbBY05omGHi`BXJ`y=26ScK~mzEdMzD6oh|ZkMOn< zyzn-W!!Q2du)5zfT&Yaldpz@ID|=h*d94538aYWB`ejAC0?v~^ZXD$H1#)PMFClWk zr<=ny(Z7>yjPyJm+ca*PaDl26E)h_c76-P(NFgMFt>~i+bCMQqy^UgVWQI0}|6Xxu z%>1lfGYc|~L<|8FB_H+AHQoXd_0M%W0TV5Z@XtLlE{UY-dq#16li~iwLxCNVC;icMcyK5Dy7i3}#_TPDM{N8yP&gU>mA#SlA0(vx zn{-5g9Dfpt*iXs~qHBc{5z}P^_foQUE9}k{EygK~GnPWRId_m_g3>Uur8kbo@)DC) zM{<6(S{(D0GMnAqbzdM>zRLkL*iS^rXd3wanTs-64oNFMK=EmKFuJVS>#W9|jGKb= zt=YCH9Xrcz){J^{vkq+K43hF~ZNGjxJE`bH-!?<|jk$VoFf;_b=lQ+p zPACYm#d>PYoAO$B6O<4b5#&CIRINQoEQl1hD`@O5bvU6LWOIoPB~UUtFW5p5sZMAz zmW=pKX3jKuDG^@Y7^$xxjRA4X+`UqU1)A7Qs9532fAb4_q=Lb6N`&H_Z?LHad;)FgPH+U3kEAvCxW5jM%5L8}D5LiEST5#aiUb6rj^`L^3h81VR5fQt0D8l*8 zdkdvOiKih-DY5b*OMO&=64g3`0dJ@FpF0!f>qgPVIiTh`bvUp>|NZY#nY(U*nX6 z&d-Uo5Lo71tuESKMrwTbBSVZOON1n=WdYtDDkHcQ7#V2%ua>1!A!VO+5a2d2 z_p+O4M6Y!W^Gxtp+KQ(l&tWp0knte|@WOybr9jB^q5L3#*NNd&twLPjVXljU3L`E1 zb{7#_p8P_*FYB3iuMP*0b&op2DrNq5!LlB+Gv898&OaUr6h z2uIs-ebH(j6p7JaCbj!LLQA9VO>%P_vHhD2DNd(~va7%ERC;*8@yQP}R7e8eT7PMp zXVgILn%t}`Ee-ZM#WJpnmsS42c|=w=#FbDI>oSw_f{NPk7+(E(?RMm_UZMNL>Ds>g zZI@4-q)wQbobAcPj0<^1gJS)~=~oGGwK9ZSF9%fBE!U z?E@XXE9Kh37~FkPj(FS#bq94C;s+0!dSm-;j1bybOE!w}1D>(z|ShPDWY{@_SG8nL%)AG)Zl?b!2w zJ=B^5B=B=oQ_8xHEUraywiw@C7Tw$Dby1x8y;1=}_k`m~$lY{ESYOyLx#NbcRNjP#&(FG&m9<*3@rHH#b>8|ZVXanUUsJJQs9le;tmXRXkEd_x_%0D zAWm{$*?evjT`v@JznjC9xBKs8L!jw0lS&cP7*av+E*pp%{c(4o`(TcIXBUY4;YRz$ zjQWW+33W5}o-lbTF(CK_T@M+AmDU)s8eKNP^kjs`Ln>lH7$^Aw4 zW2W5!$#NWd9e0p3FZqd&#d0v_6IoFf6pR2I4&?8LN5n+?pN;<*;DM`B|Mfuuc^Ce> z^7}ZoELBNsQet&oMfCmAEI0FX(`Dgf-LUiE4F#jkNqWxL}+yp`700ANW4~3w> zy$$~2|E}<#&R?P$25K&rHkMvS|E=-Y|IUzt)NcSL(C&8*P!RloGn5B3|KG0sUxsX4 zon6eFod08i?f=e1*gKI7Hya3uIRyj=?!TEp#{Zut)a`9tZO#5a8Tj|^Z~uDw|9V)N zO8;iyzi#J0dcpngZs6Y=ul;xZ0m}cT|No&4+yBnezgHgr@0LzY{->q?szd(2)Bf)( mr~kY5@WB76{lC^x|KDkcgaW>cz_%0z1OoUT<97Z1^?v|F&3{G! literal 0 HcmV?d00001 From 4ea80b73f43686017646b043742e263ed046f3a1 Mon Sep 17 00:00:00 2001 From: Keith Cantrell Date: Tue, 4 Mar 2025 16:14:26 -0600 Subject: [PATCH 02/11] Initial Version --- .../README.md | 101 +++++++++++++----- .../harvest-policy.json | 39 +++++++ .../harvest.yml | 34 ++++++ .../trust-policy.json | 13 +++ .../yace-config.yaml | 25 +++++ 5 files changed, 183 insertions(+), 29 deletions(-) create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-policy.json create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest.yml create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/trust-policy.json create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/yace-config.yaml diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md index 32407423..76b733ab 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md @@ -1,6 +1,6 @@ # Deploy NetApp Harvest on EC2 -Harvest installation for monitoring Amazon FSxN using Promethues and Grafana stack, integrating AWS Secret Manager for FSxN credentials. +Harvest installation for monitoring Amazon FSxN using Prometheus and Grafana stack, integrating AWS Secret Manager for FSxN credentials. ## Introduction @@ -11,22 +11,68 @@ Harvest installation will result in the following: * Collecting metrics about your FSxNs and adding existing Grafana dashboards for better visualization. ### Prerequisites -* A FSx for ONTAP running in the same VPC. -* If not running an AWS based Linux, ensure that the `aws` command has been instealled and configured. +* A FSx for ONTAP file system running in the same VPC as the EC2 instance. +* If not running an AWS based Linux, ensure that the `aws` command has been installed and configured. ## Installation Steps ### 1. Create AWS Secret Manager with Username and Password for each FSxN +Since this solution uses an AWS Secrets Manager secret to authenticate with the FSx for ONTAP file system +you will need to create a secret for each FSxN you want to monitor. You can use the following command to create a secret: ```sh aws secretsmanager create-secret --name --secret-string '{"username":"fsxadmin","password":""}' ``` -### 2. Create Instance Profile with Permission to AWS Secret Manager and cloudwatch metrics +### 2. Create Instance Profile with Permission to AWS Secret Manager and CloudWatch metrics #### 2.1. Create Policy with Permissions to AWS Secret Manager -Edit the harvest-policy.json file found in this repo with the ARN of the AWS Secret Manager secret created above. +Edit the harvest-policy.json file found in this repo with the ARN of the AWS Secret Manager secrets created above. +If you only have one FSxN and therefore only one secret, remove the comma after the one secret ARN (i.e. the last +entry should not have a comma after it). + +``` +{ + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + "secretsmanager:ListSecrets" + ], + "Resource": [ + "", + "" + ] + }, + { + "Effect": "Allow", + "Action": [ + "tag:GetResources", + "cloudwatch:GetMetricData", + "cloudwatch:GetMetricStatistics", + "cloudwatch:ListMetrics", + "apigateway:GET", + "aps:ListWorkspaces", + "autoscaling:DescribeAutoScalingGroups", + "dms:DescribeReplicationInstances", + "dms:DescribeReplicationTasks", + "ec2:DescribeTransitGatewayAttachments", + "ec2:DescribeSpotFleetRequests", + "shield:ListProtections", + "storagegateway:ListGateways", + "storagegateway:ListTagsForResource", + "iam:ListAccountAliases" + ], + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" +} ```sh POLICY_ARN=$(aws iam create-policy --policy-name harvest-policy --policy-document file://harvest-policy.json --query Policy.Arn --output text) @@ -45,20 +91,20 @@ Note that the `trust-policy.json` file can be found in this repo. ### 3. Create EC2 Instance -We recommend using a `t2.xlarge` instance type with 20GB disk and attaching the instance profile. +We recommend using a `t2.xlarge` or larger instance type with 20GB disk. -If you already have an ec2 instance, you can use the following command to attach the instance profile: +Once you have created your ec2 instance, you can use the following command to attach the instance profile: ```sh aws ec2 associate-iam-instance-profile --instance-id --iam-instance-profile Arn=,Name=HarvestProfile ``` You should get the instance profile ARN from step 2.2 above. -If your exiting ec2 instance already had an instance profile, then simply add the policy create in step 2.2 above. +If your exiting ec2 instance already had an instance profile, then simply add the policy create in step 2.2 above to its instance profile role. ### 4. Install Docker and Docker Compose -Use the following commands if you are running an Red Hat based Linux: +To install Docker use the following commands if you are running an Red Hat based Linux: ```sh sudo yum install docker sudo curl -L https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-compose-plugin-2.6.0-3.el7.x86_64.rpm -o ./compose-plugin.rpm @@ -75,12 +121,6 @@ sudo docker run hello-world You should get output similar to the following: ``` -Unable to find image 'hello-world:latest' locally -latest: Pulling from library/hello-world -e6590344b1a5: Pull complete -Digest: sha256:bfbb0cc14f13f9ed1ae86abc2b9f11181dc50d779807ed3a3c5e55a6936dbdd5 -Status: Downloaded newer image for hello-world:latest - Hello from Docker! This message shows that your installation appears to be working correctly. @@ -104,11 +144,12 @@ For more examples and ideas, visit: ``` ### 5. Install Harvest on EC2 -To install Harvest on your EC2 instance following the following steps: +Preform the following steps to install Harvest on your EC2 instance: #### 5.1. Generate Harvest Configuration File -Create `harvest.yml` file with your cluster details, below is an example with annotated comments. Modify as needed for your scenario: +Modify the `harvest.yml` found in this repo with your clusters details. You mostly should just have to change the `` to the IP of your FSxN. +Add as many pollers as you need to monitor all your FSxNs. There should be an AWS Secrets Manager secret for each FSxN. ```yaml Exporters: @@ -162,11 +203,11 @@ docker run --rm \ --output harvest-compose.yml ``` -:warning:**NOTE** Ignore the command that it outputs used to start Harvest. +:warning: Ignore the command that it outputs that it says will start the cluster. #### 5.3. Replace Harvest images in the harvest-compose.yml: -Replace the Harvest image that supports using AWS Secret Manager for FSxN credentials: +Replace the Harvest image with one that supports using AWS Secret Manager for FSxN credentials: ```yaml sed -i 's|ghcr.io/netapp/harvest:latest|ghcr.io/tlvdevops/harvest-fsx:latest|g' harvest-compose.yml @@ -174,8 +215,10 @@ sed -i 's|ghcr.io/netapp/harvest:latest|ghcr.io/tlvdevops/harvest-fsx:latest|g' #### 5.4. Add AWS Secret Manager Names to Docker Compose Environment Variables -`SECRET_NAME` and `AWS_REGION` are required for the credentials script. +Edit the `harvest-compose.yml` file by adding the "environment" section for each FSxN with the two variables: `SECRET_NAME` and `AWS_REGION`. +These environment variables are required for the credentials script. +For example: ```yaml services: fsx01: @@ -209,33 +252,33 @@ AWS has useful metrics regarding the FSxN file system that ONTAP doesn't provide an exporter that will expose these metrics. The following steps show how to install a recommended exporter. ##### 5.6.1 Create the yace configuration file. -Use the text in the box below to create the configuration file named `yace-config.yaml`. Replace ``, in both places, with the region where your FSxN resides: - +Edit the `yace-config.yaml` file found in this repo and replace ``, in both places, with the region where your FSxN resides: ```yaml apiVersion: v1alpha1 -sts-region: +sts-region: discovery: jobs: - type: AWS/FSx - regions: [] + regions: [] period: 300 length: 300 metrics: - name: DiskReadOperations - statistics: [Average] + statistics: [Sum] - name: DiskWriteOperations - statistics: [Average] + statistics: [Sum] - name: DiskReadBytes - statistics: [Average] + statistics: [Sum] - name: DiskWriteBytes - statistics: [Average] + statistics: [Sum] - name: DiskIopsUtilization statistics: [Average] - name: NetworkThroughputUtilization statistics: [Average] - name: FileServerDiskThroughputUtilization statistics: [Average] - + - name: CPUUtilization + statistics: [Average] ``` ##### 5.6.2 Add Yet-Another-Exporter to harvest-compose.yaml diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-policy.json b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-policy.json new file mode 100644 index 00000000..69fdb0e5 --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-policy.json @@ -0,0 +1,39 @@ +{ + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + "secretsmanager:ListSecrets" + ], + "Resource": [ + "" + ] + }, + { + "Effect": "Allow", + "Action": [ + "tag:GetResources", + "cloudwatch:GetMetricData", + "cloudwatch:GetMetricStatistics", + "cloudwatch:ListMetrics", + "apigateway:GET", + "aps:ListWorkspaces", + "autoscaling:DescribeAutoScalingGroups", + "dms:DescribeReplicationInstances", + "dms:DescribeReplicationTasks", + "ec2:DescribeTransitGatewayAttachments", + "ec2:DescribeSpotFleetRequests", + "shield:ListProtections", + "storagegateway:ListGateways", + "storagegateway:ListTagsForResource", + "iam:ListAccountAliases" + ], + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" +} diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest.yml b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest.yml new file mode 100644 index 00000000..6b27879d --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest.yml @@ -0,0 +1,34 @@ +Exporters: + prometheus1: + exporter: Prometheus + port_range: 12990-14000 + add_meta_tags: false +Defaults: + use_insecure_tls: true +Pollers: + fsx01: + datacenter: fsx + addr: + collectors: + - Rest + - RestPerf + - Ems + exporters: + - prometheus1 + credentials_script: + path: /opt/fetch-credentails + schedule: 3h + timeout: 10s + fsx02: + datacenter: fsx + addr: + collectors: + - Rest + - RestPerf + - Ems + exporters: + - prometheus1 + credentials_script: + path: /opt/fetch-credentails + schedule: 3h + timeout: 10s \ No newline at end of file diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/trust-policy.json b/Monitoring/monitor_fsxn_with_harvest_on_ec2/trust-policy.json new file mode 100644 index 00000000..69fba5af --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/trust-policy.json @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + } + \ No newline at end of file diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/yace-config.yaml b/Monitoring/monitor_fsxn_with_harvest_on_ec2/yace-config.yaml new file mode 100644 index 00000000..2a4b65df --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/yace-config.yaml @@ -0,0 +1,25 @@ +apiVersion: v1alpha1 +sts-region: +discovery: + jobs: + - type: AWS/FSx + regions: [] + period: 300 + length: 300 + metrics: + - name: DiskReadOperations + statistics: [Sum] + - name: DiskWriteOperations + statistics: [Sum] + - name: DiskReadBytes + statistics: [Sum] + - name: DiskWriteBytes + statistics: [Sum] + - name: DiskIopsUtilization + statistics: [Average] + - name: NetworkThroughputUtilization + statistics: [Average] + - name: FileServerDiskThroughputUtilization + statistics: [Average] + - name: CPUUtilization + statistics: [Average] From 030013dd6e4743b91afd1c847dda6012955a8448 Mon Sep 17 00:00:00 2001 From: Keith Cantrell Date: Tue, 4 Mar 2025 16:20:49 -0600 Subject: [PATCH 03/11] Initial Version --- Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md index 76b733ab..50ab3b92 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md @@ -26,7 +26,7 @@ aws secretsmanager create-secret --name --secret-string '{"us ### 2. Create Instance Profile with Permission to AWS Secret Manager and CloudWatch metrics -#### 2.1. Create Policy with Permissions to AWS Secret Manager +#### 2.1. Create Policy Edit the harvest-policy.json file found in this repo with the ARN of the AWS Secret Manager secrets created above. If you only have one FSxN and therefore only one secret, remove the comma after the one secret ARN (i.e. the last @@ -73,13 +73,16 @@ entry should not have a comma after it). ], "Version": "2012-10-17" } +``` +Run the following command to create the policy and obtain the policy ARN: ```sh POLICY_ARN=$(aws iam create-policy --policy-name harvest-policy --policy-document file://harvest-policy.json --query Policy.Arn --output text) ``` #### 2.2. Create Instance Profile Role +Run the following commands to create the instance profile role and attach the policy to it: ```sh aws iam create-role --role-name HarvestRole --assume-role-policy-document file://trust-policy.json aws iam attach-role-policy --role-name HarvestRole --policy-arn $POLICY_ARN @@ -91,7 +94,7 @@ Note that the `trust-policy.json` file can be found in this repo. ### 3. Create EC2 Instance -We recommend using a `t2.xlarge` or larger instance type with 20GB disk. +We recommend using a `t2.xlarge` or larger instance type with at least 20GB disk. Once you have created your ec2 instance, you can use the following command to attach the instance profile: @@ -148,7 +151,7 @@ Preform the following steps to install Harvest on your EC2 instance: #### 5.1. Generate Harvest Configuration File -Modify the `harvest.yml` found in this repo with your clusters details. You mostly should just have to change the `` to the IP of your FSxN. +Modify the `harvest.yml` found in this repo with your clusters details. You should just have to change the `` with the IP addresses of your FSxNs. Add as many pollers as you need to monitor all your FSxNs. There should be an AWS Secrets Manager secret for each FSxN. ```yaml @@ -303,7 +306,7 @@ Copy the following to the end of the `harvest-compose.yml` file: ##### 5.6.3. Add Yet-Another-Exporter target to prometheus.yml: ```yaml -sudo sed -i -e "\$a\- job_name: 'yace'" -e '$a\ static_configs:' -e "\$a\ - targets: ['yace:8080']" container/prometheus/prometheus.yml +sudo sed -i -e "\$a\- job_name: 'yace'" -e "\$a\ static_configs:" -e "\$a\ - targets: ['yace:8080']" container/prometheus/prometheus.yml ``` ##### 6. Bring Everything Up From 622083d6dd1250fd315cf0a7bc4bc18f7d67db9c Mon Sep 17 00:00:00 2001 From: Keith Cantrell Date: Tue, 4 Mar 2025 19:15:19 -0600 Subject: [PATCH 04/11] Initial Version --- .../README-Manual.md | 325 ++++++++++++ .../README.md | 496 ++++++------------ .../harvest-grafana-cf-template.yaml | 308 +++++++++++ 3 files changed, 804 insertions(+), 325 deletions(-) create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/README-Manual.md create mode 100644 Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README-Manual.md b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README-Manual.md new file mode 100644 index 00000000..50ab3b92 --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README-Manual.md @@ -0,0 +1,325 @@ +# Deploy NetApp Harvest on EC2 + +Harvest installation for monitoring Amazon FSxN using Prometheus and Grafana stack, integrating AWS Secret Manager for FSxN credentials. + +## Introduction + +### What to Expect + +Harvest installation will result in the following: +* Install NetApp Harvest with the latest version on your EC2 instance. +* Collecting metrics about your FSxNs and adding existing Grafana dashboards for better visualization. + +### Prerequisites +* A FSx for ONTAP file system running in the same VPC as the EC2 instance. +* If not running an AWS based Linux, ensure that the `aws` command has been installed and configured. + +## Installation Steps + +### 1. Create AWS Secret Manager with Username and Password for each FSxN +Since this solution uses an AWS Secrets Manager secret to authenticate with the FSx for ONTAP file system +you will need to create a secret for each FSxN you want to monitor. You can use the following command to create a secret: + +```sh +aws secretsmanager create-secret --name --secret-string '{"username":"fsxadmin","password":""}' +``` + +### 2. Create Instance Profile with Permission to AWS Secret Manager and CloudWatch metrics + +#### 2.1. Create Policy + +Edit the harvest-policy.json file found in this repo with the ARN of the AWS Secret Manager secrets created above. +If you only have one FSxN and therefore only one secret, remove the comma after the one secret ARN (i.e. the last +entry should not have a comma after it). + +``` +{ + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + "secretsmanager:ListSecrets" + ], + "Resource": [ + "", + "" + ] + }, + { + "Effect": "Allow", + "Action": [ + "tag:GetResources", + "cloudwatch:GetMetricData", + "cloudwatch:GetMetricStatistics", + "cloudwatch:ListMetrics", + "apigateway:GET", + "aps:ListWorkspaces", + "autoscaling:DescribeAutoScalingGroups", + "dms:DescribeReplicationInstances", + "dms:DescribeReplicationTasks", + "ec2:DescribeTransitGatewayAttachments", + "ec2:DescribeSpotFleetRequests", + "shield:ListProtections", + "storagegateway:ListGateways", + "storagegateway:ListTagsForResource", + "iam:ListAccountAliases" + ], + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" +} +``` + +Run the following command to create the policy and obtain the policy ARN: +```sh +POLICY_ARN=$(aws iam create-policy --policy-name harvest-policy --policy-document file://harvest-policy.json --query Policy.Arn --output text) +``` + +#### 2.2. Create Instance Profile Role + +Run the following commands to create the instance profile role and attach the policy to it: +```sh +aws iam create-role --role-name HarvestRole --assume-role-policy-document file://trust-policy.json +aws iam attach-role-policy --role-name HarvestRole --policy-arn $POLICY_ARN +aws iam create-instance-profile --instance-profile-name HarvestProfile +aws iam add-role-to-instance-profile --instance-profile-name HarvestProfile --role-name HarvestRole +``` + +Note that the `trust-policy.json` file can be found in this repo. + +### 3. Create EC2 Instance + +We recommend using a `t2.xlarge` or larger instance type with at least 20GB disk. + +Once you have created your ec2 instance, you can use the following command to attach the instance profile: + +```sh +aws ec2 associate-iam-instance-profile --instance-id --iam-instance-profile Arn=,Name=HarvestProfile +``` +You should get the instance profile ARN from step 2.2 above. + +If your exiting ec2 instance already had an instance profile, then simply add the policy create in step 2.2 above to its instance profile role. + +### 4. Install Docker and Docker Compose + +To install Docker use the following commands if you are running an Red Hat based Linux: +```sh +sudo yum install docker +sudo curl -L https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-compose-plugin-2.6.0-3.el7.x86_64.rpm -o ./compose-plugin.rpm +sudo yum install ./compose-plugin.rpm -y +sudo systemctl start docker +``` +If you aren't running a Red Hat based Linux, you can follow the instructions [here](https://docs.docker.com/engine/install/). + +To confirm that docker has been installed correctly, run the following command: + +```sh +sudo docker run hello-world +``` + +You should get output similar to the following: +``` +Hello from Docker! +This message shows that your installation appears to be working correctly. + +To generate this message, Docker took the following steps: + 1. The Docker client contacted the Docker daemon. + 2. The Docker daemon pulled the "hello-world" image from the Docker Hub. + (amd64) + 3. The Docker daemon created a new container from that image which runs the + executable that produces the output you are currently reading. + 4. The Docker daemon streamed that output to the Docker client, which sent it + to your terminal. + +To try something more ambitious, you can run an Ubuntu container with: + $ docker run -it ubuntu bash + +Share images, automate workflows, and more with a free Docker ID: + https://hub.docker.com/ + +For more examples and ideas, visit: + https://docs.docker.com/get-started/ +``` +### 5. Install Harvest on EC2 + +Preform the following steps to install Harvest on your EC2 instance: + +#### 5.1. Generate Harvest Configuration File + +Modify the `harvest.yml` found in this repo with your clusters details. You should just have to change the `` with the IP addresses of your FSxNs. +Add as many pollers as you need to monitor all your FSxNs. There should be an AWS Secrets Manager secret for each FSxN. + +```yaml +Exporters: + prometheus1: + exporter: Prometheus + port_range: 12990-14000 + add_meta_tags: false +Defaults: + use_insecure_tls: true +Pollers: + fsx01: + datacenter: fsx + addr: + collectors: + - Rest + - RestPerf + - Ems + exporters: + - prometheus1 + credentials_script: + path: /opt/fetch-credentails + schedule: 3h + timeout: 10s + fsx02: + datacenter: fsx + addr: + collectors: + - Rest + - RestPerf + - Ems + exporters: + - prometheus1 + credentials_script: + path: /opt/fetch-credentails + schedule: 3h + timeout: 10s +``` + +#### 5.2. Generate a Docker Compose from Harvest Configuration + +Run the following command to generate a Docker Compose file from the Harvest configuration: + +```sh +docker run --rm \ + --env UID=$(id -u) --env GID=$(id -g) \ + --entrypoint "bin/harvest" \ + --volume "$(pwd):/opt/temp" \ + --volume "$(pwd)/harvest.yml:/opt/harvest/harvest.yml" \ + ghcr.io/netapp/harvest \ + generate docker full \ + --output harvest-compose.yml +``` + +:warning: Ignore the command that it outputs that it says will start the cluster. + +#### 5.3. Replace Harvest images in the harvest-compose.yml: + +Replace the Harvest image with one that supports using AWS Secret Manager for FSxN credentials: + +```yaml +sed -i 's|ghcr.io/netapp/harvest:latest|ghcr.io/tlvdevops/harvest-fsx:latest|g' harvest-compose.yml +``` + +#### 5.4. Add AWS Secret Manager Names to Docker Compose Environment Variables + +Edit the `harvest-compose.yml` file by adding the "environment" section for each FSxN with the two variables: `SECRET_NAME` and `AWS_REGION`. +These environment variables are required for the credentials script. + +For example: +```yaml +services: + fsx01: + image: ghcr.io/tlvdevops/harvest-fsx:latest + container_name: poller-fsx01 + restart: unless-stopped + ports: + - "12990:12990" + command: '--poller fsx01 --promPort 12990 --config /opt/harvest.yml' + volumes: + - ./cert:/opt/harvest/cert + - ./harvest.yml:/opt/harvest.yml + - ./conf:/opt/harvest/conf + environment: + - SECRET_NAME= + - AWS_REGION= + networks: + - backend +``` +#### 5.5. Download FSxN dashboards and import into Grafana container: +The following commands will download the FSxN designed dashboards from this repo and replace the default Grafana dashboards with them: +```yaml +wget https://raw.githubusercontent.com/NetApp/FSx-ONTAP-samples-scripts/main/Monitoring/monitor_fsxn_with_grafana/fsx_dashboards.zip +unzip fsx_dashboards.zip +rm -rf grafana/dashboards +mv dashboards grafana/dashboards +``` + +#### 5.6. Configure Prometheus to use yet-another-exporter (yace) to gather AWS FSxN metrics +AWS has useful metrics regarding the FSxN file system that ONTAP doesn't provide. Therefore, it is recommended to install +an exporter that will expose these metrics. The following steps show how to install a recommended exporter. + +##### 5.6.1 Create the yace configuration file. +Edit the `yace-config.yaml` file found in this repo and replace ``, in both places, with the region where your FSxN resides: +```yaml +apiVersion: v1alpha1 +sts-region: +discovery: + jobs: + - type: AWS/FSx + regions: [] + period: 300 + length: 300 + metrics: + - name: DiskReadOperations + statistics: [Sum] + - name: DiskWriteOperations + statistics: [Sum] + - name: DiskReadBytes + statistics: [Sum] + - name: DiskWriteBytes + statistics: [Sum] + - name: DiskIopsUtilization + statistics: [Average] + - name: NetworkThroughputUtilization + statistics: [Average] + - name: FileServerDiskThroughputUtilization + statistics: [Average] + - name: CPUUtilization + statistics: [Average] +``` + +##### 5.6.2 Add Yet-Another-Exporter to harvest-compose.yaml + +Copy the following to the end of the `harvest-compose.yml` file: +```yaml + yace: + image: quay.io/prometheuscommunity/yet-another-cloudwatch-exporter:latest + container_name: yace + restart: always + expose: + - 8080 + volumes: + - ./yace-config.yaml:/tmp/config.yml + - $HOME/.aws:/exporter/.aws:ro + command: + - -listen-address=:8080 + - -config.file=/tmp/config.yml + networks: + - backend +``` + +##### 5.6.3. Add Yet-Another-Exporter target to prometheus.yml: +```yaml +sudo sed -i -e "\$a\- job_name: 'yace'" -e "\$a\ static_configs:" -e "\$a\ - targets: ['yace:8080']" container/prometheus/prometheus.yml +``` + +##### 6. Bring Everything Up + +```sh +sudo docker compose -f prom-stack.yml -f harvest-compose.yml up -d --remove-orphans +``` + +After bringing up the prom-stack.yml compose file, you can access Grafana at +http://IP_OF_GRAFANA:3000. + +You will be prompted to create a new password the first time you log in. Grafana's default credentials are: +``` +username: admin +password: admin +``` diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md index 50ab3b92..6be4786b 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md @@ -1,325 +1,171 @@ -# Deploy NetApp Harvest on EC2 - -Harvest installation for monitoring Amazon FSxN using Prometheus and Grafana stack, integrating AWS Secret Manager for FSxN credentials. - -## Introduction - -### What to Expect - -Harvest installation will result in the following: -* Install NetApp Harvest with the latest version on your EC2 instance. -* Collecting metrics about your FSxNs and adding existing Grafana dashboards for better visualization. - -### Prerequisites -* A FSx for ONTAP file system running in the same VPC as the EC2 instance. -* If not running an AWS based Linux, ensure that the `aws` command has been installed and configured. - -## Installation Steps - -### 1. Create AWS Secret Manager with Username and Password for each FSxN -Since this solution uses an AWS Secrets Manager secret to authenticate with the FSx for ONTAP file system -you will need to create a secret for each FSxN you want to monitor. You can use the following command to create a secret: - -```sh -aws secretsmanager create-secret --name --secret-string '{"username":"fsxadmin","password":""}' -``` - -### 2. Create Instance Profile with Permission to AWS Secret Manager and CloudWatch metrics - -#### 2.1. Create Policy - -Edit the harvest-policy.json file found in this repo with the ARN of the AWS Secret Manager secrets created above. -If you only have one FSxN and therefore only one secret, remove the comma after the one secret ARN (i.e. the last -entry should not have a comma after it). - -``` -{ - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - "secretsmanager:ListSecrets" - ], - "Resource": [ - "", - "" - ] - }, - { - "Effect": "Allow", - "Action": [ - "tag:GetResources", - "cloudwatch:GetMetricData", - "cloudwatch:GetMetricStatistics", - "cloudwatch:ListMetrics", - "apigateway:GET", - "aps:ListWorkspaces", - "autoscaling:DescribeAutoScalingGroups", - "dms:DescribeReplicationInstances", - "dms:DescribeReplicationTasks", - "ec2:DescribeTransitGatewayAttachments", - "ec2:DescribeSpotFleetRequests", - "shield:ListProtections", - "storagegateway:ListGateways", - "storagegateway:ListTagsForResource", - "iam:ListAccountAliases" - ], - "Resource": [ - "*" - ] - } - ], - "Version": "2012-10-17" -} -``` - -Run the following command to create the policy and obtain the policy ARN: -```sh -POLICY_ARN=$(aws iam create-policy --policy-name harvest-policy --policy-document file://harvest-policy.json --query Policy.Arn --output text) -``` - -#### 2.2. Create Instance Profile Role - -Run the following commands to create the instance profile role and attach the policy to it: -```sh -aws iam create-role --role-name HarvestRole --assume-role-policy-document file://trust-policy.json -aws iam attach-role-policy --role-name HarvestRole --policy-arn $POLICY_ARN -aws iam create-instance-profile --instance-profile-name HarvestProfile -aws iam add-role-to-instance-profile --instance-profile-name HarvestProfile --role-name HarvestRole -``` - -Note that the `trust-policy.json` file can be found in this repo. - -### 3. Create EC2 Instance - -We recommend using a `t2.xlarge` or larger instance type with at least 20GB disk. - -Once you have created your ec2 instance, you can use the following command to attach the instance profile: - -```sh -aws ec2 associate-iam-instance-profile --instance-id --iam-instance-profile Arn=,Name=HarvestProfile -``` -You should get the instance profile ARN from step 2.2 above. - -If your exiting ec2 instance already had an instance profile, then simply add the policy create in step 2.2 above to its instance profile role. - -### 4. Install Docker and Docker Compose - -To install Docker use the following commands if you are running an Red Hat based Linux: -```sh -sudo yum install docker -sudo curl -L https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-compose-plugin-2.6.0-3.el7.x86_64.rpm -o ./compose-plugin.rpm -sudo yum install ./compose-plugin.rpm -y -sudo systemctl start docker -``` -If you aren't running a Red Hat based Linux, you can follow the instructions [here](https://docs.docker.com/engine/install/). - -To confirm that docker has been installed correctly, run the following command: - -```sh -sudo docker run hello-world -``` - -You should get output similar to the following: -``` -Hello from Docker! -This message shows that your installation appears to be working correctly. - -To generate this message, Docker took the following steps: - 1. The Docker client contacted the Docker daemon. - 2. The Docker daemon pulled the "hello-world" image from the Docker Hub. - (amd64) - 3. The Docker daemon created a new container from that image which runs the - executable that produces the output you are currently reading. - 4. The Docker daemon streamed that output to the Docker client, which sent it - to your terminal. - -To try something more ambitious, you can run an Ubuntu container with: - $ docker run -it ubuntu bash - -Share images, automate workflows, and more with a free Docker ID: - https://hub.docker.com/ - -For more examples and ideas, visit: - https://docs.docker.com/get-started/ -``` -### 5. Install Harvest on EC2 - -Preform the following steps to install Harvest on your EC2 instance: - -#### 5.1. Generate Harvest Configuration File - -Modify the `harvest.yml` found in this repo with your clusters details. You should just have to change the `` with the IP addresses of your FSxNs. -Add as many pollers as you need to monitor all your FSxNs. There should be an AWS Secrets Manager secret for each FSxN. - -```yaml -Exporters: - prometheus1: - exporter: Prometheus - port_range: 12990-14000 - add_meta_tags: false -Defaults: - use_insecure_tls: true -Pollers: - fsx01: - datacenter: fsx - addr: - collectors: - - Rest - - RestPerf - - Ems - exporters: - - prometheus1 - credentials_script: - path: /opt/fetch-credentails - schedule: 3h - timeout: 10s - fsx02: - datacenter: fsx - addr: - collectors: - - Rest - - RestPerf - - Ems - exporters: - - prometheus1 - credentials_script: - path: /opt/fetch-credentails - schedule: 3h - timeout: 10s -``` - -#### 5.2. Generate a Docker Compose from Harvest Configuration - -Run the following command to generate a Docker Compose file from the Harvest configuration: - -```sh -docker run --rm \ - --env UID=$(id -u) --env GID=$(id -g) \ - --entrypoint "bin/harvest" \ - --volume "$(pwd):/opt/temp" \ - --volume "$(pwd)/harvest.yml:/opt/harvest/harvest.yml" \ - ghcr.io/netapp/harvest \ - generate docker full \ - --output harvest-compose.yml -``` - -:warning: Ignore the command that it outputs that it says will start the cluster. - -#### 5.3. Replace Harvest images in the harvest-compose.yml: - -Replace the Harvest image with one that supports using AWS Secret Manager for FSxN credentials: - -```yaml -sed -i 's|ghcr.io/netapp/harvest:latest|ghcr.io/tlvdevops/harvest-fsx:latest|g' harvest-compose.yml -``` - -#### 5.4. Add AWS Secret Manager Names to Docker Compose Environment Variables - -Edit the `harvest-compose.yml` file by adding the "environment" section for each FSxN with the two variables: `SECRET_NAME` and `AWS_REGION`. -These environment variables are required for the credentials script. - -For example: -```yaml -services: - fsx01: - image: ghcr.io/tlvdevops/harvest-fsx:latest - container_name: poller-fsx01 - restart: unless-stopped - ports: - - "12990:12990" - command: '--poller fsx01 --promPort 12990 --config /opt/harvest.yml' - volumes: - - ./cert:/opt/harvest/cert - - ./harvest.yml:/opt/harvest.yml - - ./conf:/opt/harvest/conf - environment: - - SECRET_NAME= - - AWS_REGION= - networks: - - backend -``` -#### 5.5. Download FSxN dashboards and import into Grafana container: -The following commands will download the FSxN designed dashboards from this repo and replace the default Grafana dashboards with them: -```yaml -wget https://raw.githubusercontent.com/NetApp/FSx-ONTAP-samples-scripts/main/Monitoring/monitor_fsxn_with_grafana/fsx_dashboards.zip -unzip fsx_dashboards.zip -rm -rf grafana/dashboards -mv dashboards grafana/dashboards -``` - -#### 5.6. Configure Prometheus to use yet-another-exporter (yace) to gather AWS FSxN metrics -AWS has useful metrics regarding the FSxN file system that ONTAP doesn't provide. Therefore, it is recommended to install -an exporter that will expose these metrics. The following steps show how to install a recommended exporter. - -##### 5.6.1 Create the yace configuration file. -Edit the `yace-config.yaml` file found in this repo and replace ``, in both places, with the region where your FSxN resides: -```yaml -apiVersion: v1alpha1 -sts-region: -discovery: - jobs: - - type: AWS/FSx - regions: [] - period: 300 - length: 300 - metrics: - - name: DiskReadOperations - statistics: [Sum] - - name: DiskWriteOperations - statistics: [Sum] - - name: DiskReadBytes - statistics: [Sum] - - name: DiskWriteBytes - statistics: [Sum] - - name: DiskIopsUtilization - statistics: [Average] - - name: NetworkThroughputUtilization - statistics: [Average] - - name: FileServerDiskThroughputUtilization - statistics: [Average] - - name: CPUUtilization - statistics: [Average] -``` - -##### 5.6.2 Add Yet-Another-Exporter to harvest-compose.yaml - -Copy the following to the end of the `harvest-compose.yml` file: -```yaml - yace: - image: quay.io/prometheuscommunity/yet-another-cloudwatch-exporter:latest - container_name: yace - restart: always - expose: - - 8080 - volumes: - - ./yace-config.yaml:/tmp/config.yml - - $HOME/.aws:/exporter/.aws:ro - command: - - -listen-address=:8080 - - -config.file=/tmp/config.yml - networks: - - backend -``` - -##### 5.6.3. Add Yet-Another-Exporter target to prometheus.yml: -```yaml -sudo sed -i -e "\$a\- job_name: 'yace'" -e "\$a\ static_configs:" -e "\$a\ - targets: ['yace:8080']" container/prometheus/prometheus.yml -``` - -##### 6. Bring Everything Up - -```sh -sudo docker compose -f prom-stack.yml -f harvest-compose.yml up -d --remove-orphans -``` - -After bringing up the prom-stack.yml compose file, you can access Grafana at -http://IP_OF_GRAFANA:3000. - -You will be prompted to create a new password the first time you log in. Grafana's default credentials are: -``` -username: admin -password: admin -``` +# Harvest, Grafana plus Prometheus Deployment using AWS CloudFormation +This guide provides instructions on how to deploy a Harvest + Grafana + Prometheus stack +to monitor your Amazon FSx for NetApp ONTAP resources. It also includes +the Yet Another CloudWatch Exporter (YACE) to collect the AWS FSx for ONTAP CloudWatch metrics. + +## Yet Another CloudWatch Exporter (YACE) +YACE, or Yet Another CloudWatch Exporter, is a Prometheus exporter for AWS CloudWatch metrics. It is written in +Go and uses the official AWS SDK. YACE supports auto-discovery of resources via tags, structured logging, +filtering monitored resources via regex, and more. You can read more about YACE capabilities from its +[Documentation](https://github.com/prometheus-community/yet-another-cloudwatch-exporter). + +## Prerequisites +The only prerequisite is an FSx for ONTAP file system running in your AWS account. + +## Overview + +There are two methods to deploy this solution, either via the AWS CloudFormation template or manually. +The steps below are geared towrad the CloudFormation deployment method. If you want to deploy manually, +please refer to these [instructions](README-Manual.md). + +This deployment includes: +- **Harvest**: Collects ONTAP metrics. +- **Yet Another CloudWatch Exporter (YACE)**: Collects FSxN CloudWatch metrics. +- **Prometheus**: Stores the metrics. +- **Grafana**: Visualizes the metrics. + +## Deployment Steps + +1. **Download the AWS CloudFormation Template file** + - Download the `harvest-grafana-cf-template.yaml` file from this repo. + +2. **Create the Stack** + - Open the AWS console and to the CloudFormation service page. + - Choose **Create stack** and select **With new resources**, + - Select **Choose an existing template** and **Upload a template file** + - Upload the `harvest-grafana-cf-template.yaml` file. + - Click **Next** + +3. **Specify Stack Details** + - **Parameters**: Review and modify the parameters as needed for your file system. The default values are: + - **InstanceType**: Select the instance type to run the Harvest+Grafana+Prometheus stack. You should allocate at least 2 vCPUs and 1GB of RAM for every 10 FSxN file systems you plan to monitor. The default is `t3.medium`. + - **KeyPair**: Specify the key pair to access the EC2 instance. + - **SecurityGroup**: Ensure inbound ports 22, 3000 and 9090 are open. + - **SubnetType**: Choose `public` or `private`. `Public` will allocated a public IP address to the EC2 instance. + - **Subnet**: Specify a subnet that will have connectivity to all the FSxN file systems you plan to monitor over TCP port 433. + - **InstanceAmiId**: Select a Red Hat based Linux distrubution (e.g. AWS Linux). The default is the latest AWS Linux 2 distribution. + - **FSxEndPoint**: Specify the management endpoint IP address of your FSx file system. + - **SecretName**: Specify the AWS Secrets Manager secret name containing the password for the `fsxadmin` user. + +4. **Configure Stack Options** + - Click **Next** for stack options. + +5. **Review and Create** + - Review the stack details and confirm the settings. + - Select the check box to acknowledge that the template creates IAM resources. + - Choose **Create stack**. + +6. **Monitor Stack Creation** + - Monitor the status of the stack in the AWS CloudFormation console. The status should change to `CREATE_COMPLETE` in about five minutes. + +## Accessing Grafana + +- After the deployment is complete, log in to the Grafana dashboard using your browser: + - URL: `http://:3000` + - Default credentials: + - Username: `admin` + - Password: `admin` + - **Note**: You will be prompted to change your password upon the first login. + +## Supported Dashboards + +Amazon FSx for NetApp ONTAP exposes a different set of metrics than on-premises NetApp ONTAP. +Therefore, only the following out-of-the-box Harvest dashboards tagged with `fsx` are currently supported for use with FSx for ONTAP. +Some panels in these dashboards may be missing information that is not supported: + +- **FSxN_Clusters** +- **FSxN_CW_Utilization** +- **FSxN_Data_protection** +- **FSxN_LUN** +- **FSxN_SVM** +- **FSxN_Volume** + +--- + +## Monitor additional AWS FSx for NetApp ONTAP + +To monitor additional FSxN resources, follow these steps: + +1. **Log in via SSH to the EC2 instance** + +2. **Move to the Harvest Directory** + - Navigate to the Harvest directory: + ```bash + cd /opt/harvest + ``` + +3. **Configure Additional AWS FSx for NetApp ONTAP in `harvest.yml`** + - Edit the `harvest.yml` file to add the new AWS FSx for NetApp ONTAP configuration. For example: + + ```yaml + fsx02: + datacenter: fsx + addr: + collectors: + - Rest + - RestPerf + - Ems + exporters: + - prometheus1 + credentials_script: + path: /opt/fetch-credentials + schedule: 3h + timeout: 10s + ``` + +4. **Update `harvest-compose` with the Additional FSx for NetApp ONTAP** + - In the same directory, edit the `harvest-compose.yml` file to include the new FSx for NetApp ONTAP configuration: + + ```yaml + fsx02: + image: ghcr.io/tlvdevops/harvest-fsx:latest + container_name: poller-fsx02 + restart: unless-stopped + ports: + - "12991:12991" + command: '--poller fsx02 --promPort 12991 --config /opt/harvest.yml' + volumes: + - ./cert:/opt/harvest/cert + - ./harvest.yml:/opt/harvest.yml + - ./conf:/opt/harvest/conf + environment: + - SECRET_NAME= + - AWS_REGION= + ``` + - **Note**: Change the `container_name`, `ports`, `promPort`, and `SECRET_NAME` as needed. + Make sure that you are changing the TCP port used by the new poller, in this example the new fsx02 will use port 12991 + If you are adding multiple FSx for NetApp ONTAP, always use a different TCP port. + + +5. **Add FSx for NetApp ONTAP to Prometheus Targets** + - Navigate to the Prometheus directory: + ```bash + cd /opt/harvest/container/prometheus/ + ``` + - Edit the `harvest_targets.yml` file to add the new FSx for NetApp ONTAP target: + ```yaml + - targets: [':'] + ``` + +6. **Restart Docker Compose** + - Navigate to the Harvest directory: + ```bash + cd /opt/harvest + ``` + - Bring down the Docker Compose stack: + ```bash + docker compose -f prom-stack.yml -f harvest-compose.yml down ``` + - Bring the Docker Compose stack back up: + ```bash + docker compose -f prom-stack.yml -f harvest-compose.yml up -d --remove-orphans + ``` + +--- + +Feel free to adjust the placeholders (``, ``, ``, ``, ``) with your specific details. +## Additional Information + + +--- + +[1](https://github.com/prometheus-community/yet-another-cloudwatch-exporter): [Yet Another CloudWatch Exporter on GitHub](https://github.com/prometheus-community/yet-another-cloudwatch-exporter) diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml new file mode 100644 index 00000000..41f9df34 --- /dev/null +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml @@ -0,0 +1,308 @@ + AWSTemplateFormatVersion: 2010-09-09 + + Description: Creates the Harvest and Grafana environment to monitor your Amazon FSx for Netapp ONTAP resources. + + Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: Harvest and Grafana Instance parameters + Parameters: + - InstanceType + - KeyPair + - SecurityGroup + - SubnetType + - Subnet + - Label: + default: Amazon FSx for Netapp ONTAP file system parameters + Parameters: + - FSxEndPoint + - SecretName + ParameterLabels: + InstanceType: + default: Instance Type + KeyPair: + default: Key Pair + LatestLinuxAmiId: + default: Instance AMI ID + SecurityGroup: + default: Security group ID + SubnetType: + default: Subnet Type + Subnet: + default: Subnet ID + FSxEndPoint: + default: Management endpoint IP address + SecretName: + default: AWS Secrets Manager Secret Name + + Parameters: + InstanceType: + Description: Select the instance type. Recommended config for every 10 file system's (CPU - 2 cores, Memory - 1GB) + AllowedValues: + - t3.small + - t3.medium + - t3.large + - t3.xlarge + - t3.2xlarge + - m5.large + - m5.xlarge + - m5.2xlarge + - m5.4xlarge + - m5n.large + - m5n.xlarge + - m5n.2xlarge + - m5n.4xlarge + - c5.large + - c5.xlarge + - c5.2xlarge + - c5.4xlarge + - c5.9xlarge + - c5.12xlarge + - c5.18xlarge + - c5n.xlarge + - c5n.2xlarge + - c5n.4xlarge + - r5n.large + - r5n.xlarge + - r5n.2xlarge + - r5n.4xlarge + Default: t3.medium + Type: String + + KeyPair: + Type: AWS::EC2::KeyPair::KeyName + + SecurityGroup: + Description: Security group ID for the Harvest/Grafana Instance. Ensure Inbound ports 22, 3000 and 9090 are open. + Type: List + + SubnetType: + Description: Choose between public or private. Public means that an Public facing IP address will be assigned to the instance. + Type: String + AllowedValues: + - public + - private + + Subnet: + Type: AWS::EC2::Subnet::Id + Description: Subnet ID. Specify the same subnet as your Amazon FSx for NetApp ONTAP file system's preferred subnet. + + LatestLinuxAmiId: + Description: 'Instance ID. Defaults to the latest Amazon 2023 AMI ID.' + Type: 'AWS::SSM::Parameter::Value' + Default: '/aws/service/ami-amazon-linux-latest/al2023-ami-minimal-kernel-default-x86_64' + + FSxEndPoint: + Description: File system management endpoint IP address + Type: String + + SecretName: + Description: 'AWS Secrets Manager Secret Name containing password for the file system "fsxadmin user". Validate secret is stored in format {"username" : "fsxadmin", "password" : ""}' + Type: String + + Conditions: + CreatePublicIP: !Equals [!Ref SubnetType, public] + CreatePrivateIP: !Equals [!Ref SubnetType, private] + + Resources: + Instance: + Type: AWS::EC2::Instance + Properties: + IamInstanceProfile: !Ref MyInstanceProfile + ImageId: !Ref LatestLinuxAmiId + InstanceType: !Ref InstanceType + KeyName: !Ref KeyPair + Monitoring: true + NetworkInterfaces: + - AssociatePublicIpAddress: !If [CreatePublicIP, "true", "false"] + DeviceIndex: "0" + SubnetId: !Ref Subnet + GroupSet: !Ref SecurityGroup + Tags: + - Key: Name + Value: Harvest/Grafana for monitoring FSx for NetApp ONTAP + UserData: + 'Fn::Base64': !Sub | + #!/bin/bash + # Install Docker + yum install -y docker + systemctl start docker + systemctl enable docker + + # Download and install Docker Compose plugin + curl -L https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-compose-plugin-2.6.0-3.el7.x86_64.rpm -o /home/ec2-user/compose-plugin.rpm + yum install /home/ec2-user/compose-plugin.rpm -y + + mkdir -p /opt/harvest + cd /opt/harvest + # Create harvest.yml + cat < harvest.yml + Exporters: + prometheus1: + exporter: Prometheus + port_range: 12990-14000 + add_meta_tags: false + Defaults: + use_insecure_tls: true + Pollers: + fsx01: + datacenter: fsx + addr: ${FSxEndPoint} + collectors: + - Rest + - RestPerf + - Ems + exporters: + - prometheus1 + credentials_script: + path: /opt/fetch-credentails + schedule: 3h + timeout: 10s + EOF + + # Generate Docker Compose file for Harvest + docker run --rm \ + --env UID=$(id -u) --env GID=$(id -g) \ + --entrypoint "bin/harvest" \ + --volume "$(pwd):/opt/temp" \ + --volume "$(pwd)/harvest.yml:/opt/harvest/harvest.yml" \ + ghcr.io/netapp/harvest \ + generate docker full \ + --output harvest-compose.yml + + # Add environment variables to Docker Compose file + awk "/networks:/{print \" environment:\\n - SECRET_NAME=${SecretName}\\n - AWS_REGION=${AWS::Region}\"}1" harvest-compose.yml > temp && mv temp harvest-compose.yml + + # Replace image in Docker Compose file + sed -i 's|ghcr.io/netapp/harvest:latest|ghcr.io/tlvdevops/harvest-fsx:latest|g' harvest-compose.yml + + # Download grafana dashbaords + wget https://raw.githubusercontent.com/TLVDevOps/harvest-fsx/main/dashboards.zip + unzip dashboards.zip + rm -rf grafana/dashboards && mv dashboards grafana/dashboards + + # Create yace-config.yaml + cat < yace-config.yaml + apiVersion: v1alpha1 + sts-region: ${AWS::Region} + discovery: + jobs: + - type: AWS/FSx + regions: [${AWS::Region}] + period: 300 + length: 300 + metrics: + - name: DiskReadOperations + statistics: [Sum] + - name: DiskWriteOperations + statistics: [Sum] + - name: DiskReadBytes + statistics: [Sum] + - name: DiskWriteBytes + statistics: [Sum] + - name: DiskIopsUtilization + statistics: [Average] + - name: NetworkThroughputUtilization + statistics: [Average] + - name: FileServerDiskThroughputUtilization + statistics: [Average] + - name: CPUUtilization + statistics: [Average] + EOF + + # Append YACE service to Docker Compose file + cat <<__EOF__ >> harvest-compose.yml + yace: + image: quay.io/prometheuscommunity/yet-another-cloudwatch-exporter:latest + container_name: yace + restart: always + expose: + - 8080 + volumes: + - ./yace-config.yaml:/tmp/config.yml + - \$HOME/.aws:/exporter/.aws:ro + command: + - -listen-address=:8080 + - -config.file=/tmp/config.yml + networks: + - backend + __EOF__ + + # Append YACE job to Prometheus configuration + cat <> container/prometheus/prometheus.yml + - job_name: 'yace' + static_configs: + - targets: ['yace:8080'] + EOF + + # Start Docker Compose services + docker compose -f prom-stack.yml -f harvest-compose.yml up -d --remove-orphans + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Instance --region ${AWS::Region} + CreationPolicy: + ResourceSignal: + Timeout: PT15M + + MyInstanceProfile: + Type: 'AWS::IAM::InstanceProfile' + Properties: + Path: '/' + Roles: + - Ref: MyIAMRole + + MyIAMRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - 'ec2.amazonaws.com' + Action: + - 'sts:AssumeRole' + Path: '/' + Policies: + - PolicyName: MyIAMRolePolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - 'secretsmanager:GetSecretValue' + - 'secretsmanager:DescribeSecret' + - 'secretsmanager:ListSecrets' + Resource: + - !Sub 'arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretName}-*' + - Sid: 'Statement1' + Effect: Allow + Action: + - 'tag:GetResources' + - 'cloudwatch:GetMetricData' + - 'cloudwatch:GetMetricStatistics' + - 'cloudwatch:ListMetrics' + - 'apigateway:GET' + - 'aps:ListWorkspaces' + - 'autoscaling:DescribeAutoScalingGroups' + - 'dms:DescribeReplicationInstances' + - 'dms:DescribeReplicationTasks' + - 'ec2:DescribeTransitGatewayAttachments' + - 'ec2:DescribeSpotFleetRequests' + - 'shield:ListProtections' + - 'storagegateway:ListGateways' + - 'storagegateway:ListTagsForResource' + - 'iam:ListAccountAliases' + Resource: + - '*' + + Outputs: + PublicURL: + Condition: CreatePublicIP + Description: Use below Public IP URL to connect to the Grafana endpoint + Value: !Join [ "", ["http://",!GetAtt Instance.PublicIp,":3000" ]] + + PrivateURL: + Condition: CreatePrivateIP + Description: Use below Private IP URL to connect to the Grafana endpoint + Value: !Join [ "", ["http://",!GetAtt Instance.PrivateIp,":3000" ]] From 711093576867bad924a13ae325043917d4b4e9f3 Mon Sep 17 00:00:00 2001 From: Keith Cantrell Date: Tue, 4 Mar 2025 20:00:34 -0600 Subject: [PATCH 05/11] Initial Version. --- .../README-Manual.md | 8 ++++---- .../harvest-grafana-cf-template.yaml | 12 ++++++------ .../yace-config.yaml | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README-Manual.md b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README-Manual.md index 50ab3b92..9cd0124a 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README-Manual.md +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README-Manual.md @@ -267,13 +267,13 @@ discovery: length: 300 metrics: - name: DiskReadOperations - statistics: [Sum] + statistics: [Average] - name: DiskWriteOperations - statistics: [Sum] + statistics: [Average] - name: DiskReadBytes - statistics: [Sum] + statistics: [Average] - name: DiskWriteBytes - statistics: [Sum] + statistics: [Average] - name: DiskIopsUtilization statistics: [Average] - name: NetworkThroughputUtilization diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml index 41f9df34..f56dd873 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml @@ -89,9 +89,9 @@ Description: Subnet ID. Specify the same subnet as your Amazon FSx for NetApp ONTAP file system's preferred subnet. LatestLinuxAmiId: - Description: 'Instance ID. Defaults to the latest Amazon 2023 AMI ID.' + Description: 'Instance ID. Defaults to the latest Amazon Linux 2 AMI ID.' Type: 'AWS::SSM::Parameter::Value' - Default: '/aws/service/ami-amazon-linux-latest/al2023-ami-minimal-kernel-default-x86_64' + Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2' FSxEndPoint: Description: File system management endpoint IP address @@ -194,13 +194,13 @@ length: 300 metrics: - name: DiskReadOperations - statistics: [Sum] + statistics: [Average] - name: DiskWriteOperations - statistics: [Sum] + statistics: [Average] - name: DiskReadBytes - statistics: [Sum] + statistics: [Average] - name: DiskWriteBytes - statistics: [Sum] + statistics: [Average] - name: DiskIopsUtilization statistics: [Average] - name: NetworkThroughputUtilization diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/yace-config.yaml b/Monitoring/monitor_fsxn_with_harvest_on_ec2/yace-config.yaml index 2a4b65df..4be472d0 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/yace-config.yaml +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/yace-config.yaml @@ -8,13 +8,13 @@ discovery: length: 300 metrics: - name: DiskReadOperations - statistics: [Sum] + statistics: [Average] - name: DiskWriteOperations - statistics: [Sum] + statistics: [Average] - name: DiskReadBytes - statistics: [Sum] + statistics: [Average] - name: DiskWriteBytes - statistics: [Sum] + statistics: [Average] - name: DiskIopsUtilization statistics: [Average] - name: NetworkThroughputUtilization From 63e3d019a50b9799b30580ef381439e52781a43c Mon Sep 17 00:00:00 2001 From: Keith Cantrell Date: Tue, 4 Mar 2025 20:23:55 -0600 Subject: [PATCH 06/11] Initial Version. --- .../monitor_fsxn_with_harvest_on_ec2/README.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md index 6be4786b..935971d7 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md @@ -1,4 +1,4 @@ -# Harvest, Grafana plus Prometheus Deployment using AWS CloudFormation +# Harvest/Grafana Deployment using AWS CloudFormation This guide provides instructions on how to deploy a Harvest + Grafana + Prometheus stack to monitor your Amazon FSx for NetApp ONTAP resources. It also includes the Yet Another CloudWatch Exporter (YACE) to collect the AWS FSx for ONTAP CloudWatch metrics. @@ -43,7 +43,7 @@ This deployment includes: - **SecurityGroup**: Ensure inbound ports 22, 3000 and 9090 are open. - **SubnetType**: Choose `public` or `private`. `Public` will allocated a public IP address to the EC2 instance. - **Subnet**: Specify a subnet that will have connectivity to all the FSxN file systems you plan to monitor over TCP port 433. - - **InstanceAmiId**: Select a Red Hat based Linux distrubution (e.g. AWS Linux). The default is the latest AWS Linux 2 distribution. + - **InstanceAmiId**: Specify the Amazon Linux 2 AMI ID for the EC2 instance. The default is the latest version. - **FSxEndPoint**: Specify the management endpoint IP address of your FSx file system. - **SecretName**: Specify the AWS Secrets Manager secret name containing the password for the `fsxadmin` user. @@ -154,7 +154,8 @@ To monitor additional FSxN resources, follow these steps: ``` - Bring down the Docker Compose stack: ```bash - docker compose -f prom-stack.yml -f harvest-compose.yml down ``` + docker compose -f prom-stack.yml -f harvest-compose.yml down + ``` - Bring the Docker Compose stack back up: ```bash docker compose -f prom-stack.yml -f harvest-compose.yml up -d --remove-orphans @@ -162,10 +163,6 @@ To monitor additional FSxN resources, follow these steps: --- -Feel free to adjust the placeholders (``, ``, ``, ``, ``) with your specific details. ## Additional Information - - --- - -[1](https://github.com/prometheus-community/yet-another-cloudwatch-exporter): [Yet Another CloudWatch Exporter on GitHub](https://github.com/prometheus-community/yet-another-cloudwatch-exporter) +[Yet Another CloudWatch Exporter on GitHub](https://github.com/prometheus-community/yet-another-cloudwatch-exporter) From 8363f95d1c0ac7865d055b74ce5791bfb96ceb48 Mon Sep 17 00:00:00 2001 From: shmueldanan Date: Wed, 5 Mar 2025 15:17:52 +0200 Subject: [PATCH 07/11] Update README.md Changed text --- Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md index 935971d7..ec99e8d8 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md @@ -1,9 +1,7 @@ # Harvest/Grafana Deployment using AWS CloudFormation -This guide provides instructions on how to deploy a Harvest + Grafana + Prometheus stack -to monitor your Amazon FSx for NetApp ONTAP resources. It also includes -the Yet Another CloudWatch Exporter (YACE) to collect the AWS FSx for ONTAP CloudWatch metrics. +This is the NetApp FSx for ONTAP deployment for monitoring FSx for ONTAP file systems with Grafana. +The following solution leverages Harvest and YACE(Yet Another CloudWatch Exporter) as the exporters for ONTAP and CloudWatch metrics. -## Yet Another CloudWatch Exporter (YACE) YACE, or Yet Another CloudWatch Exporter, is a Prometheus exporter for AWS CloudWatch metrics. It is written in Go and uses the official AWS SDK. YACE supports auto-discovery of resources via tags, structured logging, filtering monitored resources via regex, and more. You can read more about YACE capabilities from its @@ -19,8 +17,8 @@ The steps below are geared towrad the CloudFormation deployment method. If you w please refer to these [instructions](README-Manual.md). This deployment includes: -- **Harvest**: Collects ONTAP metrics. -- **Yet Another CloudWatch Exporter (YACE)**: Collects FSxN CloudWatch metrics. +- **Harvest**: Collects ONTAP metrics.[Documentation](https://github.com/NetApp/harvest). +- **Yet Another CloudWatch Exporter (YACE)**: Collects FSxN CloudWatch metrics.[Documentation](https://github.com/prometheus-community/yet-another-cloudwatch-exporter). - **Prometheus**: Stores the metrics. - **Grafana**: Visualizes the metrics. From 842dd0f8448546d1895fa8113f7b53abb55b8bbc Mon Sep 17 00:00:00 2001 From: shmueldanan Date: Wed, 5 Mar 2025 15:18:19 +0200 Subject: [PATCH 08/11] Update README.md --- Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md index ec99e8d8..8cfb612b 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/README.md @@ -160,7 +160,3 @@ To monitor additional FSxN resources, follow these steps: ``` --- - -## Additional Information ---- -[Yet Another CloudWatch Exporter on GitHub](https://github.com/prometheus-community/yet-another-cloudwatch-exporter) From 821f3162d8443d97acf36b84d5af50a168f4cd14 Mon Sep 17 00:00:00 2001 From: Keith Cantrell Date: Wed, 5 Mar 2025 07:22:17 -0600 Subject: [PATCH 09/11] Initial Version --- .../harvest-grafana-cf-template.yaml | 42 ++++++------------- 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml index f56dd873..a56f2874 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml @@ -40,37 +40,19 @@ InstanceType: Description: Select the instance type. Recommended config for every 10 file system's (CPU - 2 cores, Memory - 1GB) AllowedValues: - - t3.small - - t3.medium - - t3.large - - t3.xlarge - - t3.2xlarge - - m5.large - - m5.xlarge - - m5.2xlarge - - m5.4xlarge - - m5n.large - - m5n.xlarge - - m5n.2xlarge - - m5n.4xlarge - - c5.large - - c5.xlarge - - c5.2xlarge - - c5.4xlarge - - c5.9xlarge - - c5.12xlarge - - c5.18xlarge - - c5n.xlarge - - c5n.2xlarge - - c5n.4xlarge - - r5n.large - - r5n.xlarge - - r5n.2xlarge - - r5n.4xlarge + - t3.medium + - t3.large + - t3.xlarge + - t3.2xlarge + - m5n.large + - m5n.xlarge + - m5n.2xlarge + - m5n.4xlarge Default: t3.medium Type: String KeyPair: + Description: Name of an existing EC2 KeyPair to enable SSH access to the instance. Type: AWS::EC2::KeyPair::KeyName SecurityGroup: @@ -81,12 +63,12 @@ Description: Choose between public or private. Public means that an Public facing IP address will be assigned to the instance. Type: String AllowedValues: - - public - - private + - public + - private Subnet: - Type: AWS::EC2::Subnet::Id Description: Subnet ID. Specify the same subnet as your Amazon FSx for NetApp ONTAP file system's preferred subnet. + Type: AWS::EC2::Subnet::Id LatestLinuxAmiId: Description: 'Instance ID. Defaults to the latest Amazon Linux 2 AMI ID.' From 098950beccb08287dd7da3f1112ac9c173e86257 Mon Sep 17 00:00:00 2001 From: Keith Cantrell Date: Wed, 5 Mar 2025 07:47:22 -0600 Subject: [PATCH 10/11] Fixed formatting to appease yamllint --- .../harvest-grafana-cf-template.yaml | 506 +++++++++--------- 1 file changed, 253 insertions(+), 253 deletions(-) diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml index a56f2874..fd083e90 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml @@ -1,12 +1,12 @@ - AWSTemplateFormatVersion: 2010-09-09 +AWSTemplateFormatVersion: 2010-09-09 - Description: Creates the Harvest and Grafana environment to monitor your Amazon FSx for Netapp ONTAP resources. +Description: Creates the Harvest and Grafana environment to monitor your Amazon FSx for Netapp ONTAP resources. - Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: - Label: - default: Harvest and Grafana Instance parameters + default: Harvest and Grafana Instance parameters Parameters: - InstanceType - KeyPair @@ -14,277 +14,277 @@ - SubnetType - Subnet - Label: - default: Amazon FSx for Netapp ONTAP file system parameters + default: Amazon FSx for Netapp ONTAP file system parameters Parameters: - FSxEndPoint - SecretName - ParameterLabels: - InstanceType: - default: Instance Type - KeyPair: - default: Key Pair - LatestLinuxAmiId: - default: Instance AMI ID - SecurityGroup: - default: Security group ID - SubnetType: - default: Subnet Type - Subnet: - default: Subnet ID - FSxEndPoint: - default: Management endpoint IP address - SecretName: - default: AWS Secrets Manager Secret Name + ParameterLabels: + InstanceType: + default: Instance Type + KeyPair: + default: Key Pair + LatestLinuxAmiId: + default: Instance AMI ID + SecurityGroup: + default: Security group ID + SubnetType: + default: Subnet Type + Subnet: + default: Subnet ID + FSxEndPoint: + default: Management endpoint IP address + SecretName: + default: AWS Secrets Manager Secret Name - Parameters: - InstanceType: - Description: Select the instance type. Recommended config for every 10 file system's (CPU - 2 cores, Memory - 1GB) - AllowedValues: - - t3.medium - - t3.large - - t3.xlarge - - t3.2xlarge - - m5n.large - - m5n.xlarge - - m5n.2xlarge - - m5n.4xlarge - Default: t3.medium - Type: String +Parameters: + InstanceType: + Description: Select the instance type. Recommended config for every 10 file system's (CPU - 2 cores, Memory - 1GB) + AllowedValues: + - t3.medium + - t3.large + - t3.xlarge + - t3.2xlarge + - m5n.large + - m5n.xlarge + - m5n.2xlarge + - m5n.4xlarge + Default: t3.medium + Type: String - KeyPair: - Description: Name of an existing EC2 KeyPair to enable SSH access to the instance. - Type: AWS::EC2::KeyPair::KeyName + KeyPair: + Description: Name of an existing EC2 KeyPair to enable SSH access to the instance. + Type: AWS::EC2::KeyPair::KeyName - SecurityGroup: - Description: Security group ID for the Harvest/Grafana Instance. Ensure Inbound ports 22, 3000 and 9090 are open. - Type: List + SecurityGroup: + Description: Security group ID for the Harvest/Grafana Instance. Ensure Inbound ports 22, 3000 and 9090 are open. + Type: List - SubnetType: - Description: Choose between public or private. Public means that an Public facing IP address will be assigned to the instance. - Type: String - AllowedValues: - - public - - private + SubnetType: + Description: Choose between public or private. Public means that an Public facing IP address will be assigned to the instance. + Type: String + AllowedValues: + - public + - private - Subnet: - Description: Subnet ID. Specify the same subnet as your Amazon FSx for NetApp ONTAP file system's preferred subnet. - Type: AWS::EC2::Subnet::Id + Subnet: + Description: Subnet ID. Specify the same subnet as your Amazon FSx for NetApp ONTAP file system's preferred subnet. + Type: AWS::EC2::Subnet::Id - LatestLinuxAmiId: - Description: 'Instance ID. Defaults to the latest Amazon Linux 2 AMI ID.' - Type: 'AWS::SSM::Parameter::Value' - Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2' + LatestLinuxAmiId: + Description: 'Instance ID. Defaults to the latest Amazon Linux 2 AMI ID.' + Type: 'AWS::SSM::Parameter::Value' + Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2' - FSxEndPoint: - Description: File system management endpoint IP address - Type: String + FSxEndPoint: + Description: File system management endpoint IP address + Type: String - SecretName: - Description: 'AWS Secrets Manager Secret Name containing password for the file system "fsxadmin user". Validate secret is stored in format {"username" : "fsxadmin", "password" : ""}' - Type: String + SecretName: + Description: 'AWS Secrets Manager Secret Name containing password for the file system "fsxadmin user". Validate secret is stored in format {"username" : "fsxadmin", "password" : ""}' + Type: String - Conditions: - CreatePublicIP: !Equals [!Ref SubnetType, public] - CreatePrivateIP: !Equals [!Ref SubnetType, private] +Conditions: + CreatePublicIP: !Equals [!Ref SubnetType, public] + CreatePrivateIP: !Equals [!Ref SubnetType, private] - Resources: - Instance: - Type: AWS::EC2::Instance - Properties: - IamInstanceProfile: !Ref MyInstanceProfile - ImageId: !Ref LatestLinuxAmiId - InstanceType: !Ref InstanceType - KeyName: !Ref KeyPair - Monitoring: true - NetworkInterfaces: - - AssociatePublicIpAddress: !If [CreatePublicIP, "true", "false"] - DeviceIndex: "0" - SubnetId: !Ref Subnet - GroupSet: !Ref SecurityGroup - Tags: - - Key: Name - Value: Harvest/Grafana for monitoring FSx for NetApp ONTAP - UserData: - 'Fn::Base64': !Sub | - #!/bin/bash - # Install Docker - yum install -y docker - systemctl start docker - systemctl enable docker +Resources: + Instance: + Type: AWS::EC2::Instance + Properties: + IamInstanceProfile: !Ref MyInstanceProfile + ImageId: !Ref LatestLinuxAmiId + InstanceType: !Ref InstanceType + KeyName: !Ref KeyPair + Monitoring: true + NetworkInterfaces: + - AssociatePublicIpAddress: !If [CreatePublicIP, "true", "false"] + DeviceIndex: "0" + SubnetId: !Ref Subnet + GroupSet: !Ref SecurityGroup + Tags: + - Key: Name + Value: Harvest/Grafana for monitoring FSx for NetApp ONTAP + UserData: + 'Fn::Base64': !Sub | + #!/bin/bash + # Install Docker + yum install -y docker + systemctl start docker + systemctl enable docker - # Download and install Docker Compose plugin - curl -L https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-compose-plugin-2.6.0-3.el7.x86_64.rpm -o /home/ec2-user/compose-plugin.rpm - yum install /home/ec2-user/compose-plugin.rpm -y + # Download and install Docker Compose plugin + curl -L https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-compose-plugin-2.6.0-3.el7.x86_64.rpm -o /home/ec2-user/compose-plugin.rpm + yum install /home/ec2-user/compose-plugin.rpm -y - mkdir -p /opt/harvest - cd /opt/harvest - # Create harvest.yml - cat < harvest.yml - Exporters: - prometheus1: - exporter: Prometheus - port_range: 12990-14000 - add_meta_tags: false - Defaults: - use_insecure_tls: true - Pollers: - fsx01: - datacenter: fsx - addr: ${FSxEndPoint} - collectors: - - Rest - - RestPerf - - Ems - exporters: - - prometheus1 - credentials_script: - path: /opt/fetch-credentails - schedule: 3h - timeout: 10s - EOF + mkdir -p /opt/harvest + cd /opt/harvest + # Create harvest.yml + cat < harvest.yml + Exporters: + prometheus1: + exporter: Prometheus + port_range: 12990-14000 + add_meta_tags: false + Defaults: + use_insecure_tls: true + Pollers: + fsx01: + datacenter: fsx + addr: ${FSxEndPoint} + collectors: + - Rest + - RestPerf + - Ems + exporters: + - prometheus1 + credentials_script: + path: /opt/fetch-credentails + schedule: 3h + timeout: 10s + EOF - # Generate Docker Compose file for Harvest - docker run --rm \ - --env UID=$(id -u) --env GID=$(id -g) \ - --entrypoint "bin/harvest" \ - --volume "$(pwd):/opt/temp" \ - --volume "$(pwd)/harvest.yml:/opt/harvest/harvest.yml" \ - ghcr.io/netapp/harvest \ - generate docker full \ - --output harvest-compose.yml + # Generate Docker Compose file for Harvest + docker run --rm \ + --env UID=$(id -u) --env GID=$(id -g) \ + --entrypoint "bin/harvest" \ + --volume "$(pwd):/opt/temp" \ + --volume "$(pwd)/harvest.yml:/opt/harvest/harvest.yml" \ + ghcr.io/netapp/harvest \ + generate docker full \ + --output harvest-compose.yml - # Add environment variables to Docker Compose file - awk "/networks:/{print \" environment:\\n - SECRET_NAME=${SecretName}\\n - AWS_REGION=${AWS::Region}\"}1" harvest-compose.yml > temp && mv temp harvest-compose.yml + # Add environment variables to Docker Compose file + awk "/networks:/{print \" environment:\\n - SECRET_NAME=${SecretName}\\n - AWS_REGION=${AWS::Region}\"}1" harvest-compose.yml > temp && mv temp harvest-compose.yml - # Replace image in Docker Compose file - sed -i 's|ghcr.io/netapp/harvest:latest|ghcr.io/tlvdevops/harvest-fsx:latest|g' harvest-compose.yml + # Replace image in Docker Compose file + sed -i 's|ghcr.io/netapp/harvest:latest|ghcr.io/tlvdevops/harvest-fsx:latest|g' harvest-compose.yml - # Download grafana dashbaords - wget https://raw.githubusercontent.com/TLVDevOps/harvest-fsx/main/dashboards.zip - unzip dashboards.zip - rm -rf grafana/dashboards && mv dashboards grafana/dashboards + # Download grafana dashbaords + wget https://raw.githubusercontent.com/TLVDevOps/harvest-fsx/main/dashboards.zip + unzip dashboards.zip + rm -rf grafana/dashboards && mv dashboards grafana/dashboards - # Create yace-config.yaml - cat < yace-config.yaml - apiVersion: v1alpha1 - sts-region: ${AWS::Region} - discovery: - jobs: - - type: AWS/FSx - regions: [${AWS::Region}] - period: 300 - length: 300 - metrics: - - name: DiskReadOperations - statistics: [Average] - - name: DiskWriteOperations - statistics: [Average] - - name: DiskReadBytes - statistics: [Average] - - name: DiskWriteBytes - statistics: [Average] - - name: DiskIopsUtilization - statistics: [Average] - - name: NetworkThroughputUtilization - statistics: [Average] - - name: FileServerDiskThroughputUtilization - statistics: [Average] - - name: CPUUtilization - statistics: [Average] - EOF + # Create yace-config.yaml + cat < yace-config.yaml + apiVersion: v1alpha1 + sts-region: ${AWS::Region} + discovery: + jobs: + - type: AWS/FSx + regions: [${AWS::Region}] + period: 300 + length: 300 + metrics: + - name: DiskReadOperations + statistics: [Average] + - name: DiskWriteOperations + statistics: [Average] + - name: DiskReadBytes + statistics: [Average] + - name: DiskWriteBytes + statistics: [Average] + - name: DiskIopsUtilization + statistics: [Average] + - name: NetworkThroughputUtilization + statistics: [Average] + - name: FileServerDiskThroughputUtilization + statistics: [Average] + - name: CPUUtilization + statistics: [Average] + EOF - # Append YACE service to Docker Compose file - cat <<__EOF__ >> harvest-compose.yml - yace: - image: quay.io/prometheuscommunity/yet-another-cloudwatch-exporter:latest - container_name: yace - restart: always - expose: - - 8080 - volumes: - - ./yace-config.yaml:/tmp/config.yml - - \$HOME/.aws:/exporter/.aws:ro - command: - - -listen-address=:8080 - - -config.file=/tmp/config.yml - networks: - - backend - __EOF__ + # Append YACE service to Docker Compose file + cat <<__EOF__ >> harvest-compose.yml + yace: + image: quay.io/prometheuscommunity/yet-another-cloudwatch-exporter:latest + container_name: yace + restart: always + expose: + - 8080 + volumes: + - ./yace-config.yaml:/tmp/config.yml + - \$HOME/.aws:/exporter/.aws:ro + command: + - -listen-address=:8080 + - -config.file=/tmp/config.yml + networks: + - backend + __EOF__ - # Append YACE job to Prometheus configuration - cat <> container/prometheus/prometheus.yml - - job_name: 'yace' - static_configs: - - targets: ['yace:8080'] - EOF + # Append YACE job to Prometheus configuration + cat <> container/prometheus/prometheus.yml + - job_name: 'yace' + static_configs: + - targets: ['yace:8080'] + EOF - # Start Docker Compose services - docker compose -f prom-stack.yml -f harvest-compose.yml up -d --remove-orphans - /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Instance --region ${AWS::Region} - CreationPolicy: - ResourceSignal: - Timeout: PT15M + # Start Docker Compose services + docker compose -f prom-stack.yml -f harvest-compose.yml up -d --remove-orphans + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource Instance --region ${AWS::Region} + CreationPolicy: + ResourceSignal: + Timeout: PT15M - MyInstanceProfile: - Type: 'AWS::IAM::InstanceProfile' - Properties: - Path: '/' - Roles: - - Ref: MyIAMRole + MyInstanceProfile: + Type: 'AWS::IAM::InstanceProfile' + Properties: + Path: '/' + Roles: + - Ref: MyIAMRole - MyIAMRole: - Type: 'AWS::IAM::Role' - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - 'ec2.amazonaws.com' - Action: - - 'sts:AssumeRole' - Path: '/' - Policies: - - PolicyName: MyIAMRolePolicy - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - 'secretsmanager:GetSecretValue' - - 'secretsmanager:DescribeSecret' - - 'secretsmanager:ListSecrets' - Resource: - - !Sub 'arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretName}-*' - - Sid: 'Statement1' - Effect: Allow - Action: - - 'tag:GetResources' - - 'cloudwatch:GetMetricData' - - 'cloudwatch:GetMetricStatistics' - - 'cloudwatch:ListMetrics' - - 'apigateway:GET' - - 'aps:ListWorkspaces' - - 'autoscaling:DescribeAutoScalingGroups' - - 'dms:DescribeReplicationInstances' - - 'dms:DescribeReplicationTasks' - - 'ec2:DescribeTransitGatewayAttachments' - - 'ec2:DescribeSpotFleetRequests' - - 'shield:ListProtections' - - 'storagegateway:ListGateways' - - 'storagegateway:ListTagsForResource' - - 'iam:ListAccountAliases' - Resource: - - '*' + MyIAMRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - 'ec2.amazonaws.com' + Action: + - 'sts:AssumeRole' + Path: '/' + Policies: + - PolicyName: MyIAMRolePolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - 'secretsmanager:GetSecretValue' + - 'secretsmanager:DescribeSecret' + - 'secretsmanager:ListSecrets' + Resource: + - !Sub 'arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SecretName}-*' + - Sid: 'Statement1' + Effect: Allow + Action: + - 'tag:GetResources' + - 'cloudwatch:GetMetricData' + - 'cloudwatch:GetMetricStatistics' + - 'cloudwatch:ListMetrics' + - 'apigateway:GET' + - 'aps:ListWorkspaces' + - 'autoscaling:DescribeAutoScalingGroups' + - 'dms:DescribeReplicationInstances' + - 'dms:DescribeReplicationTasks' + - 'ec2:DescribeTransitGatewayAttachments' + - 'ec2:DescribeSpotFleetRequests' + - 'shield:ListProtections' + - 'storagegateway:ListGateways' + - 'storagegateway:ListTagsForResource' + - 'iam:ListAccountAliases' + Resource: + - '*' - Outputs: - PublicURL: - Condition: CreatePublicIP - Description: Use below Public IP URL to connect to the Grafana endpoint - Value: !Join [ "", ["http://",!GetAtt Instance.PublicIp,":3000" ]] +Outputs: + PublicURL: + Condition: CreatePublicIP + Description: Use below Public IP URL to connect to the Grafana endpoint + Value: !Join [ "", ["http://",!GetAtt Instance.PublicIp,":3000" ]] - PrivateURL: - Condition: CreatePrivateIP - Description: Use below Private IP URL to connect to the Grafana endpoint - Value: !Join [ "", ["http://",!GetAtt Instance.PrivateIp,":3000" ]] + PrivateURL: + Condition: CreatePrivateIP + Description: Use below Private IP URL to connect to the Grafana endpoint + Value: !Join [ "", ["http://",!GetAtt Instance.PrivateIp,":3000" ]] From f16039e1d3503dc2fc8285eaca9a19ee07ad6bc4 Mon Sep 17 00:00:00 2001 From: Keith Cantrell Date: Wed, 5 Mar 2025 07:58:25 -0600 Subject: [PATCH 11/11] Fixed formatting to appease yamllint --- .../harvest-grafana-cf-template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml index fd083e90..30fe34cd 100644 --- a/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml +++ b/Monitoring/monitor_fsxn_with_harvest_on_ec2/harvest-grafana-cf-template.yaml @@ -282,9 +282,9 @@ Outputs: PublicURL: Condition: CreatePublicIP Description: Use below Public IP URL to connect to the Grafana endpoint - Value: !Join [ "", ["http://",!GetAtt Instance.PublicIp,":3000" ]] + Value: !Join [ "", ["http://", !GetAtt Instance.PublicIp, ":3000" ]] PrivateURL: Condition: CreatePrivateIP Description: Use below Private IP URL to connect to the Grafana endpoint - Value: !Join [ "", ["http://",!GetAtt Instance.PrivateIp,":3000" ]] + Value: !Join [ "", ["http://", !GetAtt Instance.PrivateIp, ":3000" ]]