Refactor opensearch to use as submixin (modular observ lib) (#1045)

* Add filteringSelector to opensearch to use as submixin

* Format jsonnet

* Add filtering selector to alerts

* Add uid prefix to all dashboards/alerts

* Add commonlib

* Add variables file

* Add grafonnet

* Fix variables file

* Fix cluster dashboard

* Update config

* Refactor dashboard contstructor to new grafonnet

* Update node dashboard

* Update node dashboards

* Update search and index

* Add prefix

* Add os role panels

* Fmt

* Fix nodeIO panel

* jsonnetfmt

* Fix io panel

* Lint exclusions

* lint

Author: v-zhuravlev

opensearch-mixin/.lint

@@ -1,4 +1,9 @@
 exclusions:
+  panel-datasource-rule:
+    reason: Uses --mixed-- (generated from grafonnet)
+    entries:
+      - panel: 'OpenSearch cluster overview'
+      - panel: 'Roles'
   panel-units-rule:
     reason: "Custom units are used for better user experience in these panels"
     entries:
@@ -27,6 +32,8 @@ exclusions:
       - panel: "Merge count"
       - panel: "Shard count"
       - panel: "Node open connections"
+      - panel: 'OpenSearch cluster overview'
+      - panel: 'Roles'
   template-instance-rule:
     reason: "Based on new convention we are using variable names prometheus_datasource and loki_datasource where as linter expects 'datasource'"
   target-instance-rule:

opensearch-mixin/alerts/alerts.libsonnet

@@ -2,12 +2,12 @@
   prometheusAlerts+:: {
     groups+: [
       {
-        name: 'opensearch',
+        name: $._config.uid + '-alerts',
         rules: [
           {
             alert: 'OpenSearchYellowCluster',
             expr: |||
-              opensearch_cluster_status == 1
+              opensearch_cluster_status{%(filteringSelector)s} == 1
             ||| % $._config,
             'for': '5m',
             labels: {
@@ -24,7 +24,7 @@
           {
             alert: 'OpenSearchRedCluster',
             expr: |||
-              opensearch_cluster_status == 2
+              opensearch_cluster_status{%(filteringSelector)s} == 2
             ||| % $._config,
             'for': '5m',
             labels: {
@@ -41,7 +41,7 @@
           {
             alert: 'OpenSearchUnstableShardReallocation',
             expr: |||
-              sum without(type) (opensearch_cluster_shards_number{type="relocating"}) > %(alertsWarningShardReallocations)s
+              sum without(type) (opensearch_cluster_shards_number{%(filteringSelector)s, type="relocating"}) > %(alertsWarningShardReallocations)s
             ||| % $._config,
             'for': '1m',
             labels: {
@@ -57,7 +57,7 @@
           {
             alert: 'OpenSearchUnstableShardUnassigned',
             expr: |||
-              sum without(type) (opensearch_cluster_shards_number{type="unassigned"}) > %(alertsWarningShardUnassigned)s
+              sum without(type) (opensearch_cluster_shards_number{%(filteringSelector)s, type="unassigned"}) > %(alertsWarningShardUnassigned)s
             ||| % $._config,
             'for': '5m',
             labels: {
@@ -73,7 +73,7 @@
           {
             alert: 'OpenSearchModerateNodeDiskUsage',
             expr: |||
-              100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes - opensearch_fs_path_free_bytes) / opensearch_fs_path_total_bytes) > %(alertsWarningDiskUsage)s
+              100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes{%(filteringSelector)s} - opensearch_fs_path_free_bytes{%(filteringSelector)s}) / opensearch_fs_path_total_bytes{%(filteringSelector)s}) > %(alertsWarningDiskUsage)s
             ||| % $._config,
             'for': '5m',
             labels: {
@@ -89,7 +89,7 @@
           {
             alert: 'OpenSearchHighNodeDiskUsage',
             expr: |||
-              100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes - opensearch_fs_path_free_bytes) / opensearch_fs_path_total_bytes) > %(alertsCriticalDiskUsage)s
+              100 * sum without(nodeid, path, mount, type) ((opensearch_fs_path_total_bytes{%(filteringSelector)s} - opensearch_fs_path_free_bytes) / opensearch_fs_path_total_bytes{%(filteringSelector)s}) > %(alertsCriticalDiskUsage)s
             ||| % $._config,
             'for': '5m',
             labels: {
@@ -105,7 +105,7 @@
           {
             alert: 'OpenSearchModerateNodeCpuUsage',
             expr: |||
-              sum without(nodeid) (opensearch_os_cpu_percent) > %(alertsWarningCPUUsage)s
+              sum without(nodeid) (opensearch_os_cpu_percent{%(filteringSelector)s}) > %(alertsWarningCPUUsage)s
             ||| % $._config,
             'for': '5m',
             labels: {
@@ -121,7 +121,7 @@
           {
             alert: 'OpenSearchHighNodeCpuUsage',
             expr: |||
-              sum without(nodeid) (opensearch_os_cpu_percent) > %(alertsCriticalCPUUsage)s
+              sum without(nodeid) (opensearch_os_cpu_percent{%(filteringSelector)s}) > %(alertsCriticalCPUUsage)s
             ||| % $._config,
             'for': '5m',
             labels: {
@@ -137,7 +137,7 @@
           {
             alert: 'OpenSearchModerateNodeMemoryUsage',
             expr: |||
-              sum without(nodeid) (opensearch_os_mem_used_percent) > %(alertsWarningMemoryUsage)s
+              sum without(nodeid) (opensearch_os_mem_used_percent{%(filteringSelector)s}) > %(alertsWarningMemoryUsage)s
             ||| % $._config,
             'for': '5m',
             labels: {
@@ -153,7 +153,7 @@
           {
             alert: 'OpenSearchHighNodeMemoryUsage',
             expr: |||
-              sum without(nodeid) (opensearch_os_mem_used_percent) > %(alertsCriticalMemoryUsage)s
+              sum without(nodeid) (opensearch_os_mem_used_percent{%(filteringSelector)s}) > %(alertsCriticalMemoryUsage)s
             ||| % $._config,
             'for': '5m',
             labels: {
@@ -169,7 +169,7 @@
           {
             alert: 'OpenSearchModerateRequestLatency',
             expr: |||
-              sum without(context) ((increase(opensearch_index_search_fetch_time_seconds{context="total"}[5m])+increase(opensearch_index_search_query_time_seconds{context="total"}[5m])+increase(opensearch_index_search_scroll_time_seconds{context="total"}[5m])) / clamp_min(increase(opensearch_index_search_fetch_count{context="total"}[5m])+increase(opensearch_index_search_query_count{context="total"}[5m])+increase(opensearch_index_search_scroll_count{context="total"}[5m]), 1)) > %(alertsWarningRequestLatency)s
+              sum without(context) ((increase(opensearch_index_search_fetch_time_seconds{%(filteringSelector)s, context="total"}[5m])+increase(opensearch_index_search_query_time_seconds{context="total"}[5m])+increase(opensearch_index_search_scroll_time_seconds{context="total"}[5m])) / clamp_min(increase(opensearch_index_search_fetch_count{context="total"}[5m])+increase(opensearch_index_search_query_count{context="total"}[5m])+increase(opensearch_index_search_scroll_count{context="total"}[5m]), 1)) > %(alertsWarningRequestLatency)s
             ||| % $._config,
             'for': '5m',
             labels: {
@@ -185,7 +185,7 @@
           {
             alert: 'OpenSearchModerateIndexLatency',
             expr: |||
-              sum without(context) (increase(opensearch_index_indexing_index_time_seconds{context="total"}[5m]) / clamp_min(increase(opensearch_index_indexing_index_count{context="total"}[5m]), 1)) > %(alertsWarningIndexLatency)s
+              sum without(context) (increase(opensearch_index_indexing_index_time_seconds{%(filteringSelector)s, context="total"}[5m]) / clamp_min(increase(opensearch_index_indexing_index_count{context="total"}[5m]), 1)) > %(alertsWarningIndexLatency)s
             ||| % $._config,
             'for': '5m',
             labels: {

opensearch-mixin/config.libsonnet

@@ -1,9 +1,17 @@
 {
   _config+:: {
+    // extra static selector to apply to all templated variables and alerts
+    filteringSelector: 'cluster!=""',
+    groupLabels: ['job', 'cluster'],
+    instanceLabels: ['node'],
     dashboardTags: ['opensearch-mixin'],
     dashboardPeriod: 'now-1h',
     dashboardTimezone: 'default',
     dashboardRefresh: '1m',
+    dashboardNamePrefix: '',
+
+    // prefix dashboards uids
+    uid: 'opensearch',
 
     // alerts thresholds
     alertsWarningShardReallocations: 0,