feat: add operator responses dashboard (#1648)

Author: avilagaston9

grafana/provisioning/dashboards/aligned/aggregator_batcher.json

@@ -2732,7 +2732,7 @@
             "mode": "continuous-GrYlRd"
           },
           "custom": {
-            "align": "auto",
+            "align": "left",
             "cellOptions": {
               "type": "auto"
             },
@@ -2758,7 +2758,7 @@
           {
             "matcher": {
               "id": "byName",
-              "options": "Max"
+              "options": "Last *"
             },
             "properties": [
               {
@@ -2819,7 +2819,7 @@
           "options": {
             "labelsToFields": false,
             "reducers": [
-              "max"
+              "lastNotNull"
             ]
           }
         },
@@ -2830,7 +2830,136 @@
             "sort": [
               {
                 "desc": true,
-                "field": "Max"
+                "field": "Last *"
+              }
+            ]
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "dark-green",
+            "mode": "fixed"
+          },
+          "custom": {
+            "align": "left",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "noValue": "none",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Last *"
+            },
+            "properties": [
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "lcd",
+                  "type": "gauge",
+                  "valueDisplayMode": "text"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 69
+      },
+      "id": 51,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": false
+      },
+      "pluginVersion": "10.1.10",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "floor(increase(operator_response_count{job=\"aligned-tracker\"}[$__range]))",
+          "format": "time_series",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "interval": "1",
+          "legendFormat": "{{operator}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "floor(increase(aligned_aggregator_received_tasks_count{bot=\"aggregator\"}[$__range]))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Tasks Received",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "# Operator Responses",
+      "transformations": [
+        {
+          "id": "reduce",
+          "options": {
+            "labelsToFields": false,
+            "reducers": [
+              "lastNotNull"
+            ]
+          }
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "desc": false,
+                "field": "Last *"
               }
             ]
           }
@@ -2847,7 +2976,7 @@
         "h": 2,
         "w": 24,
         "x": 0,
-        "y": 69
+        "y": 77
       },
       "id": 46,
       "options": {
@@ -2922,7 +3051,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 71
+        "y": 79
       },
       "id": 47,
       "options": {
@@ -3022,7 +3151,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 71
+        "y": 79
       },
       "id": 43,
       "interval": "1s",
@@ -3102,8 +3231,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "color": "green"
               },
               {
                 "color": "red",
@@ -3119,7 +3247,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 79
+        "y": 87
       },
       "id": 45,
       "options": {
@@ -3234,8 +3362,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "color": "green"
               },
               {
                 "color": "red",
@@ -3250,7 +3377,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 79
+        "y": 87
       },
       "id": 44,
       "interval": "1s",
@@ -3293,7 +3420,7 @@
     "list": []
   },
   "time": {
-    "from": "now-30m",
+    "from": "now-1h",
     "to": "now"
   },
   "timepicker": {},

telemetry_api/lib/telemetry_api/prometheus_metrics.ex

@@ -3,6 +3,7 @@ defmodule TelemetryApi.PrometheusMetrics do
 
   @gauge [name: :gas_price, help: "Ethereum Gas Price.", labels: []]
   @counter [name: :missing_operator_count, help: "Missing Operators", labels: [:operator]]
+  @counter [name: :operator_response_count, help: "Operator Response Count", labels: [:operator]]
 
   def new_gas_price(gas_price) do
     Gauge.set(
@@ -18,6 +19,13 @@ defmodule TelemetryApi.PrometheusMetrics do
     )
   end
 
+  def operator_response(operator) do
+    Counter.inc(
+      name: :operator_response_count,
+      labels: [operator]
+    )
+  end
+
   def initialize_operator_metrics(operator) do
     value =
       Counter.value(
@@ -30,6 +38,11 @@ defmodule TelemetryApi.PrometheusMetrics do
         [name: :missing_operator_count, labels: [operator]],
         0
       )
+
+      Counter.inc(
+        [name: :operator_response_count, labels: [operator]],
+        0
+      )
     end
   end
 end

telemetry_api/lib/telemetry_api/traces.ex

@@ -91,6 +91,10 @@ defmodule TelemetryApi.Traces do
           current_stake: new_stake
       })
 
+      PrometheusMetrics.operator_response(
+        operator.name <> " - " <> String.slice(operator.address, 0..7)
+      )
+
       IO.inspect(
         "Operator response included. merkle_root: #{IO.inspect(merkle_root)} operator_id: #{IO.inspect(operator_id)}"
       )