feat: update metrics tracking for price updates and errors, enhancing Grafana dashboard descriptions

cctdaniel · cctdaniel · commit 48bfa09f52c4 · 2025-03-19T17:44:53.000+09:00
diff --git a/apps/price_pusher/README.md b/apps/price_pusher/README.md
@@ -383,11 +383,11 @@ The docker-compose setup includes a pre-configured Grafana dashboard (`grafana-d
 - **Active Price Feeds**: Displays the number of price feeds currently being actively monitored.
 - **Time Since Last Update**: Shows how long it's been since the last successful price update was published on-chain.
 - **Price Feeds List**: A table listing all configured price feeds with their details.
-- **Price Updates (Last Hour)**: Graph showing the number of price updates over the last hour with timeline.
+- **Successful Updates (Current Range)**: Graph showing the number of successful price updates over the current range with timeline.
 - **Update Conditions Distribution**: Pie chart showing the distribution of update conditions (YES/NO/EARLY) over the selected time range.
 - **Wallet Balance**: Current balance of your wallet in native token units.
 - **Wallet Balance Over Time**: Graph tracking your wallet balance over time to monitor consumption.
-- **Update Errors**: Tracks errors encountered during price update operations.
+- **Failed Updates (Current Range)**: Graph showing the number of failed price updates over the current range with timeline.
 
 When you first start the monitoring stack, the dashboard may show "No data" in the panels until the price pusher has been running for some time and has collected sufficient metrics.
 
diff --git a/apps/price_pusher/grafana-dashboard.sample.json b/apps/price_pusher/grafana-dashboard.sample.json
@@ -355,7 +355,7 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "pyth_price_updates_total",
+          "expr": "pyth_price_update_attempts_total{status=\"success\"}",
           "format": "table",
           "instant": true,
           "legendFormat": "__auto",
@@ -496,7 +496,23 @@
           },
           "unit": "none"
         },
-        "overrides": []
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Updates"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "green",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          }
+        ]
       },
       "gridPos": {
         "h": 8,
@@ -526,13 +542,13 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(increase(pyth_price_updates_total[$__range]))",
+          "expr": "sum(increase(pyth_price_update_attempts_total{status=\"success\"}[$__range]))",
           "legendFormat": "Updates",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Price Updates (Current Range)",
+      "title": "Successful Updates (Current Range)",
       "type": "timeseries"
     },
     {
@@ -645,9 +661,9 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "sum by (condition) (increase(pyth_update_conditions_total[$__range]))",
+          "expr": "sum by (trigger) (increase(pyth_price_update_attempts_total[$__range]))",
           "instant": false,
-          "legendFormat": "{{condition}}",
+          "legendFormat": "{{trigger}}",
           "range": true,
           "refId": "A"
         }
@@ -909,13 +925,13 @@
             "uid": "prometheus"
           },
           "editorMode": "code",
-          "expr": "sum(increase(pyth_price_update_errors_total[$__range]))",
+          "expr": "sum(increase(pyth_price_update_attempts_total{status=\"error\"}[$__range]))",
           "legendFormat": "Errors",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Update Errors",
+      "title": "Failed Updates (Current Range)",
       "type": "timeseries"
     }
   ],
diff --git a/apps/price_pusher/src/controller.ts b/apps/price_pusher/src/controller.ts
@@ -109,7 +109,21 @@ export class Controller {
           // Record successful updates
           if (this.metrics) {
             for (const config of pricesToPush) {
-              this.metrics.recordPriceUpdate(config.id, config.alias);
+              const triggerValue =
+                shouldUpdate(
+                  config,
+                  this.sourcePriceListener.getLatestPriceInfo(config.id),
+                  this.targetPriceListener.getLatestPriceInfo(config.id),
+                  this.logger,
+                ) === UpdateCondition.YES
+                  ? "yes"
+                  : "early";
+
+              this.metrics.recordPriceUpdate(
+                config.id,
+                config.alias,
+                triggerValue,
+              );
             }
           }
         } catch (error) {
@@ -121,7 +135,21 @@ export class Controller {
           // Record errors in metrics
           if (this.metrics) {
             for (const config of pricesToPush) {
-              this.metrics.recordPriceUpdateError(config.id, config.alias);
+              const triggerValue =
+                shouldUpdate(
+                  config,
+                  this.sourcePriceListener.getLatestPriceInfo(config.id),
+                  this.targetPriceListener.getLatestPriceInfo(config.id),
+                  this.logger,
+                ) === UpdateCondition.YES
+                  ? "yes"
+                  : "early";
+
+              this.metrics.recordPriceUpdateError(
+                config.id,
+                config.alias,
+                triggerValue,
+              );
             }
           }
         }
diff --git a/apps/price_pusher/src/metrics.ts b/apps/price_pusher/src/metrics.ts
@@ -12,10 +12,8 @@ export class PricePusherMetrics {
 
   // Metrics for price feed updates
   public lastPublishedTime: Gauge<string>;
-  public priceUpdatesTotal: Counter<string>;
+  public priceUpdateAttempts: Counter<string>;
   public priceFeedsTotal: Gauge<string>;
-  public priceUpdateErrors: Counter<string>;
-  public updateConditionsTotal: Counter<string>;
   // Wallet metrics
   public walletBalance: Gauge<string>;
 
@@ -35,10 +33,10 @@ export class PricePusherMetrics {
       registers: [this.registry],
     });
 
-    this.priceUpdatesTotal = new Counter({
-      name: "pyth_price_updates_total",
-      help: "Total number of price updates pushed to the chain",
-      labelNames: ["price_id", "alias"],
+    this.priceUpdateAttempts = new Counter({
+      name: "pyth_price_update_attempts_total",
+      help: "Total number of price update attempts with their trigger condition and status",
+      labelNames: ["price_id", "alias", "trigger", "status"],
       registers: [this.registry],
     });
 
@@ -48,20 +46,6 @@ export class PricePusherMetrics {
       registers: [this.registry],
     });
 
-    this.priceUpdateErrors = new Counter({
-      name: "pyth_price_update_errors_total",
-      help: "Total number of errors encountered during price updates",
-      labelNames: ["price_id", "alias"],
-      registers: [this.registry],
-    });
-
-    this.updateConditionsTotal = new Counter({
-      name: "pyth_update_conditions_total",
-      help: "Total number of price update condition checks by status (YES/NO/EARLY)",
-      labelNames: ["price_id", "alias", "condition"],
-      registers: [this.registry],
-    });
-
     // Wallet balance metric
     this.walletBalance = new Gauge({
       name: "pyth_wallet_balance",
@@ -97,8 +81,17 @@ export class PricePusherMetrics {
   }
 
   // Record a successful price update
-  public recordPriceUpdate(priceId: string, alias: string): void {
-    this.priceUpdatesTotal.inc({ price_id: priceId, alias });
+  public recordPriceUpdate(
+    priceId: string,
+    alias: string,
+    trigger: string = "yes",
+  ): void {
+    this.priceUpdateAttempts.inc({
+      price_id: priceId,
+      alias,
+      trigger: trigger.toLowerCase(),
+      status: "success",
+    });
   }
 
   // Record update condition status (YES/NO/EARLY)
@@ -107,19 +100,31 @@ export class PricePusherMetrics {
     alias: string,
     condition: UpdateCondition,
   ): void {
-    const conditionLabel = UpdateCondition[condition];
-    this.updateConditionsTotal.inc({
-      price_id: priceId,
-      alias,
-      condition: conditionLabel,
-    });
+    const triggerLabel = UpdateCondition[condition].toLowerCase();
+    // Only record as 'skipped' when the condition is NO
+    if (condition === UpdateCondition.NO) {
+      this.priceUpdateAttempts.inc({
+        price_id: priceId,
+        alias,
+        trigger: triggerLabel,
+        status: "skipped",
+      });
+    }
+    // YES and EARLY don't increment the counter here - they'll be counted
+    // when recordPriceUpdate or recordPriceUpdateError is called
   }
 
   // Record a price update error
-  public recordPriceUpdateError(priceId: string, alias: string): void {
-    this.priceUpdateErrors.inc({
+  public recordPriceUpdateError(
+    priceId: string,
+    alias: string,
+    trigger: string = "yes",
+  ): void {
+    this.priceUpdateAttempts.inc({
       price_id: priceId,
       alias,
+      trigger: trigger.toLowerCase(),
+      status: "error",
     });
   }
 
