Alerts for Merkle Root Upload and Claims (#176)

Added a new "epoch_percentage" field ([already used
elsewhere](https://github.com/jito-foundation/jito-tip-router/blob/master/cli/src/keeper/keeper_metrics.rs#L122)),
in creating new alerts for Merkle Root Uploads and MEV/PF Claims.

Three new alerts:

1) "Tip Router -- Merkle Root Upload Timer"
https://jitolabs.grafana.net/alerting/grafana/beub9k1y9dfcwd/view
Alerts if the number of failed MR uploads has not reached zero after 10%
of the epoch.

2) "Tip Router -- Claims Timer"
https://jitolabs.grafana.net/alerting/grafana/beub9k1y9dfcwd/view
Alerts if the number of remaining claims has not reached zero after 33%
of the epoch.

3) "Tip Router -- Claims Have Stalled"
https://jitolabs.grafana.net/alerting/grafana/fetnet1krxatcf/view
Checks the num remaining claims now vs an hour ago, and alerts if those
values are the same.

Author: mrmizz

tip-router-operator-cli/src/claim.rs

@@ -39,7 +39,7 @@ use tokio::io::BufReader;
 use tokio::sync::Mutex;
 
 use crate::{
-    merkle_tree_collection_file_name, priority_fees,
+    get_epoch_percentage, merkle_tree_collection_file_name, priority_fees,
     rpc_utils::{get_batched_accounts, send_until_blockhash_expires},
     Cli,
 };
@@ -120,12 +120,20 @@ pub async fn emit_claim_mev_tips_metrics(
     .await?;
 
     if validators_processed {
-        datapoint_info!(
-            "tip_router_cli.claim_mev_tips-send_summary",
-            ("claim_transactions_left", claims_to_process.len(), i64),
-            ("epoch", epoch, i64),
-            "cluster" => &cli.cluster,
-        );
+        match get_epoch_percentage(&rpc_client).await {
+            Ok(epoch_percentage) => {
+                datapoint_info!(
+                    "tip_router_cli.claim_mev_tips-send_summary",
+                    ("claim_transactions_left", claims_to_process.len(), i64),
+                    ("epoch", epoch, i64),
+                    ("epoch_percentage", epoch_percentage, f64),
+                    "cluster" => &cli.cluster,
+                );
+            }
+            Err(e) => {
+                warn!("Failed to fetch epoch percentage for claims: {:?}", e);
+            }
+        }
     }
 
     if validators_processed && claims_to_process.is_empty() {
@@ -326,13 +334,21 @@ pub async fn claim_mev_tips(
             .await?;
 
         if validators_processed {
-            datapoint_info!(
-                "tip_router_cli.claim_mev_tips-send_summary",
-                ("claim_transactions_left", claims_to_process.len(), i64),
-                ("epoch", epoch, i64),
-                ("operator", operator_address, String),
-                "cluster" => cluster,
-            );
+            match get_epoch_percentage(&rpc_client).await {
+                Ok(epoch_percentage) => {
+                    datapoint_info!(
+                        "tip_router_cli.claim_mev_tips-send_summary",
+                        ("claim_transactions_left", claims_to_process.len(), i64),
+                        ("epoch", epoch, i64),
+                        ("operator", operator_address, String),
+                        ("epoch_percentage", epoch_percentage, f64),
+                        "cluster" => cluster,
+                    );
+                }
+                Err(e) => {
+                    warn!("Failed to fetch epoch percentage for claims: {:?}", e);
+                }
+            }
         }
 
         if validators_processed && claims_to_process.is_empty() {

tip-router-operator-cli/src/lib.rs

@@ -38,8 +38,10 @@ use meta_merkle_tree::generated_merkle_tree::StakeMetaCollection;
 use meta_merkle_tree::{
     generated_merkle_tree::GeneratedMerkleTreeCollection, meta_merkle_tree::MetaMerkleTree,
 };
+use solana_client::nonblocking::rpc_client::RpcClient;
 use solana_metrics::{datapoint_error, datapoint_info};
 use solana_runtime::bank::Bank;
+use solana_sdk::clock::DEFAULT_SLOTS_PER_EPOCH;
 use solana_sdk::pubkey::Pubkey;
 use stake_meta_generator::generate_stake_meta_collection;
 
@@ -630,3 +632,10 @@ pub fn cleanup_tmp_files(snapshot_output_dir: &Path) -> std::result::Result<(),
 
     Ok(())
 }
+
+pub async fn get_epoch_percentage(client: &RpcClient) -> anyhow::Result<f64> {
+    let current_slot = client.get_slot().await? as f64;
+    let epoch_percentage =
+        (current_slot % DEFAULT_SLOTS_PER_EPOCH as f64) / DEFAULT_SLOTS_PER_EPOCH as f64;
+    Ok(epoch_percentage)
+}

tip-router-operator-cli/src/submit.rs

@@ -6,7 +6,7 @@ use jito_bytemuck::AccountDeserialize as JitoAccountDeserialize;
 use jito_priority_fee_distribution_sdk::PriorityFeeDistributionAccount;
 use jito_tip_distribution_sdk::TipDistributionAccount;
 use jito_tip_router_core::{ballot_box::BallotBox, config::Config};
-use log::{debug, error, info};
+use log::{debug, error, info, warn};
 use meta_merkle_tree::meta_merkle_tree::MetaMerkleTree;
 use solana_account_decoder::UiAccountEncoding;
 use solana_client::nonblocking::rpc_client::RpcClient as AsyncRpcClient;
@@ -18,7 +18,7 @@ use solana_metrics::{datapoint_error, datapoint_info};
 use solana_sdk::{pubkey::Pubkey, signature::Keypair};
 
 use crate::tip_router::send_set_merkle_root_txs;
-use crate::{meta_merkle_tree_file_name, Version};
+use crate::{get_epoch_percentage, meta_merkle_tree_file_name, Version};
 use crate::{
     tip_router::{
         cast_vote, get_ncn_config, set_merkle_root_instructions,
@@ -261,19 +261,29 @@ pub async fn submit_to_ncn(
             Ok(res) => {
                 let num_success = res.iter().filter(|r| r.is_ok()).count();
                 let num_failed = res.iter().filter(|r| r.is_err()).count();
-
-                datapoint_info!(
-                    "tip_router_cli.set_merkle_root",
-                    ("operator_address", operator_address.to_string(), String),
-                    ("epoch", tip_router_target_epoch, i64),
-                    ("num_success", num_success, i64),
-                    ("num_failed", num_failed, i64),
-                    "cluster" => cluster,
-                );
-                info!(
-                    "Set merkle root for {} tip distribution accounts, failed for {}",
-                    num_success, num_failed
-                );
+                match get_epoch_percentage(client).await {
+                    Ok(epoch_percentage) => {
+                        datapoint_info!(
+                            "tip_router_cli.set_merkle_root",
+                            ("operator_address", operator_address.to_string(), String),
+                            ("epoch", tip_router_target_epoch, i64),
+                            ("num_success", num_success, i64),
+                            ("num_failed", num_failed, i64),
+                            ("epoch_percentage", epoch_percentage, f64),
+                            "cluster" => cluster,
+                        );
+                        info!(
+                            "Set merkle root for {} tip distribution accounts, failed for {}",
+                            num_success, num_failed
+                        );
+                    }
+                    Err(e) => {
+                        warn!(
+                            "Failed to fetch epoch percentage for set merkle root: {:?}",
+                            e
+                        );
+                    }
+                }
             }
             Err(e) => {
                 datapoint_error!(