Skip to content

Commit 735429c

Browse files
chliddlechliddle
authored
fix: axelar broadcaster metrics (#127)
1 parent 43a6da4 commit 735429c

File tree

8 files changed

+1323
-121
lines changed

8 files changed

+1323
-121
lines changed

‎.docker/grafana/dashboards/axelar-broadcaster.json‎

Lines changed: 705 additions & 6 deletions
Large diffs are not rendered by default.

‎src/blockchains/axelar/broadcaster.rs‎

Lines changed: 424 additions & 10 deletions
Large diffs are not rendered by default.

‎src/blockchains/axelar/metrics.rs‎

Lines changed: 32 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ lazy_static! {
1010
"rcosmos_axelar_evm_votes_yes",
1111
"Total number of EVM votes (Yes) cast by a validator"
1212
),
13-
&["validator_address", "chain_id", "network", "alerts", "sender_chain", "recipient_chain"]
13+
&["validator_address", "moniker", "chain_id", "network", "alerts", "sender_chain", "recipient_chain"]
1414
)
1515
.unwrap();
1616

@@ -20,7 +20,7 @@ lazy_static! {
2020
"rcosmos_axelar_evm_votes_no",
2121
"Total number of EVM votes (No) cast by a validator"
2222
),
23-
&["validator_address", "chain_id", "network", "alerts", "sender_chain", "recipient_chain"]
23+
&["validator_address", "moniker", "chain_id", "network", "alerts", "sender_chain", "recipient_chain"]
2424
)
2525
.unwrap();
2626

@@ -30,7 +30,7 @@ lazy_static! {
3030
"rcosmos_axelar_evm_votes_total",
3131
"Total number of EVM polls a validator participated in"
3232
),
33-
&["validator_address", "chain_id", "network", "alerts", "sender_chain", "recipient_chain"]
33+
&["validator_address", "moniker", "chain_id", "network", "alerts", "sender_chain", "recipient_chain"]
3434
)
3535
.unwrap();
3636

@@ -40,7 +40,7 @@ lazy_static! {
4040
"rcosmos_axelar_evm_votes_late",
4141
"Number of late EVM votes cast by a validator"
4242
),
43-
&["validator_address", "chain_id", "network", "alerts", "sender_chain", "recipient_chain"]
43+
&["validator_address", "moniker", "chain_id", "network", "alerts", "sender_chain", "recipient_chain"]
4444
)
4545
.unwrap();
4646

@@ -50,7 +50,7 @@ lazy_static! {
5050
"rcosmos_axelar_evm_votes_latest_height",
5151
"Latest poll height at which a validator cast a vote (baseline for tracking progress)"
5252
),
53-
&["validator_address", "chain_id", "network", "alerts"]
53+
&["validator_address", "moniker", "chain_id", "network", "alerts"]
5454
)
5555
.unwrap();
5656

@@ -73,6 +73,27 @@ lazy_static! {
7373
&["chain_id", "network"]
7474
)
7575
.unwrap();
76+
77+
/// Number of EVM votes missed by a validator (validator was expected to vote but didn't)
78+
pub static ref AXELAR_EVM_VOTES_MISSED: CounterVec = CounterVec::new(
79+
Opts::new(
80+
"rcosmos_axelar_evm_votes_missed",
81+
"Number of EVM votes missed by a validator (validator was in participants list but didn't cast a vote)"
82+
),
83+
&["validator_address", "moniker", "chain_id", "network", "alerts", "sender_chain", "recipient_chain"]
84+
)
85+
.unwrap();
86+
87+
/// Indicates that a validator's broadcaster_address is missing from the getValidators API response
88+
/// Set to 1 if broadcaster_address is missing, 0 otherwise
89+
pub static ref AXELAR_EVM_BROADCASTER_ADDRESS_UNKNOWN: IntGaugeVec = IntGaugeVec::new(
90+
Opts::new(
91+
"rcosmos_axelar_evm_broadcaster_address_unknown",
92+
"Indicates that a validator's broadcaster_address is missing from the getValidators API response (1 = missing, 0 = present)"
93+
),
94+
&["validator_address", "moniker", "chain_id", "network", "alerts"]
95+
)
96+
.unwrap();
7697
}
7798

7899
pub fn axelar_custom_metrics() {
@@ -97,4 +118,10 @@ pub fn axelar_custom_metrics() {
97118
REGISTRY
98119
.register(Box::new(AXELAR_EVM_VOTES_LATEST_HEIGHT.clone()))
99120
.unwrap();
121+
REGISTRY
122+
.register(Box::new(AXELAR_EVM_VOTES_MISSED.clone()))
123+
.unwrap();
124+
REGISTRY
125+
.register(Box::new(AXELAR_EVM_BROADCASTER_ADDRESS_UNKNOWN.clone()))
126+
.unwrap();
100127
}

‎src/blockchains/axelar/types.rs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,34 @@ pub struct Vote {
1818
pub height: u64,
1919
pub confirmed: Option<bool>,
2020
}
21+
22+
#[derive(Debug, Deserialize, Clone)]
23+
pub struct ValidatorDescription {
24+
pub moniker: String,
25+
#[serde(default)]
26+
#[allow(dead_code)]
27+
pub identity: String,
28+
#[serde(default)]
29+
#[allow(dead_code)]
30+
pub website: String,
31+
#[serde(default)]
32+
#[allow(dead_code)]
33+
pub security_contact: String,
34+
#[serde(default)]
35+
#[allow(dead_code)]
36+
pub details: String,
37+
}
38+
39+
#[derive(Debug, Deserialize, Clone)]
40+
pub struct AxelarscanValidator {
41+
pub operator_address: String,
42+
pub delegator_address: String,
43+
#[serde(default)]
44+
pub broadcaster_address: Option<String>,
45+
pub description: ValidatorDescription,
46+
}
47+
48+
#[derive(Debug, Deserialize, Clone)]
49+
pub struct GetValidatorsResponse {
50+
pub data: Vec<AxelarscanValidator>,
51+
}

‎src/blockchains/sei/block.rs‎

Lines changed: 51 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,11 @@ impl Block {
112112
height, page, PER_PAGE
113113
));
114114

115-
// Tx fetch: use endpoint preference to prioritize nodes with tx_search support
116-
// NodePool will try preferred nodes first, then fall back to all nodes if they fail
117-
// This ensures we get tx data when available, but don't block for long if preferred nodes are down
115+
// Tx fetch: use endpoint preference to prioritize nodes with tx_search support
116+
// NodePool will try preferred nodes first, then fall back to all nodes if they fail
117+
// This ensures we get tx data when available, but don't block for long if preferred nodes are down
118118
match rpc.get_with_endpoint_preference(tx_path.clone(), Some("tx_search")).await {
119-
Ok(res) => {
119+
Ok(res) => {
120120
// Try to parse the response
121121
match from_str::<crate::blockchains::sei::types::SeiTxResponse>(&res) {
122122
Ok(resp) => {
@@ -170,7 +170,7 @@ impl Block {
170170
Err(_) => {
171171
// Fallback to flexible JSON parsing for first page only
172172
if page == 1 {
173-
match serde_json::from_str::<serde_json::Value>(&res) {
173+
match serde_json::from_str::<serde_json::Value>(&res) {
174174
Ok(json) => {
175175
if let Some(txs_val) = extract_txs_from_response(&json) {
176176
match serde_json::from_value::<Vec<crate::blockchains::sei::types::SeiTx>>(
@@ -181,34 +181,34 @@ impl Block {
181181
// For fallback parsing, we can't determine total, so stop after first page
182182
break;
183183
}
184-
Err(e) => {
184+
Err(e) => {
185185
let preview = create_error_preview(&res, 200);
186-
warn!(
186+
warn!(
187187
"(Sei Block) Unable to parse tx response for height {} (page {}): {} (response length: {}, preview: {}). Continuing without txs.",
188-
height,
188+
height,
189189
page,
190-
e,
191-
res.len(),
192-
preview
193-
);
194-
return None;
195-
}
196-
}
190+
e,
191+
res.len(),
192+
preview
193+
);
194+
return None;
195+
}
196+
}
197197
} else {
198198
// No txs found - treat as empty
199199
return Some(Vec::new());
200200
}
201-
}
202-
Err(e) => {
201+
}
202+
Err(e) => {
203203
let preview = create_error_preview(&res, 200);
204-
warn!(
204+
warn!(
205205
"(Sei Block) Unable to parse tx response as JSON for height {} (page {}): {} (response length: {}, preview: {}). Continuing without txs.",
206-
height,
206+
height,
207207
page,
208-
e,
209-
res.len(),
210-
preview
211-
);
208+
e,
209+
res.len(),
210+
preview
211+
);
212212
return None;
213213
}
214214
}
@@ -221,17 +221,17 @@ impl Block {
221221
);
222222
break;
223223
}
224-
}
225224
}
226225
}
227-
Err(e) => {
226+
}
227+
Err(e) => {
228228
if page == 1 {
229229
// First page failed - return None to indicate tx_search unavailable
230-
warn!(
231-
"(Sei Block) Unable to fetch tx data for height {}: {}. Continuing without txs.",
232-
height,
233-
e
234-
);
230+
warn!(
231+
"(Sei Block) Unable to fetch tx data for height {}: {}. Continuing without txs.",
232+
height,
233+
e
234+
);
235235
return None;
236236
} else {
237237
// Subsequent page failed - we've likely reached the end or hit an error
@@ -404,11 +404,11 @@ impl Block {
404404

405405
// tx count - only update in catch-up mode if should_update_all_metrics
406406
if should_update_all_metrics {
407-
COMETBFT_BLOCK_TXS
408-
.with_label_values(&[
409-
&self.app_context.chain_id,
410-
&self.app_context.config.general.network,
411-
])
407+
COMETBFT_BLOCK_TXS
408+
.with_label_values(&[
409+
&self.app_context.chain_id,
410+
&self.app_context.config.general.network,
411+
])
412412
.set(tx_count as f64);
413413
}
414414

@@ -447,8 +447,8 @@ impl Block {
447447
txs_info.len()
448448
);
449449
}
450-
let mut gas_wanted = Vec::new();
451-
let mut gas_used = Vec::new();
450+
let mut gas_wanted = Vec::new();
451+
let mut gas_used = Vec::new();
452452

453453
for tx in txs_info.iter() {
454454
if let Some(result) = tx.tx_result.as_ref() {
@@ -461,13 +461,13 @@ impl Block {
461461
if let Ok(v) = gu.parse::<usize>() {
462462
gas_used.push(v);
463463
}
464+
}
464465
}
465466
}
466-
}
467467

468-
block_gas_wanted = gas_wanted.iter().sum::<usize>() as f64;
469-
block_gas_used = gas_used.iter().sum::<usize>() as f64;
470-
if !gas_wanted.is_empty() {
468+
block_gas_wanted = gas_wanted.iter().sum::<usize>() as f64;
469+
block_gas_used = gas_used.iter().sum::<usize>() as f64;
470+
if !gas_wanted.is_empty() {
471471
block_avg_tx_gas_wanted =
472472
gas_wanted.iter().sum::<usize>() as f64 / gas_wanted.len() as f64;
473473
block_avg_tx_gas_used =
@@ -885,20 +885,20 @@ impl Block {
885885
+ 1;
886886
if h <= 1 {
887887
let latest = last_block
888-
.header
889-
.height
890-
.parse::<usize>()
891-
.context("Could not parse last block height")?;
888+
.header
889+
.height
890+
.parse::<usize>()
891+
.context("Could not parse last block height")?;
892892
h = latest.saturating_sub(1);
893893
}
894894
h
895895
} else {
896896
let mut h = self
897-
.signature_storage
898-
.get_last_processed_height()
899-
.await?
900-
.unwrap_or(0)
901-
+ 1;
897+
.signature_storage
898+
.get_last_processed_height()
899+
.await?
900+
.unwrap_or(0)
901+
+ 1;
902902
if h <= 1 {
903903
let latest = last_block
904904
.header
@@ -1385,7 +1385,7 @@ impl Block {
13851385
if buffered_height != height_to_process {
13861386
error!(
13871387
"(Sei Block) CRITICAL: Block height mismatch in buffer remove: expected {}, got {}",
1388-
height_to_process,
1388+
height_to_process,
13891389
buffered_height
13901390
);
13911391
anyhow::bail!(

‎src/core/config.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -765,14 +765,27 @@ impl Default for AxelarBroadcasterConfig {
765765

766766
#[derive(Debug, Deserialize, Clone)]
767767
pub struct AxelarBroadcasterAlertingConfig {
768+
/// List of broadcaster/delegator addresses (axelar1...) to track.
769+
/// If you also know the operator address (axelarvaloper1...), you can optionally provide
770+
/// a mapping in `validators` for more reliable missed vote detection.
768771
#[serde(default)]
769772
pub addresses: Vec<String>,
773+
774+
/// Optional mapping of broadcaster/delegator addresses (axelar1...) to operator addresses (axelarvaloper1...).
775+
/// If provided, this is used directly for missed vote detection. If not provided, we attempt
776+
/// to resolve it via the LCD API, but this may fail if the broadcaster hasn't delegated.
777+
/// Example:
778+
/// validators:
779+
/// axelar1dexdcyf67247kz09vh4hu3phfep2yfut6p6zfk: axelarvaloper13s44uvtzf578zjze9eqeh0mnemj60pwn83frcp
780+
#[serde(default)]
781+
pub validators: std::collections::HashMap<String, String>,
770782
}
771783

772784
impl Default for AxelarBroadcasterAlertingConfig {
773785
fn default() -> Self {
774786
Self {
775787
addresses: Vec::new(),
788+
validators: std::collections::HashMap::new(),
776789
}
777790
}
778791
}

‎test/env/sei-testnet-node.yaml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,16 +42,18 @@ general:
4242

4343
# Node mode configuration
4444
# You must set the NODE_NAME environment variable to use this mode, in k8s this could be the pod name, it will be used to identify the node in the metrics.
45+
# NOTE: Intervals should be kept low (5-10s) for fast-producing chains to avoid false positive "blocks behind" alerts.
46+
# With a 30s interval on a 2bps chain, nodes can lag by up to 60 blocks, triggering alerts unnecessarily.
4547
node:
4648
client: p2p
4749
tendermint:
4850
nodeInfo: # Tendermint node info metrics, calls /cosmos/base/node/v1beta1/node_info LCD endpoint.
4951
enabled: true
50-
interval: 30
52+
interval: 5 # Reduced from 30s to minimize block height lag for accurate "blocks behind" alerts
5153
cometbft:
52-
status: # CometBFT status metrics, calls /status RPC endpoint.
54+
status: # CometBFT status metrics, calls /status RPC endpoint. Updates rcosmos_cometbft_node_latest_block_height.
5355
enabled: true
54-
interval: 30
56+
interval: 5 # Reduced from 30s to minimize block height lag for accurate "blocks behind" alerts
5557

5658
# Network mode configuration
5759
network:

0 commit comments

Comments
 (0)