(feat/malloc_size_of): inspect transition frontier and snark pool resources usage

Author: vlad9486

node/src/rpc/mod.rs

@@ -1,5 +1,5 @@
 mod rpc_state;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::str::FromStr;
 
 use ark_ff::fields::arithmetic::InvalidBigInt;
@@ -461,7 +461,29 @@ pub struct RpcNodeStatus {
     pub snark_pool: RpcNodeStatusSnarkPool,
     pub transaction_pool: RpcNodeStatusTransactionPool,
     pub current_block_production_attempt: Option<BlockProductionAttempt>,
+    pub resources_status: RpcNodeStatusResources,
+}
+
+#[derive(Serialize, Debug, Clone)]
+pub struct RpcNodeStatusResources {
     pub p2p_malloc_size: usize,
+    pub transition_frontier: RpcNodeStatusResourcesTransitionFrontier,
+    pub snark_pool: RpcNodeStatusResourcesSnarkPool,
+}
+
+#[derive(Serialize, Debug, Clone)]
+pub struct RpcNodeStatusResourcesTransitionFrontier {
+    pub best_chain_size: usize,
+    pub needed_protocol_states_size: usize,
+    pub blacklist_size: usize,
+    pub diff_tx_size: usize,
+}
+
+#[derive(Serialize, Debug, Clone)]
+pub struct RpcNodeStatusResourcesSnarkPool {
+    pub pool_size: usize,
+    pub candidates_size: usize,
+    pub candidates_inconsistency: BTreeSet<(PeerId, SnarkJobId)>,
 }
 
 #[derive(Serialize, Debug, Clone)]

node/src/rpc_effectful/rpc_effectful_effects.rs

@@ -14,13 +14,15 @@ use crate::{
         AccountQuery, AccountSlim, ActionStatsQuery, ActionStatsResponse, CurrentMessageProgress,
         MessagesStats, NodeHeartbeat, RootLedgerSyncProgress, RootStagedLedgerSyncProgress,
         RpcAction, RpcBlockProducerStats, RpcMessageProgressResponse, RpcNodeStatus,
-        RpcNodeStatusTransactionPool, RpcNodeStatusTransitionFrontier,
-        RpcNodeStatusTransitionFrontierBlockSummary, RpcNodeStatusTransitionFrontierSync,
-        RpcRequestExtraData, RpcScanStateSummary, RpcScanStateSummaryBlock,
-        RpcScanStateSummaryBlockTransaction, RpcScanStateSummaryBlockTransactionKind,
-        RpcScanStateSummaryScanStateJob, RpcSnarkPoolJobFull, RpcSnarkPoolJobSnarkWork,
-        RpcSnarkPoolJobSummary, RpcSnarkerJobCommitResponse, RpcSnarkerJobSpecResponse,
-        RpcTransactionInjectResponse, TransactionStatus,
+        RpcNodeStatusResources, RpcNodeStatusResourcesSnarkPool,
+        RpcNodeStatusResourcesTransitionFrontier, RpcNodeStatusTransactionPool,
+        RpcNodeStatusTransitionFrontier, RpcNodeStatusTransitionFrontierBlockSummary,
+        RpcNodeStatusTransitionFrontierSync, RpcRequestExtraData, RpcScanStateSummary,
+        RpcScanStateSummaryBlock, RpcScanStateSummaryBlockTransaction,
+        RpcScanStateSummaryBlockTransactionKind, RpcScanStateSummaryScanStateJob,
+        RpcSnarkPoolJobFull, RpcSnarkPoolJobSnarkWork, RpcSnarkPoolJobSummary,
+        RpcSnarkerJobCommitResponse, RpcSnarkerJobSpecResponse, RpcTransactionInjectResponse,
+        TransactionStatus,
     },
     snark_pool::SnarkPoolAction,
     transition_frontier::sync::{
@@ -810,11 +812,33 @@ fn compute_node_status<S: Service>(store: &mut Store<S>) -> RpcNodeStatus {
             transaction_candidates: state.transaction_pool.candidates.transactions_count(),
         },
         current_block_production_attempt,
-        p2p_malloc_size: {
-            let mut set = BTreeSet::new();
-            let fun = move |ptr: *const c_void| !set.insert(ptr.addr());
-            let mut ops = MallocSizeOfOps::new(None, Some(Box::new(fun)));
-            size_of_val(&state.p2p) + state.p2p.size_of(&mut ops)
+        resources_status: RpcNodeStatusResources {
+            p2p_malloc_size: {
+                let mut set = BTreeSet::new();
+                let fun = move |ptr: *const c_void| !set.insert(ptr.addr());
+                let mut ops = MallocSizeOfOps::new(None, Some(Box::new(fun)));
+                size_of_val(&state.p2p) + state.p2p.size_of(&mut ops)
+            },
+            transition_frontier: RpcNodeStatusResourcesTransitionFrontier {
+                best_chain_size: state.transition_frontier.best_chain.len(),
+                needed_protocol_states_size: state.transition_frontier.needed_protocol_states.len(),
+                blacklist_size: state.transition_frontier.blacklist.len(),
+                diff_tx_size: state
+                    .transition_frontier
+                    .chain_diff
+                    .as_ref()
+                    .map(|d| d.new_commands.len() + d.removed_commands.len())
+                    .unwrap_or_default(),
+            },
+            snark_pool: {
+                let (candidates_size, candidates_inconsistency) =
+                    state.snark_pool.candidates.check();
+                RpcNodeStatusResourcesSnarkPool {
+                    pool_size: state.snark_pool.pool.len(),
+                    candidates_size,
+                    candidates_inconsistency,
+                }
+            },
         },
     };
     status

node/src/snark_pool/candidate/snark_pool_candidate_state.rs

@@ -51,6 +51,24 @@ impl SnarkPoolCandidatesState {
         Self::default()
     }
 
+    pub fn check(&self) -> (usize, BTreeSet<(PeerId, SnarkJobId)>) {
+        let len = self.by_peer.iter().map(|(_, v)| v.len()).sum::<usize>();
+        let lhs = self
+            .by_job_id
+            .iter()
+            .map(|(job_id, v)| v.iter().map(|peer_id| (*peer_id, job_id.clone())))
+            .flatten()
+            .collect::<BTreeSet<_>>();
+        let rhs = self
+            .by_peer
+            .iter()
+            .map(|(peer_id, v)| v.keys().map(|job_id| (*peer_id, job_id.clone())))
+            .flatten()
+            .collect::<BTreeSet<_>>();
+        let inconsistency = lhs.symmetric_difference(&rhs).cloned().collect();
+        (len, inconsistency)
+    }
+
     pub fn peer_work_count(&self, peer_id: &PeerId) -> usize {
         self.by_peer.get(peer_id).map(|v| v.len()).unwrap_or(0)
     }

node/src/snark_pool/snark_pool_state.rs

@@ -15,7 +15,7 @@ use super::SnarkPoolConfig;
 #[derive(Serialize, Deserialize, Clone)]
 pub struct SnarkPoolState {
     config: SnarkPoolConfig,
-    pool: DistributedPool<JobState, SnarkJobId>,
+    pub pool: DistributedPool<JobState, SnarkJobId>,
     pub candidates: SnarkPoolCandidatesState,
     pub(super) last_check_timeouts: Timestamp,
 }