feat(meta-service): Add comprehensive snapshot database metrics (#18422)

Add detailed snapshot database metrics to monitor storage performance and statistics:
- Block count, data size, index size metrics
- Average block size and keys per block calculations
- Block read statistics (total, from cache, from disk)
- Upgrade rotbl dependency to 0.2.7 for enhanced access statistics

These metrics provide visibility into snapshot storage efficiency and cache performance.

All metrics are `gauge` type:

```config
# number of keys in the last snapshot.
metasrv_server_snapshot_key_count 0

# number of primary keys in the last snapshot.
metasrv_server_snapshot_primary_index_count 0

# number of expire index keys in the last snapshot.
metasrv_server_snapshot_expire_index_count 0

# number of blocks in the last snapshot.
metasrv_server_snapshot_block_count 0

# size of data section in the last snapshot.
metasrv_server_snapshot_data_size 0

# size of index section in the last snapshot.
metasrv_server_snapshot_index_size 0

# average size of a block in the last snapshot.
metasrv_server_snapshot_avg_block_size 0

# average number of keys per block in the last snapshot.
metasrv_server_snapshot_avg_keys_per_block 0

# total number of read block from cache or from disk.
metasrv_server_snapshot_read_block 0

# total number of read block from cache.
metasrv_server_snapshot_read_block_from_cache 0

# total number of read block from disk.
metasrv_server_snapshot_read_block_from_disk 0
```

Author: drmingdrmer

Cargo.lock

@@ -466,7 +466,7 @@ reqwest-hickory-resolver = "0.2"
 ringbuffer = "0.14.2"
 rmp-serde = "1.1.1"
 roaring = { version = "^0.10", features = ["serde"] }
-rotbl = { version = "0.2.6", features = [] }
+rotbl = { version = "0.2.7", features = [] }
 rust_decimal = "1.26"
 rustix = "0.38.37"
 rustls = { version = "0.23.27", features = ["ring", "tls12"], default-features = false }

Cargo.toml

@@ -577,7 +577,8 @@ impl MetaService for MetaServiceImpl {
                 wal_closed_chunk_sizes: status.raft_log.wal_closed_chunk_sizes,
             }),
 
-            snapshot_key_count: status.snapshot_key_count as u64,
+            snapshot_key_count: status.snapshot_key_count,
+
             state: status.state,
             is_leader: status.is_leader,
             current_term: status.current_term,

src/meta/service/src/api/grpc/grpc_service.rs

@@ -53,6 +53,7 @@ use databend_common_meta_types::raft_types::MembershipNode;
 use databend_common_meta_types::raft_types::NodeId;
 use databend_common_meta_types::raft_types::RaftMetrics;
 use databend_common_meta_types::raft_types::TypeConfig;
+use databend_common_meta_types::snapshot_db::DBStat;
 use databend_common_meta_types::AppliedState;
 use databend_common_meta_types::Cmd;
 use databend_common_meta_types::Endpoint;
@@ -487,6 +488,19 @@ impl MetaNode {
                     )
                 }
 
+                {
+                    let db_stat = meta_node.get_snapshot_db_stat().await;
+                    let snapshot = server_metrics::snapshot();
+                    snapshot.block_count.set(db_stat.block_num as i64);
+                    snapshot.data_size.set(db_stat.data_size as i64);
+                    snapshot.index_size.set(db_stat.index_size as i64);
+                    snapshot.avg_block_size.set(db_stat.avg_block_size as i64);
+                    snapshot
+                        .avg_keys_per_block
+                        .set(db_stat.avg_keys_per_block as i64);
+                    snapshot.read_block.set(db_stat.read_block as i64);
+                }
+
                 last_leader = mm.current_leader;
             }
 
@@ -898,6 +912,10 @@ impl MetaNode {
         self.raft_store.get_snapshot_key_space_stat().await
     }
 
+    async fn get_snapshot_db_stat(&self) -> DBStat {
+        self.raft_store.get_snapshot_db_stat().await
+    }
+
     pub async fn get_status(&self) -> Result<MetaNodeStatus, MetaError> {
         let voters = self
             .raft_store

src/meta/service/src/meta_service/meta_node.rs

@@ -47,6 +47,26 @@ pub mod server_metrics {
         };
     }
 
+    #[derive(Default, Debug, Clone)]
+    pub struct SnapshotStat {
+        /// The total number of blocks in the snapshot.
+        pub block_count: Gauge,
+        /// The total size in bytes of the block data section in the snapshot.
+        pub data_size: Gauge,
+        /// The total size in bytes of the block index section in the snapshot.
+        pub index_size: Gauge,
+        /// The average size in bytes of a block.
+        pub avg_block_size: Gauge,
+        /// The average number of keys per block.
+        pub avg_keys_per_block: Gauge,
+        /// The total number of read block from cache or from disk.
+        pub read_block: Gauge,
+        /// The total number of read block from cache.
+        pub read_block_from_cache: Gauge,
+        /// The total number of read block from disk.
+        pub read_block_from_disk: Gauge,
+    }
+
     struct ServerMetrics {
         current_leader_id: Gauge,
         is_leader: Gauge,
@@ -65,6 +85,8 @@ pub mod server_metrics {
         /// `snapshot_key_count = snapshot_primary_index_count + snapshot_expire_index_count`
         snapshot_expire_index_count: Gauge,
 
+        snapshot_stat: SnapshotStat,
+
         raft_log_cache_items: Gauge,
         raft_log_cache_used_size: Gauge,
         raft_log_wal_open_chunk_size: Gauge,
@@ -92,10 +114,13 @@ pub mod server_metrics {
                 node_is_health: Gauge::default(),
                 leader_changes: Counter::default(),
                 applying_snapshot: Gauge::default(),
+
                 snapshot_key_count: Gauge::default(),
                 snapshot_primary_index_count: Gauge::default(),
                 snapshot_expire_index_count: Gauge::default(),
 
+                snapshot_stat: Default::default(),
+
                 raft_log_cache_items: Gauge::default(),
                 raft_log_cache_used_size: Gauge::default(),
                 raft_log_wal_open_chunk_size: Gauge::default(),
@@ -151,6 +176,46 @@ pub mod server_metrics {
                 "number of expire index keys in the last snapshot",
                 metrics.snapshot_expire_index_count.clone(),
             );
+            registry.register(
+                key!("snapshot_block_count"),
+                "number of blocks in the last snapshot",
+                metrics.snapshot_stat.block_count.clone(),
+            );
+            registry.register(
+                key!("snapshot_data_size"),
+                "size of data section in the last snapshot",
+                metrics.snapshot_stat.data_size.clone(),
+            );
+            registry.register(
+                key!("snapshot_index_size"),
+                "size of index section in the last snapshot",
+                metrics.snapshot_stat.index_size.clone(),
+            );
+            registry.register(
+                key!("snapshot_avg_block_size"),
+                "average size of a block in the last snapshot",
+                metrics.snapshot_stat.avg_block_size.clone(),
+            );
+            registry.register(
+                key!("snapshot_avg_keys_per_block"),
+                "average number of keys per block in the last snapshot",
+                metrics.snapshot_stat.avg_keys_per_block.clone(),
+            );
+            registry.register(
+                key!("snapshot_read_block"),
+                "total number of read block from cache or from disk",
+                metrics.snapshot_stat.read_block.clone(),
+            );
+            registry.register(
+                key!("snapshot_read_block_from_cache"),
+                "total number of read block from cache",
+                metrics.snapshot_stat.read_block_from_cache.clone(),
+            );
+            registry.register(
+                key!("snapshot_read_block_from_disk"),
+                "total number of read block from disk",
+                metrics.snapshot_stat.read_block_from_disk.clone(),
+            );
 
             registry.register(
                 key!("raft_log_cache_items"),
@@ -258,6 +323,10 @@ pub mod server_metrics {
         SERVER_METRICS.snapshot_expire_index_count.set(n as i64);
     }
 
+    pub fn snapshot() -> &'static SnapshotStat {
+        &SERVER_METRICS.snapshot_stat
+    }
+
     pub fn set_raft_log_stat(st: RaftLogStat) {
         SERVER_METRICS
             .raft_log_cache_items

src/meta/service/src/metrics/meta_metrics.rs

@@ -43,6 +43,7 @@ use databend_common_meta_types::raft_types::NodeId;
 use databend_common_meta_types::raft_types::Snapshot;
 use databend_common_meta_types::raft_types::SnapshotMeta;
 use databend_common_meta_types::raft_types::StorageError;
+use databend_common_meta_types::snapshot_db::DBStat;
 use databend_common_meta_types::snapshot_db::DB;
 use databend_common_meta_types::Endpoint;
 use databend_common_meta_types::MetaNetworkError;
@@ -310,6 +311,15 @@ impl RaftStoreInner {
         db.sys_data().key_counts().clone()
     }
 
+    /// Get the statistics of the snapshot database.
+    pub(crate) async fn get_snapshot_db_stat(&self) -> DBStat {
+        let sm = self.state_machine.read().await;
+        let Some(db) = sm.levels().persisted() else {
+            return Default::default();
+        };
+        db.db_stat()
+    }
+
     /// Install a snapshot to build a state machine from it and replace the old state machine with the new one.
     #[fastrace::trace]
     pub async fn do_install_snapshot(&self, db: DB) -> Result<(), MetaStorageError> {

src/meta/service/src/store/store_inner.rs

@@ -109,62 +109,62 @@ async fn test_metrics() -> anyhow::Result<()> {
     // metasrv_meta_network_watch_initialization_total 0
     // metasrv_raft_network_active_peers{id="1",addr="127.0.0.1:29003"} 1
     // metasrv_raft_network_active_peers{id="2",addr="127.0.0.1:29006"} 1
-    // metasrv_raft_network_append_sent_seconds_bucket{le="+Inf",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="+Inf",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="+Inf",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.001",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.001",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.001",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.002",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.002",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.002",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.004",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.004",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.004",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.008",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.008",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.008",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.016",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.016",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.016",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.032",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.032",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.032",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.064",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.064",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.064",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.128",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.128",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.128",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.256",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.256",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.256",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.512",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.512",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.512",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="1.024",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="1.024",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="1.024",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="131.072",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="131.072",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="131.072",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="16.384",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="16.384",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="16.384",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="2.048",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="2.048",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="2.048",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="262.144",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="262.144",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="262.144",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="32.768",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="32.768",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="32.768",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="4.096",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="4.096",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="4.096",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="524.288",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="524.288",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="524.288",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="65.536",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="65.536",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="65.536",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="8.192",to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_bucket{le="8.192",to="1"} 9
     // metasrv_raft_network_append_sent_seconds_bucket{le="8.192",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_count{to="1"} 8
+    // metasrv_raft_network_append_sent_seconds_count{to="1"} 9
     // metasrv_raft_network_append_sent_seconds_count{to="2"} 6
     // metasrv_raft_network_append_sent_seconds_sum{to="1"} 0.0
     // metasrv_raft_network_append_sent_seconds_sum{to="2"} 0.0
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57382"} 1830
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57384"} 809
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57385"} 1764
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57387"} 537
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57388"} 338
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57389"} 533
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57390"} 673
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:62962"} 1794
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:62964"} 797
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:62965"} 1728
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:62967"} 537
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:62968"} 537
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:62969"} 533
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:62970"} 673
     // metasrv_raft_network_recv_bytes_total{from="addr"} 1
-    // metasrv_raft_network_sent_bytes_total{to="1"} 3650
-    // metasrv_raft_network_sent_bytes_total{to="2"} 2834
+    // metasrv_raft_network_sent_bytes_total{to="1"} 3661
+    // metasrv_raft_network_sent_bytes_total{to="2"} 2938
     // metasrv_raft_storage_raft_store_write_failed_total{func="fun"} 1
     // metasrv_raft_storage_snapshot_building 0
     // metasrv_raft_storage_snapshot_written_entries_total 0
@@ -180,16 +180,24 @@ async fn test_metrics() -> anyhow::Result<()> {
     // metasrv_server_proposals_failed_total 0
     // metasrv_server_proposals_pending 0
     // metasrv_server_raft_log_cache_items 10
-    // metasrv_server_raft_log_cache_used_size 867
-    // metasrv_server_raft_log_size 1712
+    // metasrv_server_raft_log_cache_used_size 771
+    // metasrv_server_raft_log_size 1694
     // metasrv_server_raft_log_wal_closed_chunk_count 0
     // metasrv_server_raft_log_wal_closed_chunk_total_size 0
-    // metasrv_server_raft_log_wal_offset 1712
-    // metasrv_server_raft_log_wal_open_chunk_size 1712
+    // metasrv_server_raft_log_wal_offset 1694
+    // metasrv_server_raft_log_wal_open_chunk_size 1694
     // metasrv_server_read_failed_total 0
+    // metasrv_server_snapshot_avg_block_size 0
+    // metasrv_server_snapshot_avg_keys_per_block 0
+    // metasrv_server_snapshot_block_count 0
+    // metasrv_server_snapshot_data_size 0
     // metasrv_server_snapshot_expire_index_count 0
+    // metasrv_server_snapshot_index_size 0
     // metasrv_server_snapshot_key_count 0
     // metasrv_server_snapshot_primary_index_count 0
+    // metasrv_server_snapshot_read_block 0
+    // metasrv_server_snapshot_read_block_from_cache 0
+    // metasrv_server_snapshot_read_block_from_disk 0
     // metasrv_server_watchers 0
 
     let b = response.take_body();
@@ -258,6 +266,16 @@ async fn test_metrics() -> anyhow::Result<()> {
     assert!(metric_keys.contains("metasrv_server_snapshot_primary_index_count"));
     assert!(metric_keys.contains("metasrv_server_snapshot_expire_index_count"));
 
+    // Server snapshot internal metrics
+    assert!(metric_keys.contains("metasrv_server_snapshot_block_count"));
+    assert!(metric_keys.contains("metasrv_server_snapshot_data_size"));
+    assert!(metric_keys.contains("metasrv_server_snapshot_index_size"));
+    assert!(metric_keys.contains("metasrv_server_snapshot_avg_block_size"));
+    assert!(metric_keys.contains("metasrv_server_snapshot_avg_keys_per_block"));
+    assert!(metric_keys.contains("metasrv_server_snapshot_read_block"));
+    assert!(metric_keys.contains("metasrv_server_snapshot_read_block_from_cache"));
+    assert!(metric_keys.contains("metasrv_server_snapshot_read_block_from_disk"));
+
     // Meta network metrics
     assert!(metric_keys.contains("metasrv_meta_network_recv_bytes_total"));
     assert!(metric_keys.contains("metasrv_meta_network_req_failed_total"));