feat(meta-service): add metrics for snapshot building (#18212)

Add two snapshot metrics:
- whether it is building snapshot,
- and the entries written to snapshot.

```
# 0 or 1
metasrv_raft_storage_snapshot_building                0

# total number of entries written to snapshot
metasrv_raft_storage_snapshot_written_entries_total   123
```

Author: drmingdrmer

src/meta/service/src/metrics/meta_metrics.rs

@@ -377,6 +377,7 @@ pub mod raft_metrics {
             sent_bytes: Family<ToLabels, Counter>,
             recv_bytes: Family<FromLabels, Counter>,
             sent_failures: Family<ToLabels, Counter>,
+
             append_sent_seconds: Family<ToLabels, Histogram>,
             snapshot_send_success: Family<ToLabels, Counter>,
             snapshot_send_failure: Family<ToLabels, Counter>,
@@ -601,6 +602,7 @@ pub mod raft_metrics {
         use prometheus_client::encoding::EncodeLabelSet;
         use prometheus_client::metrics::counter::Counter;
         use prometheus_client::metrics::family::Family;
+        use prometheus_client::metrics::gauge::Gauge;
 
         use crate::metrics::registry::load_global_registry;
 
@@ -618,13 +620,24 @@ pub mod raft_metrics {
         struct StorageMetrics {
             raft_store_write_failed: Family<FuncLabels, Counter>,
             raft_store_read_failed: Family<FuncLabels, Counter>,
+
+            /// The number of tasks that are building a snapshot.
+            ///
+            /// It should be 0 or 1.
+            snapshot_building: Gauge,
+
+            /// The number of entries written to the snapshot file.
+            snapshot_written_entries: Counter,
         }
 
         impl StorageMetrics {
             fn init() -> Self {
                 let metrics = Self {
                     raft_store_write_failed: Family::default(),
                     raft_store_read_failed: Family::default(),
+
+                    snapshot_building: Gauge::default(),
+                    snapshot_written_entries: Counter::default(),
                 };
 
                 let mut registry = load_global_registry();
@@ -638,6 +651,18 @@ pub mod raft_metrics {
                     "raft store read failed",
                     metrics.raft_store_read_failed.clone(),
                 );
+
+                registry.register(
+                    key!("snapshot_building"),
+                    "The number of tasks that are building a snapshot. It should be 0 or 1.",
+                    metrics.snapshot_building.clone(),
+                );
+                registry.register(
+                    key!("snapshot_written_entries"),
+                    "The number of entries written to the snapshot file.",
+                    metrics.snapshot_written_entries.clone(),
+                );
+
                 metrics
             }
         }
@@ -682,6 +707,14 @@ pub mod raft_metrics {
                     .inc();
             }
         }
+
+        pub fn incr_snapshot_building_by(cnt: i64) {
+            STORAGE_METRICS.snapshot_building.inc_by(cnt);
+        }
+
+        pub fn incr_snapshot_written_entries() {
+            STORAGE_METRICS.snapshot_written_entries.inc();
+        }
     }
 }
 
@@ -918,3 +951,12 @@ pub fn meta_metrics_to_prometheus_string() -> String {
     prometheus_encode(&mut text, &registry).unwrap();
     text
 }
+
+#[derive(Default)]
+pub(crate) struct SnapshotBuilding;
+
+impl count::Count for SnapshotBuilding {
+    fn incr_count(&mut self, n: i64) {
+        raft_metrics::storage::incr_snapshot_building_by(n);
+    }
+}

src/meta/service/src/metrics/mod.rs

@@ -21,3 +21,4 @@ pub use meta_metrics::raft_metrics;
 pub use meta_metrics::server_metrics;
 pub(crate) use meta_metrics::ProposalPending;
 pub(crate) use meta_metrics::RequestInFlight;
+pub(crate) use meta_metrics::SnapshotBuilding;

src/meta/service/src/store/store_inner.rs

@@ -48,13 +48,17 @@ use databend_common_meta_types::snapshot_db::DB;
 use databend_common_meta_types::Endpoint;
 use databend_common_meta_types::MetaNetworkError;
 use databend_common_meta_types::MetaStartupError;
+use databend_common_metrics::count::Count;
 use futures::TryStreamExt;
 use log::debug;
 use log::error;
 use log::info;
 use raft_log::api::raft_log_writer::RaftLogWriter;
 use tokio::time::sleep;
 
+use crate::metrics::raft_metrics;
+use crate::metrics::SnapshotBuilding;
+
 /// This is the inner store that implements the raft log storage API.
 pub struct RaftStoreInner {
     /// The ID of the Raft node for which this storage instances is configured.
@@ -187,6 +191,8 @@ impl RaftStoreInner {
 
         info!(id = self.id; "do_build_snapshot start");
 
+        let _guard = SnapshotBuilding::guard();
+
         let mut compactor = {
             let mut w = self.state_machine.write().await;
             w.freeze_writable();
@@ -238,6 +244,8 @@ impl RaftStoreInner {
                 tx.send(WriteEntry::Data(ent))
                     .await
                     .map_err(|e| StorageError::write_snapshot(Some(signature.clone()), &e))?;
+
+                raft_metrics::storage::incr_snapshot_written_entries();
             }
 
             tx.send(WriteEntry::Finish(sys_data))

src/meta/service/tests/it/api/http/metrics.rs

@@ -102,65 +102,72 @@ async fn test_metrics() -> anyhow::Result<()> {
     // metasrv_meta_network_rpc_delay_seconds_count 0
     // metasrv_meta_network_rpc_delay_seconds_sum 0.0
     // metasrv_meta_network_sent_bytes_total 0
+    // metasrv_meta_network_stream_get_item_sent_total 0
+    // metasrv_meta_network_stream_list_item_sent_total 0
+    // metasrv_meta_network_stream_mget_item_sent_total 0
+    // metasrv_meta_network_watch_change_total 0
+    // metasrv_meta_network_watch_initialization_total 0
     // metasrv_raft_network_active_peers{id="1",addr="127.0.0.1:29003"} 1
     // metasrv_raft_network_active_peers{id="2",addr="127.0.0.1:29006"} 1
-    // metasrv_raft_network_append_sent_seconds_bucket{le="+Inf",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="+Inf",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="+Inf",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.001",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.001",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.001",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.002",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.002",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.002",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.004",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.004",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.004",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.008",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.008",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.008",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.016",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.016",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.016",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.032",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.032",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.032",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.064",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.064",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.064",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.128",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.128",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.128",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.256",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.256",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.256",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="0.512",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="0.512",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="0.512",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="1.024",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="1.024",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="1.024",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="131.072",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="131.072",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="131.072",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="16.384",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="16.384",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="16.384",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="2.048",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="2.048",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="2.048",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="262.144",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="262.144",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="262.144",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="32.768",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="32.768",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="32.768",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="4.096",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="4.096",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="4.096",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="524.288",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="524.288",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="524.288",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="65.536",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="65.536",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="65.536",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_bucket{le="8.192",to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_bucket{le="8.192",to="1"} 8
     // metasrv_raft_network_append_sent_seconds_bucket{le="8.192",to="2"} 6
-    // metasrv_raft_network_append_sent_seconds_count{to="1"} 9
+    // metasrv_raft_network_append_sent_seconds_count{to="1"} 8
     // metasrv_raft_network_append_sent_seconds_count{to="2"} 6
     // metasrv_raft_network_append_sent_seconds_sum{to="1"} 0.0
     // metasrv_raft_network_append_sent_seconds_sum{to="2"} 0.0
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:59841"} 1830
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:59843"} 1764
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:59844"} 809
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:59846"} 537
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:59847"} 537
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:59848"} 533
-    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:59849"} 673
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57382"} 1830
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57384"} 809
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57385"} 1764
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57387"} 537
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57388"} 338
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57389"} 533
+    // metasrv_raft_network_recv_bytes_total{from="127.0.0.1:57390"} 673
     // metasrv_raft_network_recv_bytes_total{from="addr"} 1
-    // metasrv_raft_network_sent_bytes_total{to="1"} 3709
-    // metasrv_raft_network_sent_bytes_total{to="2"} 2974
+    // metasrv_raft_network_sent_bytes_total{to="1"} 3650
+    // metasrv_raft_network_sent_bytes_total{to="2"} 2834
     // metasrv_raft_storage_raft_store_write_failed_total{func="fun"} 1
+    // metasrv_raft_storage_snapshot_building 0
+    // metasrv_raft_storage_snapshot_written_entries_total 0
     // metasrv_server_applying_snapshot 0
     // metasrv_server_current_leader_id 0
     // metasrv_server_current_term 1
@@ -280,6 +287,8 @@ async fn test_metrics() -> anyhow::Result<()> {
 
     // Raft storage metrics
     assert!(metric_keys.contains("metasrv_raft_storage_raft_store_write_failed_total"));
+    assert!(metric_keys.contains("metasrv_raft_storage_snapshot_building"));
+    assert!(metric_keys.contains("metasrv_raft_storage_snapshot_written_entries_total"));
 
     // Watch
     assert!(metric_keys.contains("metasrv_meta_network_watch_initialization_total"));