refactor(meta-service): optimize state machine block loading with early cache checks (#18541)

Restructure load_block method to check cache before acquiring locks,
reducing contention and improving concurrent access performance.

- Add early cache check before lock acquisition
- Reduce lock scope to only critical sections

These changes improve cache hit performance and reduce lock contention
in multi-threaded scenarios.

Other changes:

- Refine state-machine locking log

Author: drmingdrmer

Cargo.lock

@@ -467,7 +467,7 @@ reqwest-hickory-resolver = "0.2"
 ringbuffer = "0.14.2"
 rmp-serde = "1.1.1"
 roaring = { version = "^0.10", features = ["serde"] }
-rotbl = { version = "0.2.7", features = [] }
+rotbl = { version = "0.2.8", features = [] }
 rust_decimal = "1.26"
 rustix = "0.38.37"
 rustls = { version = "0.23.27", features = ["ring", "tls12"], default-features = false }

Cargo.toml

@@ -19,7 +19,6 @@ use databend_common_meta_types::snapshot_db::DB;
 use databend_common_meta_types::sys_data::SysData;
 use futures::Stream;
 use futures_util::TryStreamExt;
-use log::debug;
 use log::info;
 use rotbl::v001::SeqMarked;
 use tokio::sync::mpsc;
@@ -93,7 +92,7 @@ impl WriterV003 {
         mut kv_rx: mpsc::Receiver<WriteEntry<(String, SeqMarked), (MetaSnapshotId, SysData)>>,
     ) -> Result<DB, io::Error> {
         while let Some(ent) = kv_rx.blocking_recv() {
-            debug!(entry :? =(&ent); "write kv");
+            // debug!(entry :? =(&ent); "write kv");
 
             let (k, v) = match ent {
                 WriteEntry::Data(ent) => ent,

src/meta/raft-store/src/sm_v003/writer_v003.rs

@@ -58,8 +58,8 @@ where T: fmt::Display
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(
             f,
-            "StateMachineLock-Write-Guard({}): {}",
-            self.purpose, self.inner
+            "StateMachineLock-Write-Guard: {} {}",
+            self.inner, self.purpose,
         )
     }
 }
@@ -68,8 +68,8 @@ impl<T: ?Sized> Drop for WriteGuard<'_, T> {
     fn drop(&mut self) {
         let elapsed = self.acquired.elapsed();
         info!(
-            "StateMachineLock-Write-Guard({}) released after {:?}",
-            self.purpose, elapsed
+            "StateMachineLock-Write-Release: total: {:?}; {}",
+            elapsed, self.purpose,
         );
     }
 }
@@ -105,8 +105,8 @@ where T: fmt::Display
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(
             f,
-            "StateMachineLock-Read-Guard({}): {}",
-            self.purpose, self.inner
+            "StateMachineLock-Read-Guard: {} {}",
+            self.inner, self.purpose
         )
     }
 }
@@ -115,8 +115,8 @@ impl<T: ?Sized> Drop for ReadGuard<'_, T> {
     fn drop(&mut self) {
         let elapsed = self.acquired.elapsed();
         info!(
-            "StateMachineLock-Read-Guard({}) released after {:?}",
-            self.purpose, elapsed
+            "StateMachineLock-Read-Release: total: {:?}; {}",
+            elapsed, self.purpose,
         );
     }
 }

src/meta/service/src/store/lock_guards.rs

@@ -190,7 +190,7 @@ impl RaftStoreInner {
             .read()
             .with_timing(|_output, total, busy| {
                 info!(
-                    "StateMachineLock-Read-Acquire-Elapsed: total: {:?}, busy: {:?}; {}",
+                    "StateMachineLock-Read-Acquire: total: {:?}, busy: {:?}; {}",
                     total, busy, &for_what
                 );
             })
@@ -210,7 +210,7 @@ impl RaftStoreInner {
             .write()
             .with_timing(|_output, total, busy| {
                 info!(
-                    "StateMachineLock-Write-Acquire-Elapsed: total: {:?}, busy: {:?}; {}",
+                    "StateMachineLock-Write-Acquire: total: {:?}, busy: {:?}; {}",
                     total, busy, &for_what
                 );
             })