Merge #1948

1948: adding in memory structure to store pool information r=abhilashshetty04 a=abhilashshetty04



Co-authored-by: Abhilash Shetty <abhilash.shetty@datacore.com>

Author: mayastor-bors

io-engine/src/grpc/v1/pool.rs

@@ -11,6 +11,7 @@ use crate::{
         self, FindPoolArgs, IPoolFactory, ListPoolArgs, PoolArgs, PoolBackend, PoolFactory,
         PoolOps, ReplicaArgs,
     },
+    pool_information::pool_info_read,
 };
 use ::function_name::named;
 use futures::FutureExt;
@@ -461,13 +462,25 @@ impl Deref for PoolErrorsNt {
     }
 }
 impl PoolErrorsNt {
-    fn new(environ: &MayastorEnvironment, stats: &BdevErrorStats) -> Self {
+    fn new(pool: &str, environ: &MayastorEnvironment, stats: &BdevErrorStats) -> Self {
+        let mut io_stalled = false;
+        let mut io_stall_transition_count: u64 = 0;
+
+        let cache = pool_info_read();
+
+        if let Some(pool_lock) = cache.get(pool) {
+            let mut pool_mut = pool_lock.write();
+            pool_mut.update_transition_timestamp(*environ.pool_args.io_stall_transition_window);
+            io_stalled = pool_mut.io_stalled;
+            io_stall_transition_count = pool_mut.transition_timestamps.len() as u64;
+        }
+
         Self(PoolErrors {
             alerts: None,
             io_error_count: stats.error_count(),
             io_error_threshold: environ.pool_args.io_error_threshold,
-            io_stalled: false,
-            io_stall_transition_count: 0,
+            io_stalled,
+            io_stall_transition_count,
             io_stall_transition_threshold: environ.pool_args.io_stall_transition_threshold,
         })
     }
@@ -488,8 +501,9 @@ impl PoolErrorsNt {
 
         // todo: add sliding window parameters
         match self.io_stall_transition_count {
-            0 => {}
-            errors if errors < self.io_stall_transition_threshold => {
+            // todo: Notice should be raised on 1 transition.
+            0 | 1 => {}
+            num_stalls if num_stalls < self.io_stall_transition_threshold => {
                 self.set_alert(PoolAlertStatus::Attention, PoolAlert::IoStallIntermittent)
             }
             _ => self.set_alert(PoolAlertStatus::Warning, PoolAlert::IoStallIntermittentExc),
@@ -576,7 +590,7 @@ impl AsyncFrom<&dyn PoolOps> for Pool {
 
         let errors = stats.as_ref().map(|stats| {
             let environ = MayastorEnvironment::global();
-            PoolErrorsNt::new(&environ, stats)
+            PoolErrorsNt::new(value.name(), &environ, stats)
                 .with_io_errors()
                 .with_io_stall()
         });

io-engine/src/lib.rs

@@ -31,6 +31,7 @@ pub mod lvm;
 pub mod lvs;
 pub mod persistent_store;
 pub mod pool_backend;
+pub mod pool_information;
 pub mod prctl;
 pub mod rebuild;
 pub mod replica_backend;

io-engine/src/lvs/lvs_store.rs

@@ -1,4 +1,5 @@
 use core::f64;
+use parking_lot::RwLock;
 use std::{convert::TryFrom, fmt::Debug, os::raw::c_void, pin::Pin, ptr::NonNull, str::FromStr};
 
 use crate::sleep::mayastor_sleep;
@@ -39,6 +40,7 @@ use crate::{
         LvolSnapshotDescriptor,
     },
     pool_backend::{PoolArgs, ReplicaArgs},
+    pool_information::{pool_info_write, PoolInfo},
 };
 
 static ROUND_TO_MB: u32 = 1024 * 1024;
@@ -223,6 +225,18 @@ impl Lvs {
         uuid::Uuid::from_bytes(t).to_string()
     }
 
+    /// Adds pool information to in memory cache.
+    pub fn add_info(&self) {
+        let mut cache = pool_info_write();
+        cache.insert(self.name().to_string(), RwLock::new(PoolInfo::default()));
+    }
+
+    /// Removes pool information from in memory cache.
+    pub fn remove_info(pool: &str) {
+        let mut cache = pool_info_write();
+        cache.remove(pool);
+    }
+
     // checks for the disks length and parses to correct format
     pub fn parse_disk(disks: Vec<String>) -> Result<String, LvsError> {
         let disk = match disks.first() {
@@ -314,6 +328,7 @@ impl Lvs {
         } else {
             lvs.share_all().await;
             info!("{:?}: existing lvs imported successfully", lvs);
+            lvs.add_info();
             Ok(lvs)
         }
     }
@@ -556,6 +571,7 @@ impl Lvs {
         match Self::lookup(&args.name) {
             Some(pool) => {
                 info!("{:?}: new lvs created successfully", pool);
+                pool.add_info();
                 Ok(pool)
             }
             None => Err(LvsError::PoolCreate {
@@ -713,12 +729,19 @@ impl Lvs {
 
         unsafe { vbdev_lvs_unload(self.as_inner_ptr(), Some(Self::lvs_op_cb), cb_arg(s)) };
 
-        r.await
+        let result = r
+            .await
             .expect("callback gone while exporting lvs")
             .to_result(|e| LvsError::Export {
                 source: BsError::from_i32(e),
                 name: pool.clone(),
-            })?;
+            });
+
+        if Lvs::lookup(&pool).is_none() {
+            Lvs::remove_info(&pool);
+        }
+
+        result?;
 
         info!(
             "{}: lvs exported successfully. base bdev: {}",
@@ -822,12 +845,19 @@ impl Lvs {
 
         unsafe { vbdev_lvs_destruct(self.as_inner_ptr(), Some(Self::lvs_op_cb), cb_arg(s)) };
 
-        r.await
+        let result = r
+            .await
             .expect("callback gone while destroying lvs")
             .to_result(|e| LvsError::Export {
                 source: BsError::from_i32(e),
                 name: pool.clone(),
-            })?;
+            });
+
+        if Lvs::lookup(&pool).is_none() {
+            Lvs::remove_info(&pool);
+        }
+
+        result?;
 
         info!(
             "{}: lvs destroyed successfully. base_bdev: {base_bdev:?}",

io-engine/src/pool_information.rs

@@ -0,0 +1,37 @@
+use once_cell::sync::Lazy;
+use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard};
+use std::{
+    collections::{HashMap, VecDeque},
+    time::{Duration, Instant},
+};
+
+/// In-memory structure containing pool runtime information.
+pub static POOL_INFO: Lazy<RwLock<HashMap<String, RwLock<PoolInfo>>>> =
+    Lazy::new(|| RwLock::new(HashMap::new()));
+
+/// Pool specfic information stored in-memory.
+#[derive(Default, Debug, Clone)]
+pub struct PoolInfo {
+    /// Set when Pools io is under stall.
+    pub io_stalled: bool,
+    /// Contains list of timestamps when pool went into stall state.
+    pub transition_timestamps: VecDeque<Instant>,
+}
+
+impl PoolInfo {
+    /// Retains only transition timestamps that are within the stall_transition_window.
+    pub fn update_transition_timestamp(&mut self, stall_transition_window: Duration) {
+        self.transition_timestamps
+            .retain(|ts| ts.elapsed() < stall_transition_window);
+    }
+}
+
+/// Returns write lock of the hashmap.
+pub fn pool_info_write() -> RwLockWriteGuard<'static, HashMap<String, RwLock<PoolInfo>>> {
+    POOL_INFO.write()
+}
+
+/// Returns read lock of the hashmap.
+pub fn pool_info_read() -> RwLockReadGuard<'static, HashMap<String, RwLock<PoolInfo>>> {
+    POOL_INFO.read()
+}