Add recovery_status to lustrefs_exporter and

enhance recovery status parser to include additional metrics:

- RecoveryDuration
- RecoveryTimeRemaining
- RecoveryTotalClients

Author: breuhan

lustre-collector/src/fixtures/valid/lustre-2.14.0_ddn212/2.14.0_ddn212_recovery.txt

@@ -0,0 +1,36 @@
+obdfilter.200NVX2-OST0000.recovery_status=
+status: COMPLETE
+recovery_start: 1761567697
+recovery_duration: 1
+completed_clients: 8/8
+replayed_requests: 0
+last_transno: 17184814233
+VBR: DISABLED
+IR: ENABLED
+obdfilter.200NVX2-OST0003.recovery_status=
+status: COMPLETE
+recovery_start: 1759494115
+recovery_duration: 15
+completed_clients: 8/8
+replayed_requests: 0
+last_transno: 12934942643
+VBR: DISABLED
+IR: DISABLED
+obdfilter.200NVX2-OST0004.recovery_status=
+status: COMPLETE
+recovery_start: 1759494115
+recovery_duration: 14
+completed_clients: 8/8
+replayed_requests: 0
+last_transno: 12934956643
+VBR: DISABLED
+IR: DISABLED
+obdfilter.200NVX2-OST0007.recovery_status=
+status: COMPLETE
+recovery_start: 1759494115
+recovery_duration: 14
+completed_clients: 8/8
+replayed_requests: 0
+last_transno: 12934943652
+VBR: DISABLED
+IR: DISABLED

lustre-collector/src/parser.rs

@@ -6,7 +6,7 @@ use crate::{
     ldlm, llite, mdd_parser,
     mds::{self, client_count_parser},
     mgs::mgs_parser,
-    nodemap, osd_parser, oss, quota, top_level_parser,
+    nodemap, osd_parser, oss, quota, recovery_status_parser, top_level_parser,
     types::Record,
 };
 use combine::{Parser, Stream, choice, error::ParseError, many};
@@ -24,6 +24,7 @@ pub fn params() -> Vec<String> {
         .chain(mdd_parser::params())
         .chain(quota::params())
         .chain(nodemap::params())
+        .chain(recovery_status_parser::params())
         .collect()
 }
 
@@ -44,6 +45,7 @@ where
         mdd_parser::parse().map(|x| vec![x]),
         quota::parse().map(|x| vec![x]),
         nodemap::parse().map(|x| vec![x]),
+        recovery_status_parser::parse().map(|x| x),
     )))
     .map(|xs: Vec<_>| xs.into_iter().flatten().collect())
 }
@@ -86,6 +88,7 @@ mod tests {
     test_fixtures!(test_lustre_2_14_0_ddn145_fixtures, "*ddn145*");
 
     test_fixtures!(test_lustre_2_14_0_ddn133_fixtures, "*ddn133*");
+    test_fixtures!(test_lustre_2_14_0_ddn212_fixtures, "*ddn212*");
 
     #[test]
     fn test_params() {

lustre-collector/src/recovery_status_parser.rs

@@ -71,7 +71,7 @@ where
         })
 }
 
-fn clients_line<I>(x: &'static str) -> impl Parser<I, Output = u64>
+fn clients_line<I>(x: &'static str) -> impl Parser<I, Output = (u64, Option<u64>)>
 where
     I: Stream<Token = char>,
     I::Error: ParseError<I::Token, I::Range, I::Position>,
@@ -83,15 +83,18 @@ where
         optional((token('/'), digits())),
         optional(newline().map(drop).or(eof())),
     )
-        .map(|(_, _, x, _, _): (_, _, u64, _, _)| x)
+        .map(|(_, _, x, y, _): (_, _, u64, Option<(_, u64)>, _)| (x, y.map(|(_, v)| v)))
 }
 
 #[derive(Debug)]
 enum RecoveryStat {
-    Status(RecoveryStatus),
     Completed(u64),
     Connected(u64),
     Evicted(u64),
+    RecoveryDuration(u64),
+    Status(RecoveryStatus),
+    TimeRemaining(u64),
+    Total(Option<u64>),
 }
 
 pub struct StatName(pub String);
@@ -105,6 +108,19 @@ where
     many1(alpha_num().or(one_of("_-".chars()))).map(StatName)
 }
 
+fn simple_client_stat<I>(
+    name: &'static str,
+    constructor: fn(u64) -> RecoveryStat,
+) -> impl Parser<I, Output = Vec<RecoveryStat>>
+where
+    I: Stream<Token = char>,
+    I::Error: ParseError<I::Token, I::Range, I::Position>,
+{
+    clients_line(name)
+        .skip(optional(newline()))
+        .map(move |(x, _)| vec![constructor(x)])
+}
+
 fn target_recovery_stats<I>() -> impl Parser<I, Output = Vec<RecoveryStat>>
 where
     I: Stream<Token = char>,
@@ -113,22 +129,22 @@ where
     many(choice((
         status_line()
             .skip(optional(newline()))
-            .map(RecoveryStat::Status)
-            .map(Some),
-        clients_line("completed_clients")
-            .skip(optional(newline()))
-            .map(RecoveryStat::Completed)
-            .map(Some),
+            .map(|x| vec![RecoveryStat::Status(x)]),
+        simple_client_stat("recovery_duration", RecoveryStat::RecoveryDuration),
+        simple_client_stat("completed_clients", RecoveryStat::Completed),
+        simple_client_stat("time_remaining", RecoveryStat::TimeRemaining),
+        simple_client_stat("evicted_clients", RecoveryStat::Evicted),
         clients_line("connected_clients")
             .skip(optional(newline()))
-            .map(RecoveryStat::Connected)
-            .map(Some),
-        clients_line("evicted_clients")
-            .skip(optional(newline()))
-            .map(RecoveryStat::Evicted)
-            .map(Some),
+            .map(|(x, y)| {
+                let mut stats = vec![RecoveryStat::Connected(x)];
+                if let Some(total) = y {
+                    stats.push(RecoveryStat::Total(Some(total)));
+                }
+                stats
+            }),
         // This will ignore line/field we don't care
-        attempt((stat_name(), token(':'), till_newline().skip(newline()))).map(|_| None),
+        attempt((stat_name(), token(':'), till_newline().skip(newline()))).map(|_| vec![]),
     )))
     .map(|xs: Vec<_>| xs.into_iter().flatten().collect())
 }
@@ -176,6 +192,28 @@ where
                             value: *value,
                         })
                     }
+                    RecoveryStat::RecoveryDuration(value) => {
+                        TargetStats::RecoveryDuration(TargetStat {
+                            kind,
+                            param: param.clone(),
+                            target: target.clone(),
+                            value: *value,
+                        })
+                    }
+                    RecoveryStat::TimeRemaining(value) => {
+                        TargetStats::RecoveryTimeRemaining(TargetStat {
+                            kind,
+                            param: param.clone(),
+                            target: target.clone(),
+                            value: *value,
+                        })
+                    }
+                    RecoveryStat::Total(value) => TargetStats::RecoveryTotalClients(TargetStat {
+                        kind,
+                        param: param.clone(),
+                        target: target.clone(),
+                        value: value.unwrap_or(0),
+                    }),
                 })
                 .collect()
         })
@@ -186,19 +224,17 @@ where
     I: Stream<Token = char>,
     I::Error: ParseError<I::Token, I::Range, I::Position>,
 {
-    many(
-        (
-            target_status(),
-            skip_until(attempt(ost_or_mdt().map(drop)).or(eof())),
-        )
-            .map(|(x, _)| x.into_iter().map(Record::Target).collect()),
+    (
+        target_status(),
+        skip_until(attempt(ost_or_mdt().map(drop)).or(eof())),
     )
-    .map(|x: Vec<Vec<Record>>| x.into_iter().flatten().collect())
+        .map(|(x, _)| x.into_iter().map(Record::Target).collect())
 }
 
 #[cfg(test)]
 mod tests {
-    use crate::recovery_status_parser::{clients_line, parse, target_recovery_stats};
+    use crate::parser::parse;
+    use crate::recovery_status_parser::{clients_line, target_recovery_stats};
     use combine::{Parser, parser::EasyParser, stream::position};
 
     #[test]
@@ -238,11 +274,11 @@ mod tests {
     #[test]
     fn test_clients_line() {
         let result = clients_line("completed_clients").parse("completed_clients: 3/7\n");
-        assert_eq!(result, Ok((3, "")));
+        assert_eq!(result, Ok(((3, Some(7)), "")));
         let result = clients_line("connected_clients").parse("connected_clients: 3/7\n");
-        assert_eq!(result, Ok((3, "")));
+        assert_eq!(result, Ok(((3, Some(7)), "")));
         let result = clients_line("completed_clients").parse("completed_clients: 3\n");
-        assert_eq!(result, Ok((3, "")));
+        assert_eq!(result, Ok(((3, None), "")));
     }
 
     #[test]
@@ -259,7 +295,19 @@ IR: ENABLED
 
         let (records, _): (Vec<_>, _) = target_recovery_stats().parse(x).unwrap();
 
-        insta::assert_debug_snapshot!(records);
+        insta::assert_debug_snapshot!(records, @r"
+        [
+            Status(
+                Complete,
+            ),
+            RecoveryDuration(
+                150,
+            ),
+            Completed(
+                4,
+            ),
+        ]
+        ");
     }
 
     #[test]
@@ -275,6 +323,26 @@ completed_clients: 3
 
         let (records, _): (Vec<_>, _) = target_recovery_stats().parse(x).unwrap();
 
-        insta::assert_debug_snapshot!(records);
+        insta::assert_debug_snapshot!(records, @r"
+        [
+            Status(
+                Recovering,
+            ),
+            TimeRemaining(
+                119,
+            ),
+            Connected(
+                3,
+            ),
+            Total(
+                Some(
+                    7,
+                ),
+            ),
+            Completed(
+                3,
+            ),
+        ]
+        ");
     }
 }

lustre-collector/src/snapshots/lustre_collector__parser__tests__params.snap

@@ -67,4 +67,6 @@ expression: params()
     "qmt.*.*.glb-grp",
     "nodemap.*.dt_stats",
     "nodemap.*.md_stats",
+    "obdfilter.*OST*.recovery_status",
+    "mdt.*MDT*.recovery_status",
 ]

lustre-collector/src/snapshots/lustre_collector__parser__tests__valid_fixture_lustre-2.14.0_ddn145__2.14.0_ddn145_recovery.txt.snap

@@ -3,6 +3,77 @@ source: lustre-collector/src/parser.rs
 expression: result
 ---
 (
-    [],
-    "obdfilter.fs-OST0000.recovery_status=status: INACTIVE\nobdfilter.fs-OST0001.recovery_status=status: INACTIVE\nmdt.fs-MDT0000.recovery_status=\nstatus: COMPLETE\nrecovery_start: 1715712434\nrecovery_duration: 0\ncompleted_clients: 1/1\nreplayed_requests: 0\nlast_transno: 4294967298\nVBR: DISABLED\nIR: DISABLED\n",
+    [
+        Target(
+            RecoveryStatus(
+                TargetStat {
+                    kind: Ost,
+                    param: Param(
+                        "recovery_status",
+                    ),
+                    target: Target(
+                        "fs-OST0000",
+                    ),
+                    value: Inactive,
+                },
+            ),
+        ),
+        Target(
+            RecoveryStatus(
+                TargetStat {
+                    kind: Ost,
+                    param: Param(
+                        "recovery_status",
+                    ),
+                    target: Target(
+                        "fs-OST0001",
+                    ),
+                    value: Inactive,
+                },
+            ),
+        ),
+        Target(
+            RecoveryStatus(
+                TargetStat {
+                    kind: Mdt,
+                    param: Param(
+                        "recovery_status",
+                    ),
+                    target: Target(
+                        "fs-MDT0000",
+                    ),
+                    value: Complete,
+                },
+            ),
+        ),
+        Target(
+            RecoveryDuration(
+                TargetStat {
+                    kind: Mdt,
+                    param: Param(
+                        "recovery_status",
+                    ),
+                    target: Target(
+                        "fs-MDT0000",
+                    ),
+                    value: 0,
+                },
+            ),
+        ),
+        Target(
+            RecoveryCompletedClients(
+                TargetStat {
+                    kind: Mdt,
+                    param: Param(
+                        "recovery_status",
+                    ),
+                    target: Target(
+                        "fs-MDT0000",
+                    ),
+                    value: 1,
+                },
+            ),
+        ),
+    ],
+    "",
 )