feat(metrics/block): add file access latency metrics

These metrics measure how long Firecracker VMM thread
is blocked on the read/write syscalls when accessing
data in the drive file.
By looking at them, we will be able to see what portion
of block latency is attributed to factors external
to Firecracker.

(cherry picked from commit e00d62d)

Signed-off-by: Nikita Kalyazin <[email protected]>

Author: kalyazin

src/vmm/src/devices/virtio/block/virtio/metrics.rs

@@ -83,7 +83,7 @@ use std::sync::{Arc, RwLock};
 use serde::ser::SerializeMap;
 use serde::{Serialize, Serializer};
 
-use crate::logger::{IncMetric, SharedIncMetric};
+use crate::logger::{IncMetric, LatencyAggregateMetrics, SharedIncMetric};
 
 /// map of block drive id and metrics
 /// this should be protected by a lock before accessing.
@@ -105,7 +105,7 @@ impl BlockMetricsPerDevice {
                 .write()
                 .unwrap()
                 .metrics
-                .insert(drive_id.clone(), Arc::new(BlockDeviceMetrics::default()));
+                .insert(drive_id.clone(), Arc::new(BlockDeviceMetrics::new()));
         }
         METRICS
             .read()
@@ -178,6 +178,10 @@ pub struct BlockDeviceMetrics {
     pub read_count: SharedIncMetric,
     /// Number of successful write operations.
     pub write_count: SharedIncMetric,
+    /// Duration of all read operations.
+    pub read_agg: LatencyAggregateMetrics,
+    /// Duration of all write operations.
+    pub write_agg: LatencyAggregateMetrics,
     /// Number of rate limiter throttling events.
     pub rate_limiter_throttled_events: SharedIncMetric,
     /// Number of virtio events throttled because of the IO engine.
@@ -186,6 +190,15 @@ pub struct BlockDeviceMetrics {
 }
 
 impl BlockDeviceMetrics {
+    /// Const default construction.
+    pub fn new() -> Self {
+        Self {
+            read_agg: LatencyAggregateMetrics::new(),
+            write_agg: LatencyAggregateMetrics::new(),
+            ..Default::default()
+        }
+    }
+
     /// block metrics are SharedIncMetric where the diff of current vs
     /// old is serialized i.e. serialize_u64(current-old).
     /// So to have the aggregate serialized in same way we need to
@@ -210,6 +223,10 @@ impl BlockDeviceMetrics {
         self.write_bytes.add(other.write_bytes.fetch_diff());
         self.read_count.add(other.read_count.fetch_diff());
         self.write_count.add(other.write_count.fetch_diff());
+        self.read_agg.sum_us.add(other.read_agg.sum_us.fetch_diff());
+        self.write_agg
+            .sum_us
+            .add(other.write_agg.sum_us.fetch_diff());
         self.rate_limiter_throttled_events
             .add(other.rate_limiter_throttled_events.fetch_diff());
         self.io_engine_throttled_events

src/vmm/src/devices/virtio/block/virtio/request.rs

@@ -373,10 +373,12 @@ impl Request {
         let pending = self.to_pending_request(desc_idx);
         let res = match self.r#type {
             RequestType::In => {
+                let _metric = block_metrics.read_agg.record_latency_metrics();
                 disk.file_engine
                     .read(self.offset(), mem, self.data_addr, self.data_len, pending)
             }
             RequestType::Out => {
+                let _metric = block_metrics.write_agg.record_latency_metrics();
                 disk.file_engine
                     .write(self.offset(), mem, self.data_addr, self.data_len, pending)
             }

tests/host_tools/fcmetrics.py

@@ -234,7 +234,12 @@ def validate_fc_metrics(metrics):
         ],
     }
 
-    latency_agg_metrics = {}
+    latency_agg_metrics = {
+        "block": [
+            "read_agg",
+            "write_agg",
+        ],
+    }
 
     # validate timestamp before jsonschema validation which some more time
     utc_time = datetime.datetime.now(datetime.timezone.utc)