feat(metrics/block): add queue backlog length metric

This metric accumulates the number of pending
requests sitting in the virtio queue while a request
is being processed.
By looking at this number, one may make a decision
on whether an async IO solution would be beneficial.

(cherry picked from commit 58a87ed)

Signed-off-by: Nikita Kalyazin <[email protected]>

Author: kalyazin

src/vmm/src/devices/virtio/block/virtio/device.rs

@@ -424,6 +424,7 @@ impl VirtioBlock {
         let mut used_any = false;
 
         while let Some(head) = queue.pop_or_enable_notification(mem) {
+            self.metrics.remaining_reqs_count.add(queue.len(mem).into());
             let processing_result = match Request::parse(&head, mem, self.disk.nsectors) {
                 Ok(request) => {
                     if request.rate_limit(&mut self.rate_limiter) {

src/vmm/src/devices/virtio/block/virtio/metrics.rs

@@ -187,6 +187,8 @@ pub struct BlockDeviceMetrics {
     /// Number of virtio events throttled because of the IO engine.
     /// This happens when the io_uring submission queue is full.
     pub io_engine_throttled_events: SharedIncMetric,
+    /// Number of remaining requests in the queue.
+    pub remaining_reqs_count: SharedIncMetric,
 }
 
 impl BlockDeviceMetrics {
@@ -231,6 +233,8 @@ impl BlockDeviceMetrics {
             .add(other.rate_limiter_throttled_events.fetch_diff());
         self.io_engine_throttled_events
             .add(other.io_engine_throttled_events.fetch_diff());
+        self.remaining_reqs_count
+            .add(other.remaining_reqs_count.fetch_diff());
     }
 }
 

tests/host_tools/fcmetrics.py

@@ -59,6 +59,7 @@ def validate_fc_metrics(metrics):
             "write_count",
             "rate_limiter_throttled_events",
             "io_engine_throttled_events",
+            "remaining_reqs_count",
         ],
         "deprecated_api": [
             "deprecated_http_api_calls",