chore: report number of failed outgoing requests

Author: ogenev

crates/metrics/src/overlay.rs

@@ -16,6 +16,7 @@ use crate::labels::{MessageDirectionLabel, MessageLabel, UtpDirectionLabel, UtpO
 #[derive(Clone)]
 pub struct OverlayMetrics {
     pub message_total: IntCounterVec,
+    pub failed_message_sent: IntCounterVec,
     pub utp_outcome_total: IntCounterVec,
     pub utp_active_gauge: IntGaugeVec,
     /// Total bytes transferred inbound
@@ -33,6 +34,14 @@ impl OverlayMetrics {
             &["protocol", "direction", "type"],
             registry
         )?;
+        let failed_message_sent = register_int_counter_vec_with_registry!(
+            opts!(
+                "trin_failed_message_sent",
+                "count all network messages sent"
+            ),
+            &["protocol", "direction", "type"],
+            registry
+        )?;
         let utp_outcome_total = register_int_counter_vec_with_registry!(
             opts!(
                 "trin_utp_outcome_total",
@@ -67,6 +76,7 @@ impl OverlayMetrics {
         )?;
         Ok(Self {
             message_total,
+            failed_message_sent,
             utp_outcome_total,
             utp_active_gauge,
             bytes_inbound_total,
@@ -115,13 +125,29 @@ impl OverlayMetricsReporter {
         self.increment_message_total(MessageDirectionLabel::Received, response.into());
     }
 
+    pub fn report_failed_outbound_request(&self, request: &Request) {
+        self.increment_failed_message_sent(MessageDirectionLabel::Sent, request.into());
+    }
+
     fn increment_message_total(&self, direction: MessageDirectionLabel, message: MessageLabel) {
         let labels: [&str; 3] = [&self.protocol, direction.into(), message.into()];
         self.overlay_metrics
             .message_total
             .with_label_values(&labels)
             .inc();
     }
+    /// Increment the failed message sent metric
+    fn increment_failed_message_sent(
+        &self,
+        direction: MessageDirectionLabel,
+        message: MessageLabel,
+    ) {
+        let labels: [&str; 3] = [&self.protocol, direction.into(), message.into()];
+        self.overlay_metrics
+            .failed_message_sent
+            .with_label_values(&labels)
+            .inc();
+    }
     /// Increase the total bytes inbound metric by the given length.
     pub fn report_bytes_inbound(&self, bytes_len: u64) {
         let labels: [&str; 1] = [&self.protocol];

crates/portalnet/src/overlay/service.rs

@@ -372,7 +372,11 @@ impl<
                                 self.metrics.report_inbound_response(&response);
                                 self.process_response(response, request.destination, request.request, request.query_id, request.request_permit)
                             }
-                            Err(error) => self.process_request_failure(response.request_id, request.destination, error),
+                            Err(error) => {
+                                // Metric repord failed request
+                                self.metrics.report_failed_outbound_request(&request.request);
+                                self.process_request_failure(response.request_id, request.destination, error)
+                            },
                         }
 
                     } else {