chore: add a metric for how many times a limit has been reached when creating an ingress payload (#9283)

This could be useful to decide whether we should bump limits on the
payload sizes

Author: kpop-dfinity

rs/ingress_manager/src/ingress_selector.rs

@@ -196,7 +196,10 @@ impl IngressSelector for IngressManager {
                         Ok(()) => (),
                         Err(ValidationError::InvalidArtifact(
                             InvalidIngressPayloadReason::IngressPayloadTooManyMessages(_, _),
-                        )) => break 'outer,
+                        )) => {
+                            self.metrics.observe_limit_reached("messages_count_limit");
+                            break 'outer;
+                        }
                         _ => {
                             queue.msgs.pop();
                             continue;
@@ -206,9 +209,12 @@ impl IngressSelector for IngressManager {
                     let size_estimates = self.message_size_estimates(&ingress.signed_ingress);
 
                     // Break criterion #1: global byte limit
-                    if accumulated_wire_size + size_estimates.wire > wire_byte_limit
-                        || accumulated_memory_size + size_estimates.memory > memory_byte_limit
-                    {
+                    if accumulated_wire_size + size_estimates.wire > wire_byte_limit {
+                        self.metrics.observe_limit_reached("wire_byte_limit");
+                        break 'outer;
+                    }
+                    if accumulated_memory_size + size_estimates.memory > memory_byte_limit {
+                        self.metrics.observe_limit_reached("memory_byte_limit");
                         break 'outer;
                     }
 
@@ -247,9 +253,12 @@ impl IngressSelector for IngressManager {
                 }
             }
 
-            if wire_byte_limit <= accumulated_wire_size
-                || memory_byte_limit <= accumulated_memory_size
-            {
+            if wire_byte_limit <= accumulated_wire_size {
+                self.metrics.observe_limit_reached("wire_byte_limit");
+                // No remaining quota means the block is full. No more iterations needed.
+                break;
+            } else if memory_byte_limit <= accumulated_memory_size {
+                self.metrics.observe_limit_reached("memory_byte_limit");
                 // No remaining quota means the block is full. No more iterations needed.
                 break;
             } else {
@@ -291,6 +300,7 @@ impl IngressSelector for IngressManager {
                 wire_byte_limit.get()
             );
             messages_in_payload.pop();
+            self.metrics.observe_limit_reached("serialized");
             if messages_in_payload.is_empty() {
                 break IngressPayload::default();
             }
@@ -305,6 +315,7 @@ impl IngressSelector for IngressManager {
 
         debug_assert!(size_estimates.wire <= wire_byte_limit);
         debug_assert!(size_estimates.memory <= memory_byte_limit);
+        debug_assert!(payload.payload.message_count() <= settings.max_ingress_messages_per_block);
         payload
     }
 

rs/ingress_manager/src/metrics.rs

@@ -9,6 +9,7 @@ pub(crate) struct IngressManagerMetrics {
     pub(crate) ingress_selector_get_payload_time: Histogram,
     pub(crate) ingress_selector_validate_payload_time: Histogram,
     pub(crate) ingress_payload_cache_size: IntGauge,
+    limit_reached: IntCounterVec,
 
     validated_ingress_message_size: Histogram,
     validated_ingress_message_field_size: HistogramVec,
@@ -62,6 +63,12 @@ impl IngressManagerMetrics {
                 "The number of invalidated ingress messages, partitioned by the reason",
                 &["reason"],
             ),
+            limit_reached: metrics_registry.int_counter_vec(
+                "ingress_selector_limit_reached_total",
+                "The total number of times we reached a limit while creating an ingress payload, \
+                labeled by a type of the limit",
+                &["limit_type"],
+            ),
         }
     }
 
@@ -104,4 +111,8 @@ impl IngressManagerMetrics {
             .with_label_values(&["remainder"])
             .observe(everything_else_size as f64);
     }
+
+    pub(crate) fn observe_limit_reached(&self, limit_type: &str) {
+        self.limit_reached.with_label_values(&[limit_type]).inc();
+    }
 }