repair: prometheus metrics

book/api/metrics-generated.md

@@ -447,6 +447,8 @@
 | <span class="metrics-name">shred_&#8203;force_&#8203;complete_&#8203;request</span> | counter | The number of times we received a FEC force complete message |
 | <span class="metrics-name">shred_&#8203;force_&#8203;complete_&#8203;failure</span> | counter | The number of times we failed to force complete a FEC set on request |
 | <span class="metrics-name">shred_&#8203;force_&#8203;complete_&#8203;success</span> | counter | The number of times we successfully forced completed a FEC set on request |
+| <span class="metrics-name">shred_&#8203;shred_&#8203;repair_&#8203;rcv</span> | counter | The number of times we received a repair shred |
+| <span class="metrics-name">shred_&#8203;shred_&#8203;turbine_&#8203;rcv</span> | counter | The number of times we received a turbine shred |
 | <span class="metrics-name">shred_&#8203;store_&#8203;insert_&#8203;wait</span> | histogram | Time in seconds spent waiting for the store to insert a new FEC set |
 | <span class="metrics-name">shred_&#8203;store_&#8203;insert_&#8203;work</span> | histogram | Time in seconds spent on inserting a new FEC set |
 
@@ -744,12 +746,13 @@
 | <span class="metrics-name">repair_&#8203;recv_&#8203;serv_&#8203;pkt_&#8203;types</span><br/>{repair_&#8203;serv_&#8203;pkt_&#8203;types="<span class="metrics-enum">orphan</span>"} | counter | Server messages received (Orphan) |
 | <span class="metrics-name">repair_&#8203;recv_&#8203;serv_&#8203;pkt_&#8203;types</span><br/>{repair_&#8203;serv_&#8203;pkt_&#8203;types="<span class="metrics-enum">unknown</span>"} | counter | Server messages received (Unknown) |
 | <span class="metrics-name">repair_&#8203;recv_&#8203;pkt_&#8203;corrupted_&#8203;msg</span> | counter | How many corrupt messages have we received |
-| <span class="metrics-name">repair_&#8203;send_&#8203;pkt_&#8203;cnt</span> | counter | How many packets have sent |
+| <span class="metrics-name">repair_&#8203;shred_&#8203;repair_&#8203;req</span> | counter | How many repair requests have we sent |
 | <span class="metrics-name">repair_&#8203;sent_&#8203;pkt_&#8203;types</span><br/>{repair_&#8203;sent_&#8203;request_&#8203;types="<span class="metrics-enum">needed_&#8203;window</span>"} | counter | What types of client messages are we sending (Need Window) |
 | <span class="metrics-name">repair_&#8203;sent_&#8203;pkt_&#8203;types</span><br/>{repair_&#8203;sent_&#8203;request_&#8203;types="<span class="metrics-enum">needed_&#8203;highest_&#8203;window</span>"} | counter | What types of client messages are we sending (Need Highest Window) |
 | <span class="metrics-name">repair_&#8203;sent_&#8203;pkt_&#8203;types</span><br/>{repair_&#8203;sent_&#8203;request_&#8203;types="<span class="metrics-enum">needed_&#8203;orphan</span>"} | counter | What types of client messages are we sending (Need Orphans) |
 | <span class="metrics-name">repair_&#8203;store_&#8203;link_&#8203;wait</span> | histogram | Time in seconds spent waiting for the store to link a new FEC set |
 | <span class="metrics-name">repair_&#8203;store_&#8203;link_&#8203;work</span> | histogram | Time in seconds spent on linking a new FEC set |
+| <span class="metrics-name">repair_&#8203;slot_&#8203;complete_&#8203;time</span> | histogram | Time in seconds it took to complete a slot |
 | <span class="metrics-name">repair_&#8203;sign_&#8203;duration_&#8203;seconds</span> | histogram | Duration of signing a message |
 
 </div>

contrib/repair-analysis/report.py

@@ -375,7 +375,6 @@ def completion_times( fec_stats, shred_data, first_turbine, pdf ):
     # Batch completion times (ref_tick)
     # We get this by keeping the first shred of fec0, and the completion time of fec1
 
-
     batch_stats = fec_stats.groupby(['slot', 'ref_tick']).agg({'first_shred_ts': 'min', 'timestamp': 'max'}).reset_index()
     batch_stats['time_to_complete'] = batch_stats['timestamp'] - batch_stats['first_shred_ts']
     batch_stats['time_to_complete(ms)'] = batch_stats['time_to_complete'] / 1_000_000  # Convert to milliseconds
@@ -459,7 +458,48 @@ def completion_times( fec_stats, shred_data, first_turbine, pdf ):
     pdf.savefig(fig, bbox_inches='tight')
     plt.close(fig)
 
+    # Time in between slot completions
+
+    time_between_slots_live = slot_cmpl_live['timestamp_fec1'].sort_values().diff().fillna(0)
+    time_between_slots_catchup = slot_cmpl_catchup['timestamp_fec1'].sort_values().diff().fillna(0)
+    time_between_slots_live = time_between_slots_live / 1_000_000  # Convert to milliseconds
+    time_between_slots_catchup = time_between_slots_catchup / 1_000_000  # Convert to milliseconds
+
+    # plot the time between slots
+    fig = plt.figure(figsize=(12, 6))
+    sns.histplot(time_between_slots_live, bins=50, kde=True)
+    plt.title('Time Between Completing Live Slots')
+    plt.xlabel('Time Between Slots (ms)')
+    plt.ylabel('Frequency')
+
+    pdf.savefig(fig, bbox_inches='tight')
+    plt.close(fig)
 
+    print("\n\033[1mTime Between Completing Live Slots\033[0m\n")
+    print(time_between_slots_live.describe())
+    print("\n\033[1mTime Between Completing Catchup Slots\033[0m\n")
+    print(time_between_slots_catchup.describe())
+
+
+def show_turbine_arrivals(live, pdf):
+    # plot the turbine arrivals
+    fig = plt.figure(figsize=(12, 6))
+    live_turbine = live[live['is_turbine']]
+    live_turbine = live_turbine[live_turbine['slot'] >= 348905600]
+    #bucket it by every 10 ms, and round to to the nearest int
+    live_turbine['timestamp'] = (live_turbine['timestamp'] // 10_000_000).astype(int)
+    live_turbine = live_turbine.groupby('timestamp').size().reset_index(name='count')
+    sns.barplot(data=live_turbine, x='timestamp', y='count')
+    plt.title('Turbine Arrivals')
+    plt.xlabel('Timestamp')
+    plt.ylabel('count')
+    # show labels for every 10 ticks only
+    plt.setp(plt.gca().get_xticklabels(), visible=False)
+    plt.setp(plt.gca().get_xticklabels()[::5], visible=True)
+
+    plt.tight_layout()
+    pdf.savefig(fig, bbox_inches='tight')
+    plt.close(fig)
 
 def turbine_stats(catchup, live):
     print('\n\033[1mTurbine Statistics\033[0m\n')
@@ -577,9 +617,13 @@ def generate_report( log_path, request_data_path, shred_data_path, peers_data_pa
                                    skipfooter=1 ) # because of the buffered writer the last row is probably incomplete
 
     if request_data_path:
-        repair_requests = pd.read_csv( request_data_path,
-                                    dtype={'dst_ip': str, 'dst_port': int, 'timestamp': int, 'slot': int, 'idx': int, 'nonce': int },
-                                    skipfooter=1 )
+        try:
+            repair_requests = pd.read_csv( request_data_path,
+                                        dtype={'dst_ip': str, 'dst_port': int, 'timestamp': int, 'slot': int, 'idx': int, 'nonce': int },
+                                        skipfooter=1 )
+        except Exception as e:
+            print(f'Error reading repair requests: {e}')
+            request_data_path = None
 
     if peers_data_path:
         peers_data      = pd.read_csv( peers_data_path,
@@ -610,6 +654,7 @@ def generate_report( log_path, request_data_path, shred_data_path, peers_data_pa
     catchup = shreds_data[shreds_data['slot'].between(snapshot_slot, first_turbine - 1)]
     live    = shreds_data[shreds_data['slot'].between(first_turbine, last_executed)]
 
+    show_turbine_arrivals(live, pdf)
 
     if request_data_path:
         catchup_rq = repair_requests[repair_requests['slot'].between(snapshot_slot, first_turbine - 1)]

src/ballet/shred/fd_shred.h

@@ -259,7 +259,7 @@ struct __attribute__((packed)) fd_shred {
       /* Total number of coding shreds in FEC set. Must be positive <= FD_REEDSOL_CODE_SHREDS_MAX. */
       /* 0x55 */ ushort code_cnt;
 
-      /* Index within the vector of coding shreds in slot. In [0,
+      /* Index within the vector of coding shreds in FEC set. In [0,
          code_cnt).  Also, shred.code.idx <= shred.idx. */
       /* 0x57 */ ushort idx;
     } code;

src/disco/fd_disco_base.h

@@ -155,8 +155,8 @@ FD_FN_CONST static inline ulong fd_disco_replay_old_sig_slot( ulong sig ) { retu
    The encoded fields vary depending on the type of the sig.  The
    diagram below describes the encoding.
 
-   completes (1) | slot (32) | fec_set_idx (15) | is_code (1) | shred_idx or data_cnt (15)
-   [63]          | [31, 62]  | [16, 30]         | [15]        | [0, 14]
+   is_turbine (1) | slot (32) | fec_set_idx (15) | is_code (1) | shred_idx or data_cnt (15)
+   [63]           | [31, 62]  | [16, 30]         | [15]        | [0, 14]
 
    There are two types of messages on the shred_repair link.  The first
    type is a generic shred message. The second is a FEC set completion
@@ -165,8 +165,8 @@ FD_FN_CONST static inline ulong fd_disco_replay_old_sig_slot( ulong sig ) { retu
 
    For the first message type (SHRED):
 
-   The first bit [63] describes whether this shred marks the end of a
-   batch and/or a slot, i.e. shred.flags & (DATA_COMPLETE | SLOT_COMPLETE)
+   The first bit [63] describes whether this shred source was turbine
+   or repair.
 
    The next 32 bits [31, 62] describe the slot number. Note: if the slot
    number is >= UINT_MAX, the sender will store the value UINT_MAX in
@@ -194,24 +194,24 @@ FD_FN_CONST static inline ulong fd_disco_replay_old_sig_slot( ulong sig ) { retu
    are uniformly coding shreds and fixed size. */
 
 FD_FN_CONST static inline ulong
-fd_disco_shred_repair_shred_sig( int   completes,
+fd_disco_shred_repair_shred_sig( int   is_turbine,
                                  ulong slot,
                                  uint  fec_set_idx,
                                  int   is_code,
                                  uint  shred_idx_or_data_cnt ) {
    ulong slot_ul                  = fd_ulong_min( slot, (ulong)UINT_MAX );
    ulong shred_idx_or_data_cnt_ul = fd_ulong_min( (ulong)shred_idx_or_data_cnt, (ulong)FD_SHRED_BLK_MAX );
    ulong fec_set_idx_ul           = fd_ulong_min( (ulong)fec_set_idx, (ulong)FD_SHRED_BLK_MAX );
-   ulong completes_ul             = !!completes;
+   ulong is_turbine_ul            = !!is_turbine;
    ulong is_code_ul               = !!is_code;
 
-  return completes_ul << 63 | slot_ul << 31 | fec_set_idx_ul << 16 | is_code_ul << 15 | shred_idx_or_data_cnt_ul;
+  return is_turbine_ul << 63 | slot_ul << 31 | fec_set_idx_ul << 16 | is_code_ul << 15 | shred_idx_or_data_cnt_ul;
 }
 
 /* fd_disco_shred_repair_shred_sig_{...} are accessors for the fields encoded
    in the sig described above. */
 
-FD_FN_CONST static inline int   fd_disco_shred_repair_shred_sig_completes  ( ulong sig ) { return       fd_ulong_extract_bit( sig, 63     ); }
+FD_FN_CONST static inline int   fd_disco_shred_repair_shred_sig_is_turbine ( ulong sig ) { return       fd_ulong_extract_bit( sig, 63     ); }
 FD_FN_CONST static inline ulong fd_disco_shred_repair_shred_sig_slot       ( ulong sig ) { return       fd_ulong_extract    ( sig, 31, 62 ); }
 FD_FN_CONST static inline uint  fd_disco_shred_repair_shred_sig_fec_set_idx( ulong sig ) { return (uint)fd_ulong_extract    ( sig, 16, 30 ); }
 FD_FN_CONST static inline int   fd_disco_shred_repair_shred_sig_is_code    ( ulong sig ) { return       fd_ulong_extract_bit( sig, 15     ); }

src/disco/metrics/fd_metrics_base.h

@@ -78,7 +78,7 @@ typedef struct {
           ulong min;
           ulong max;
         } none;
-        
+
         struct {
           double min;
           double max;

src/disco/metrics/generated/fd_metrics_repair.c

@@ -13,11 +13,12 @@ const fd_metrics_meta_t FD_METRICS_REPAIR[FD_METRICS_REPAIR_TOTAL] = {
     DECLARE_METRIC_ENUM( REPAIR_RECV_SERV_PKT_TYPES, COUNTER, REPAIR_SERV_PKT_TYPES, ORPHAN ),
     DECLARE_METRIC_ENUM( REPAIR_RECV_SERV_PKT_TYPES, COUNTER, REPAIR_SERV_PKT_TYPES, UNKNOWN ),
     DECLARE_METRIC( REPAIR_RECV_PKT_CORRUPTED_MSG, COUNTER ),
-    DECLARE_METRIC( REPAIR_SEND_PKT_CNT, COUNTER ),
+    DECLARE_METRIC( REPAIR_SHRED_REPAIR_REQ, COUNTER ),
     DECLARE_METRIC_ENUM( REPAIR_SENT_PKT_TYPES, COUNTER, REPAIR_SENT_REQUEST_TYPES, NEEDED_WINDOW ),
     DECLARE_METRIC_ENUM( REPAIR_SENT_PKT_TYPES, COUNTER, REPAIR_SENT_REQUEST_TYPES, NEEDED_HIGHEST_WINDOW ),
     DECLARE_METRIC_ENUM( REPAIR_SENT_PKT_TYPES, COUNTER, REPAIR_SENT_REQUEST_TYPES, NEEDED_ORPHAN ),
     DECLARE_METRIC_HISTOGRAM_SECONDS( REPAIR_STORE_LINK_WAIT ),
     DECLARE_METRIC_HISTOGRAM_SECONDS( REPAIR_STORE_LINK_WORK ),
+    DECLARE_METRIC_HISTOGRAM_SECONDS( REPAIR_SLOT_COMPLETE_TIME ),
     DECLARE_METRIC_HISTOGRAM_SECONDS( REPAIR_SIGN_DURATION_SECONDS ),
 };

src/disco/metrics/generated/fd_metrics_repair.h

@@ -52,11 +52,11 @@
 #define FD_METRICS_COUNTER_REPAIR_RECV_PKT_CORRUPTED_MSG_DESC "How many corrupt messages have we received"
 #define FD_METRICS_COUNTER_REPAIR_RECV_PKT_CORRUPTED_MSG_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_COUNTER_REPAIR_SEND_PKT_CNT_OFF  (27UL)
-#define FD_METRICS_COUNTER_REPAIR_SEND_PKT_CNT_NAME "repair_send_pkt_cnt"
-#define FD_METRICS_COUNTER_REPAIR_SEND_PKT_CNT_TYPE (FD_METRICS_TYPE_COUNTER)
-#define FD_METRICS_COUNTER_REPAIR_SEND_PKT_CNT_DESC "How many packets have sent"
-#define FD_METRICS_COUNTER_REPAIR_SEND_PKT_CNT_CVT  (FD_METRICS_CONVERTER_NONE)
+#define FD_METRICS_COUNTER_REPAIR_SHRED_REPAIR_REQ_OFF  (27UL)
+#define FD_METRICS_COUNTER_REPAIR_SHRED_REPAIR_REQ_NAME "repair_shred_repair_req"
+#define FD_METRICS_COUNTER_REPAIR_SHRED_REPAIR_REQ_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_REPAIR_SHRED_REPAIR_REQ_DESC "How many repair requests have we sent"
+#define FD_METRICS_COUNTER_REPAIR_SHRED_REPAIR_REQ_CVT  (FD_METRICS_CONVERTER_NONE)
 
 #define FD_METRICS_COUNTER_REPAIR_SENT_PKT_TYPES_OFF  (28UL)
 #define FD_METRICS_COUNTER_REPAIR_SENT_PKT_TYPES_NAME "repair_sent_pkt_types"
@@ -85,13 +85,21 @@
 #define FD_METRICS_HISTOGRAM_REPAIR_STORE_LINK_WORK_MIN  (1e-08)
 #define FD_METRICS_HISTOGRAM_REPAIR_STORE_LINK_WORK_MAX  (0.0005)
 
-#define FD_METRICS_HISTOGRAM_REPAIR_SIGN_DURATION_SECONDS_OFF  (65UL)
+#define FD_METRICS_HISTOGRAM_REPAIR_SLOT_COMPLETE_TIME_OFF  (65UL)
+#define FD_METRICS_HISTOGRAM_REPAIR_SLOT_COMPLETE_TIME_NAME "repair_slot_complete_time"
+#define FD_METRICS_HISTOGRAM_REPAIR_SLOT_COMPLETE_TIME_TYPE (FD_METRICS_TYPE_HISTOGRAM)
+#define FD_METRICS_HISTOGRAM_REPAIR_SLOT_COMPLETE_TIME_DESC "Time in seconds it took to complete a slot"
+#define FD_METRICS_HISTOGRAM_REPAIR_SLOT_COMPLETE_TIME_CVT  (FD_METRICS_CONVERTER_SECONDS)
+#define FD_METRICS_HISTOGRAM_REPAIR_SLOT_COMPLETE_TIME_MIN  (0.2)
+#define FD_METRICS_HISTOGRAM_REPAIR_SLOT_COMPLETE_TIME_MAX  (2.0)
+
+#define FD_METRICS_HISTOGRAM_REPAIR_SIGN_DURATION_SECONDS_OFF  (82UL)
 #define FD_METRICS_HISTOGRAM_REPAIR_SIGN_DURATION_SECONDS_NAME "repair_sign_duration_seconds"
 #define FD_METRICS_HISTOGRAM_REPAIR_SIGN_DURATION_SECONDS_TYPE (FD_METRICS_TYPE_HISTOGRAM)
 #define FD_METRICS_HISTOGRAM_REPAIR_SIGN_DURATION_SECONDS_DESC "Duration of signing a message"
 #define FD_METRICS_HISTOGRAM_REPAIR_SIGN_DURATION_SECONDS_CVT  (FD_METRICS_CONVERTER_SECONDS)
 #define FD_METRICS_HISTOGRAM_REPAIR_SIGN_DURATION_SECONDS_MIN  (1e-08)
 #define FD_METRICS_HISTOGRAM_REPAIR_SIGN_DURATION_SECONDS_MAX  (0.001)
 
-#define FD_METRICS_REPAIR_TOTAL (18UL)
+#define FD_METRICS_REPAIR_TOTAL (19UL)
 extern const fd_metrics_meta_t FD_METRICS_REPAIR[FD_METRICS_REPAIR_TOTAL];

src/disco/metrics/generated/fd_metrics_shred.c

@@ -22,6 +22,8 @@ const fd_metrics_meta_t FD_METRICS_SHRED[FD_METRICS_SHRED_TOTAL] = {
     DECLARE_METRIC( SHRED_FORCE_COMPLETE_REQUEST, COUNTER ),
     DECLARE_METRIC( SHRED_FORCE_COMPLETE_FAILURE, COUNTER ),
     DECLARE_METRIC( SHRED_FORCE_COMPLETE_SUCCESS, COUNTER ),
+    DECLARE_METRIC( SHRED_SHRED_REPAIR_RCV, COUNTER ),
+    DECLARE_METRIC( SHRED_SHRED_TURBINE_RCV, COUNTER ),
     DECLARE_METRIC_HISTOGRAM_SECONDS( SHRED_STORE_INSERT_WAIT ),
     DECLARE_METRIC_HISTOGRAM_SECONDS( SHRED_STORE_INSERT_WORK ),
 };

src/disco/metrics/generated/fd_metrics_shred.h

@@ -111,21 +111,33 @@
 #define FD_METRICS_COUNTER_SHRED_FORCE_COMPLETE_SUCCESS_DESC "The number of times we successfully forced completed a FEC set on request"
 #define FD_METRICS_COUNTER_SHRED_FORCE_COMPLETE_SUCCESS_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WAIT_OFF  (116UL)
+#define FD_METRICS_COUNTER_SHRED_SHRED_REPAIR_RCV_OFF  (116UL)
+#define FD_METRICS_COUNTER_SHRED_SHRED_REPAIR_RCV_NAME "shred_shred_repair_rcv"
+#define FD_METRICS_COUNTER_SHRED_SHRED_REPAIR_RCV_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_SHRED_SHRED_REPAIR_RCV_DESC "The number of times we received a repair shred"
+#define FD_METRICS_COUNTER_SHRED_SHRED_REPAIR_RCV_CVT  (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_SHRED_SHRED_TURBINE_RCV_OFF  (117UL)
+#define FD_METRICS_COUNTER_SHRED_SHRED_TURBINE_RCV_NAME "shred_shred_turbine_rcv"
+#define FD_METRICS_COUNTER_SHRED_SHRED_TURBINE_RCV_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_SHRED_SHRED_TURBINE_RCV_DESC "The number of times we received a turbine shred"
+#define FD_METRICS_COUNTER_SHRED_SHRED_TURBINE_RCV_CVT  (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WAIT_OFF  (118UL)
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WAIT_NAME "shred_store_insert_wait"
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WAIT_TYPE (FD_METRICS_TYPE_HISTOGRAM)
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WAIT_DESC "Time in seconds spent waiting for the store to insert a new FEC set"
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WAIT_CVT  (FD_METRICS_CONVERTER_SECONDS)
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WAIT_MIN  (1e-08)
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WAIT_MAX  (0.0005)
 
-#define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WORK_OFF  (133UL)
+#define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WORK_OFF  (135UL)
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WORK_NAME "shred_store_insert_work"
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WORK_TYPE (FD_METRICS_TYPE_HISTOGRAM)
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WORK_DESC "Time in seconds spent on inserting a new FEC set"
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WORK_CVT  (FD_METRICS_CONVERTER_SECONDS)
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WORK_MIN  (1e-08)
 #define FD_METRICS_HISTOGRAM_SHRED_STORE_INSERT_WORK_MAX  (0.0005)
 
-#define FD_METRICS_SHRED_TOTAL (22UL)
+#define FD_METRICS_SHRED_TOTAL (24UL)
 extern const fd_metrics_meta_t FD_METRICS_SHRED[FD_METRICS_SHRED_TOTAL];