feat: adding additional metric to track total missing blocks

Author: chatton

internal/metrics/metrics.go

@@ -12,8 +12,9 @@ import (
 // Metrics contains Prometheus metrics for DA verification failures
 type Metrics struct {
 	// track ranges of unsubmitted blocks
-	UnsubmittedRangeStart *prometheus.GaugeVec
-	UnsubmittedRangeEnd   *prometheus.GaugeVec
+	UnsubmittedRangeStart  *prometheus.GaugeVec
+	UnsubmittedRangeEnd    *prometheus.GaugeVec
+	UnsubmittedBlocksTotal *prometheus.GaugeVec
 
 	mu     sync.Mutex
 	ranges map[string][]*blockRange // key: blobType -> sorted slice of ranges
@@ -49,14 +50,35 @@ func NewWithRegistry(namespace string, registerer prometheus.Registerer) *Metric
 			},
 			[]string{"chain_id", "blob_type", "range_id"},
 		),
+		UnsubmittedBlocksTotal: factory.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Name:      "unsubmitted_blocks_total",
+				Help:      "total number of unsubmitted blocks",
+			},
+			[]string{"chain_id", "blob_type"},
+		),
 		ranges: make(map[string][]*blockRange),
 	}
 }
 
+// RecordTotalMissingBlocks updates the total count of missing blocks metric.
+func (m *Metrics) RecordTotalMissingBlocks(chainID, blobType string) {
+	ranges := m.ranges[blobType]
+	total := uint64(0)
+	for _, r := range ranges {
+		total += r.end - r.start + 1 // inclusive count
+	}
+	m.UnsubmittedBlocksTotal.WithLabelValues(chainID, blobType).Set(float64(total))
+}
+
 // RecordMissingBlock records a block that is missing from Celestia
-func (m *Metrics) RecordMissingBlock(chain, blobType string, blockHeight uint64) {
+func (m *Metrics) RecordMissingBlock(chainID, blobType string, blockHeight uint64) {
 	m.mu.Lock()
-	defer m.mu.Unlock()
+	defer func() {
+		m.RecordTotalMissingBlocks(chainID, blobType)
+		m.mu.Unlock()
+	}()
 
 	ranges := m.ranges[blobType]
 	if ranges == nil {
@@ -81,12 +103,12 @@ func (m *Metrics) RecordMissingBlock(chain, blobType string, blockHeight uint64)
 		leftRange := ranges[idx-1]
 		rightRange := ranges[idx]
 
-		m.deleteRange(chain, blobType, leftRange)
-		m.deleteRange(chain, blobType, rightRange)
+		m.deleteRange(chainID, blobType, leftRange)
+		m.deleteRange(chainID, blobType, rightRange)
 
 		// extend left range to include right range
 		leftRange.end = rightRange.end
-		m.updateRange(chain, blobType, leftRange)
+		m.updateRange(chainID, blobType, leftRange)
 
 		// remove right range from slice
 		m.ranges[blobType] = append(ranges[:idx], ranges[idx+1:]...)
@@ -96,18 +118,18 @@ func (m *Metrics) RecordMissingBlock(chain, blobType string, blockHeight uint64)
 	if canMergeLeft {
 		// extend left range
 		leftRange := ranges[idx-1]
-		m.deleteRange(chain, blobType, leftRange)
+		m.deleteRange(chainID, blobType, leftRange)
 		leftRange.end = blockHeight
-		m.updateRange(chain, blobType, leftRange)
+		m.updateRange(chainID, blobType, leftRange)
 		return
 	}
 
 	if canMergeRight {
 		// extend right range
 		rightRange := ranges[idx]
-		m.deleteRange(chain, blobType, rightRange)
+		m.deleteRange(chainID, blobType, rightRange)
 		rightRange.start = blockHeight
-		m.updateRange(chain, blobType, rightRange)
+		m.updateRange(chainID, blobType, rightRange)
 		return
 	}
 
@@ -118,14 +140,17 @@ func (m *Metrics) RecordMissingBlock(chain, blobType string, blockHeight uint64)
 	}
 	// insert at idx
 	ranges = append(ranges[:idx], append([]*blockRange{newRange}, ranges[idx:]...)...)
-	m.updateRange(chain, blobType, newRange)
+	m.updateRange(chainID, blobType, newRange)
 	m.ranges[blobType] = ranges
 }
 
 // RemoveVerifiedBlock removes a block from the missing ranges when it gets verified
-func (m *Metrics) RemoveVerifiedBlock(chain, blobType string, blockHeight uint64) {
+func (m *Metrics) RemoveVerifiedBlock(chainID, blobType string, blockHeight uint64) {
 	m.mu.Lock()
-	defer m.mu.Unlock()
+	defer func() {
+		m.RecordTotalMissingBlocks(chainID, blobType)
+		m.mu.Unlock()
+	}()
 
 	ranges := m.ranges[blobType]
 	if ranges == nil {
@@ -147,7 +172,7 @@ func (m *Metrics) RemoveVerifiedBlock(chain, blobType string, blockHeight uint64
 
 	// range contains only this block, delete it.
 	if r.start == r.end {
-		m.deleteRange(chain, blobType, r)
+		m.deleteRange(chainID, blobType, r)
 		// remove range from slice
 		m.ranges[blobType] = append(ranges[:idx], ranges[idx+1:]...)
 		return
@@ -156,28 +181,28 @@ func (m *Metrics) RemoveVerifiedBlock(chain, blobType string, blockHeight uint64
 	// block is at start of range, shrink the range
 	if blockHeight == r.start {
 		// remove from start of range
-		m.deleteRange(chain, blobType, r)
+		m.deleteRange(chainID, blobType, r)
 		r.start++ // modify existing range
-		m.updateRange(chain, blobType, r)
+		m.updateRange(chainID, blobType, r)
 		return
 	}
 
 	// block is at end of range, shrink the range
 	if blockHeight == r.end {
 		// remove from end of range
-		m.deleteRange(chain, blobType, r)
+		m.deleteRange(chainID, blobType, r)
 		r.end-- // modify existing range
-		m.updateRange(chain, blobType, r)
+		m.updateRange(chainID, blobType, r)
 		return
 	}
 
 	// block is in middle of range, split into two ranges
 	oldEnd := r.end
-	m.deleteRange(chain, blobType, r)
+	m.deleteRange(chainID, blobType, r)
 
 	// update first range
 	r.end = blockHeight - 1
-	m.updateRange(chain, blobType, r)
+	m.updateRange(chainID, blobType, r)
 
 	// create new range for the second part
 	newRange := &blockRange{
@@ -186,7 +211,7 @@ func (m *Metrics) RemoveVerifiedBlock(chain, blobType string, blockHeight uint64
 	}
 	// insert after current range
 	ranges = append(ranges[:idx+1], append([]*blockRange{newRange}, ranges[idx+1:]...)...)
-	m.updateRange(chain, blobType, newRange)
+	m.updateRange(chainID, blobType, newRange)
 
 	m.ranges[blobType] = ranges
 }

internal/metrics/metrics_test.go

@@ -118,6 +118,20 @@ func TestMetrics_RecordMissingBlock(t *testing.T) {
 				}
 				require.True(t, found, "expected to find range %s [%d-%d]", expected.blobType, expected.start, expected.end)
 			}
+
+			// verify total missing blocks metric for each blob type
+			blobTypes := make(map[string]bool)
+			for _, r := range tt.expectedRanges {
+				blobTypes[r.blobType] = true
+			}
+			for blobType := range blobTypes {
+				expectedTotal := calculateExpectedTotal(tt.expectedRanges, blobType)
+				actualTotal := getMetricValue(t, reg, "test_unsubmitted_blocks_total", map[string]string{
+					"chain_id":  "testchain",
+					"blob_type": blobType,
+				})
+				require.Equal(t, float64(expectedTotal), actualTotal, "total missing blocks for %s should be %d", blobType, expectedTotal)
+			}
 		})
 	}
 }
@@ -256,6 +270,13 @@ func TestMetrics_RemoveVerifiedBlock(t *testing.T) {
 					totalRanges += len(ranges)
 				}
 				require.Equal(t, 0, totalRanges, "expected no ranges")
+
+				// verify total blocks metric is 0
+				actualTotal := getMetricValue(t, reg, "test_unsubmitted_blocks_total", map[string]string{
+					"chain_id":  tt.removeBlock.chain,
+					"blob_type": tt.removeBlock.blobType,
+				})
+				require.Equal(t, float64(0), actualTotal, "total missing blocks should be 0")
 			} else {
 				totalRanges := 0
 				for _, ranges := range m.ranges {
@@ -280,6 +301,20 @@ func TestMetrics_RemoveVerifiedBlock(t *testing.T) {
 					}
 					require.True(t, found, "expected to find range %s [%d-%d]", expected.blobType, expected.start, expected.end)
 				}
+
+				// verify total missing blocks metric for each blob type
+				blobTypes := make(map[string]bool)
+				for _, r := range tt.expectedRanges {
+					blobTypes[r.blobType] = true
+				}
+				for blobType := range blobTypes {
+					expectedTotal := calculateExpectedTotal(tt.expectedRanges, blobType)
+					actualTotal := getMetricValue(t, reg, "test_unsubmitted_blocks_total", map[string]string{
+						"chain_id":  tt.removeBlock.chain,
+						"blob_type": blobType,
+					})
+					require.Equal(t, float64(expectedTotal), actualTotal, "total missing blocks for %s should be %d", blobType, expectedTotal)
+				}
 			}
 		})
 	}
@@ -300,6 +335,13 @@ func TestMetrics_ComplexScenario(t *testing.T) {
 	}
 	require.Equal(t, 1, totalRanges, "should start with one range")
 
+	// verify total blocks: 100-110 = 11 blocks
+	actualTotal := getMetricValue(t, reg, "test_unsubmitted_blocks_total", map[string]string{
+		"chain_id":  "testchain",
+		"blob_type": "header",
+	})
+	require.Equal(t, float64(11), actualTotal, "should have 11 total missing blocks")
+
 	// remove block 103 - splits into two ranges
 	m.RemoveVerifiedBlock("testchain", "header", 103)
 	totalRanges = 0
@@ -308,6 +350,13 @@ func TestMetrics_ComplexScenario(t *testing.T) {
 	}
 	require.Equal(t, 2, totalRanges, "should have two ranges after first split")
 
+	// verify total blocks: 100-102 (3) + 104-110 (7) = 10 blocks
+	actualTotal = getMetricValue(t, reg, "test_unsubmitted_blocks_total", map[string]string{
+		"chain_id":  "testchain",
+		"blob_type": "header",
+	})
+	require.Equal(t, float64(10), actualTotal, "should have 10 total missing blocks after removing block 103")
+
 	// remove block 107 - splits second range
 	m.RemoveVerifiedBlock("testchain", "header", 107)
 	totalRanges = 0
@@ -316,6 +365,13 @@ func TestMetrics_ComplexScenario(t *testing.T) {
 	}
 	require.Equal(t, 3, totalRanges, "should have three ranges after second split")
 
+	// verify total blocks: 100-102 (3) + 104-106 (3) + 108-110 (3) = 9 blocks
+	actualTotal = getMetricValue(t, reg, "test_unsubmitted_blocks_total", map[string]string{
+		"chain_id":  "testchain",
+		"blob_type": "header",
+	})
+	require.Equal(t, float64(9), actualTotal, "should have 9 total missing blocks after removing block 107")
+
 	// verify final ranges: 100-102, 104-106, 108-110
 	expectedRanges := []expectedRange{
 		{blobType: "header", start: 100, end: 102},
@@ -352,6 +408,13 @@ func TestMetrics_ComplexScenario(t *testing.T) {
 	}
 	require.Equal(t, 2, totalRanges, "should have two ranges after removing first range")
 
+	// verify total blocks: 104-106 (3) + 108-110 (3) = 6 blocks
+	actualTotal = getMetricValue(t, reg, "test_unsubmitted_blocks_total", map[string]string{
+		"chain_id":  "testchain",
+		"blob_type": "header",
+	})
+	require.Equal(t, float64(6), actualTotal, "should have 6 total missing blocks after removing first range")
+
 	// verify remaining ranges: 104-106, 108-110
 	expectedRanges = []expectedRange{
 		{blobType: "header", start: 104, end: 106},
@@ -395,3 +458,43 @@ type expectedRange struct {
 	start    uint64
 	end      uint64
 }
+
+// calculateExpectedTotal calculates the total number of blocks from expected ranges
+func calculateExpectedTotal(ranges []expectedRange, blobType string) uint64 {
+	total := uint64(0)
+	for _, r := range ranges {
+		if r.blobType == blobType {
+			total += r.end - r.start + 1
+		}
+	}
+	return total
+}
+
+// getMetricValue retrieves the current value of a gauge metric
+func getMetricValue(t *testing.T, reg *prometheus.Registry, metricName string, labels map[string]string) float64 {
+	t.Helper()
+	metrics, err := reg.Gather()
+	require.NoError(t, err)
+
+	for _, mf := range metrics {
+		if mf.GetName() == metricName {
+			for _, m := range mf.GetMetric() {
+				// check if labels match
+				match := true
+				for _, label := range m.GetLabel() {
+					if expectedVal, ok := labels[label.GetName()]; ok {
+						if label.GetValue() != expectedVal {
+							match = false
+							break
+						}
+					}
+				}
+				if match && len(m.GetLabel()) == len(labels) {
+					return m.GetGauge().GetValue()
+				}
+			}
+		}
+	}
+	t.Fatalf("metric %s not found with labels %v", metricName, labels)
+	return 0
+}