app/health: add fallback beacon node usage health check (#3667)

Add health check which monitor increase in usage of fallback beacon nodes.

category: feature
ticket: #3328

Author: DiogoSantoss

app/health/checks.go

@@ -145,6 +145,19 @@ var checks = []check{
 				return false, err
 			}
 
+			return maxVal > 0, nil
+		},
+	},
+	{
+		Name:        "using_fallback_beacon_nodes",
+		Description: "Using fallback beacon nodes. Please check primary beacon nodes health.",
+		Severity:    severityWarning,
+		Func: func(q query, _ Metadata) (bool, error) {
+			maxVal, err := q("app_eth2_using_fallback", sumLabels(), gaugeMax)
+			if err != nil {
+				return false, err
+			}
+
 			return maxVal > 0, nil
 		},
 	},

app/health/checks_internal_test.go

@@ -395,6 +395,32 @@ func TestMetricsHighCardinalityCheck(t *testing.T) {
 	})
 }
 
+func TestUsingFallbackBeaconNodesCheck(t *testing.T) {
+	m := Metadata{}
+	checkName := "using_fallback_beacon_nodes"
+	metricName := "app_eth2_using_fallback"
+
+	t.Run("no data", func(t *testing.T) {
+		testCheck(t, m, checkName, false, nil)
+	})
+
+	t.Run("no fallback", func(t *testing.T) {
+		testCheck(t, m, checkName, false,
+			genFam(metricName,
+				genGauge(nil, 0, 0, 0),
+			),
+		)
+	})
+
+	t.Run("single fallback", func(t *testing.T) {
+		testCheck(t, m, checkName, true,
+			genFam(metricName,
+				genGauge(nil, 0, 1, 0),
+			),
+		)
+	})
+}
+
 func testCheck(t *testing.T, m Metadata, checkName string, expect bool, metrics []*pb.MetricFamily) {
 	t.Helper()