fix(multicluster): Do not ignore timeout and gateway metrics errors during multicluster check (#14418)

Signed-off-by: Zahari Dichev <[email protected]>

Author: zaharidichev

multicluster/cmd/check.go

@@ -730,13 +730,25 @@ func (hc *healthChecker) checkIfGatewayMirrorsHaveEndpoints(ctx context.Context,
 
 		// Get and parse the gateway metrics so that we can extract liveness
 		// information.
-		gatewayMetrics := getGatewayMetrics(hc.KubeAPIClient(), pods.Items, leaders, wait)
+		gatewayMetrics, err := getGatewayMetrics(hc.KubeAPIClient(), pods.Items, leaders, wait)
+		if err != nil {
+			errors = append(errors, fmt.Errorf("failed to get gateway metrics for target cluster %s: %w", link.Spec.TargetClusterName, err))
+			continue
+		}
+
 		if len(gatewayMetrics) != 1 {
 			errors = append(errors, fmt.Errorf("expected exactly one gateway metric for target cluster %s; got %d", link.Spec.TargetClusterName, len(gatewayMetrics)))
 			continue
 		}
+
+		gatewayMetric := gatewayMetrics[0]
+		if gatewayMetric.err != nil {
+			errors = append(errors, fmt.Errorf("Failed to get gateway status for %s: %w\n", gatewayMetric.clusterName, gatewayMetric.err))
+			continue
+		}
+
 		var metricsParser expfmt.TextParser
-		parsedMetrics, err := metricsParser.TextToMetricFamilies(bytes.NewReader(gatewayMetrics[0].metrics))
+		parsedMetrics, err := metricsParser.TextToMetricFamilies(bytes.NewReader(gatewayMetric.metrics))
 		if err != nil {
 			errors = append(errors, fmt.Errorf("failed to parse gateway metrics for target cluster %s: %w", link.Spec.TargetClusterName, err))
 			continue

multicluster/cmd/gateways.go

@@ -98,7 +98,12 @@ func newGatewaysCommand() *cobra.Command {
 			}
 
 			var statuses []gatewayStatus
-			gatewayMetrics := getGatewayMetrics(k8sAPI, pods.Items, leaders, opts.wait)
+			gatewayMetrics, err := getGatewayMetrics(k8sAPI, pods.Items, leaders, opts.wait)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Failed to get gateway metrics for cluster %s: %s\n", opts.clusterName, err)
+				os.Exit(1)
+			}
+
 			for _, gateway := range gatewayMetrics {
 				if gateway.err != nil {
 					fmt.Fprintf(os.Stderr, "Failed to get gateway status for %s: %s\n", gateway.clusterName, gateway.err)
@@ -195,7 +200,7 @@ func newGatewaysCommand() *cobra.Command {
 	return cmd
 }
 
-func getGatewayMetrics(k8sAPI *k8s.KubernetesAPI, pods []corev1.Pod, leaders map[string]struct{}, wait time.Duration) []gatewayMetrics {
+func getGatewayMetrics(k8sAPI *k8s.KubernetesAPI, pods []corev1.Pod, leaders map[string]struct{}, wait time.Duration) ([]gatewayMetrics, error) {
 	var metrics []gatewayMetrics
 	metricsChan := make(chan gatewayMetrics)
 	var wg sync.WaitGroup
@@ -250,11 +255,11 @@ wait:
 			}
 			metrics = append(metrics, metric)
 		case <-timeout.C:
-			break wait
+			return nil, fmt.Errorf("timed out waiting for metrics")
 		}
 	}
 
-	return metrics
+	return metrics, nil
 }
 
 func getServiceMirrorContainer(pod corev1.Pod) (corev1.Container, error) {