fix(kuma-dp): add logging for MeshMetric application scraping failures (backport of #15513) (#15518)

Automatic cherry-pick of #15513 for branch release-2.11

Generated by
[action](https://github.com/kumahq/kuma/actions/runs/21663883137)

cherry-picked commit 9074d92

---------

Signed-off-by: Marcin Skalski <skalskimarcin33@gmail.com>
Co-authored-by: Marcin Skalski <marcin.skalski@konghq.com>
Co-authored-by: Marcin Skalski <skalskimarcin33@gmail.com>

Author: kumahq[bot]

app/kuma-dp/pkg/dataplane/metrics/metrics_format_mapper.go

@@ -25,6 +25,10 @@ func FromPrometheusMetrics(appMetrics map[string]*io_prometheus_client.MetricFam
 	extraAttributes := extraAttributesFrom(mesh, dataplane, service, extraLabels)
 
 	scopedMetrics := map[instrumentation.Scope][]metricdata.Metrics{}
+	if len(appMetrics) == 0 {
+		log.V(1).Info("FromPrometheusMetrics called with empty input")
+		return scopedMetrics
+	}
 	for _, prometheusMetric := range appMetrics {
 		var scopedAggregations map[instrumentation.Scope]metricdata.Aggregation
 		switch prometheusMetric.GetType() {

app/kuma-dp/pkg/dataplane/metrics/metrics_producer.go

@@ -55,6 +55,14 @@ func (ap *AggregatedProducer) Produce(ctx context.Context) ([]metricdata.ScopeMe
 	appsToScrape = append(appsToScrape, ap.applicationsToScrape...)
 	ap.applicationsToScrapeMutex.Unlock()
 
+	if len(appsToScrape) > 0 {
+		appNames := make([]string, 0, len(appsToScrape))
+		for _, app := range appsToScrape {
+			appNames = append(appNames, app.Name)
+		}
+		log.V(1).Info("starting metrics collection", "applications", appNames, "count", len(appsToScrape))
+	}
+
 	out := make(chan map[instrumentation.Scope][]metricdata.Metrics, len(appsToScrape))
 	var wg sync.WaitGroup
 	done := make(chan []byte)
@@ -77,7 +85,11 @@ func (ap *AggregatedProducer) Produce(ctx context.Context) ([]metricdata.ScopeMe
 	case <-ctx.Done():
 		return nil, nil
 	case <-done:
-		return combineMetrics(out), nil
+		result := combineMetrics(out)
+		if len(result) == 0 && len(appsToScrape) > 0 {
+			log.Info("metrics collection completed with zero metrics", "applicationCount", len(appsToScrape))
+		}
+		return result, nil
 	}
 }
 
@@ -115,13 +127,20 @@ func (ap *AggregatedProducer) fetchStats(ctx context.Context, app ApplicationToS
 		return nil
 	}
 	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		log.Info("application returned non-200 status", "name", app.Name, "status", resp.StatusCode, "path", app.Path, "port", app.Port)
+		return nil
+	}
 	requestTime := time.Now().UTC()
 
 	metricsFromApplication, err := app.MeshMetricMutator(resp.Body)
 	if err != nil {
 		log.Error(err, "failed to mutate metrics")
 		return nil
 	}
+	if len(metricsFromApplication) == 0 {
+		log.Info("application returned empty metrics after parsing", "name", app.Name, "path", app.Path, "port", app.Port)
+	}
 	return FromPrometheusMetrics(metricsFromApplication, ap.mesh, ap.dataplane, ap.service, ap.kumaVersion, app.ExtraLabels, requestTime)
 }
 

app/kuma-dp/pkg/dataplane/metrics/server.go

@@ -361,6 +361,11 @@ func (s *Hijacker) getStats(ctx context.Context, initReq *http.Request, app Appl
 		logger.Error(err, "failed call", "name", app.Name, "path", app.Path, "port", app.Port)
 		return nil, ""
 	}
+	if resp.StatusCode != http.StatusOK {
+		bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
+		logger.Info("application returned non-200 status", "name", app.Name, "status", resp.StatusCode, "path", app.Path, "port", app.Port, "body", string(bodyBytes))
+		return nil, ""
+	}
 
 	respContentType := responseFormat(resp.Header)
 
@@ -379,6 +384,9 @@ func (s *Hijacker) getStats(ctx context.Context, initReq *http.Request, app Appl
 			return nil, ""
 		}
 	}
+	if len(bodyBytes) == 0 {
+		logger.V(1).Info("application returned empty body", "name", app.Name, "path", app.Path, "port", app.Port)
+	}
 	return bodyBytes, respContentType
 }