exporters/prometheus: sort labels (#2278)

Some prometheus implementations require the labels to be sorted, or they
will reject write requests

Author: fredr

runtimes/core/src/metrics/exporter/prometheus.rs

@@ -166,6 +166,9 @@ impl Prometheus {
                 value: metric_name,
             });
 
+            // Sort labels lexicographically by name, as required by some Prometheus implementations.
+            labels.sort_unstable_by(|a, b| a.name.cmp(&b.name));
+
             // Convert metric value to float64
             let value = match metric.value {
                 MetricValue::CounterU64(val) => val as f64,
@@ -212,3 +215,66 @@ fn from_time(t: SystemTime) -> i64 {
 mod prompb {
     include!(concat!(env!("OUT_DIR"), "/prometheus.rs"));
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::encore::runtime::v1 as pb;
+    use std::time::SystemTime;
+
+    /// Test that labels are sorted lexicographically by name.
+    /// Some Prometheus implementations require this or they will reject the request
+    /// with "out of order labels" error.
+    #[tokio::test]
+    async fn test_labels_are_sorted() {
+        let env = pb::Environment::default();
+        let container_meta_client = ContainerMetaClient::new(env, reqwest::Client::new());
+
+        // Create a metric with labels in non-sorted order: "zebra", "apple", "middle"
+        let key = metrics::Key::from_parts(
+            "test_metric",
+            vec![
+                metrics::Label::new("zebra", "last"),
+                metrics::Label::new("apple", "first"),
+                metrics::Label::new("middle", "mid"),
+            ],
+        );
+
+        let collected = vec![CollectedMetric {
+            key,
+            value: MetricValue::CounterU64(42),
+            registered_at: SystemTime::now(),
+        }];
+
+        let prometheus = Prometheus {
+            client: reqwest::Client::new(),
+            remote_write_url: Url::parse("http://localhost:9090/api/v1/write").unwrap(),
+            container_meta_client,
+            container_labels: OnceCell::new(),
+        };
+
+        let time_series = prometheus.get_metric_data(collected).await;
+        assert_eq!(time_series.len(), 1);
+
+        let label_names: Vec<&str> = time_series[0]
+            .labels
+            .iter()
+            .map(|l| l.name.as_str())
+            .collect();
+
+        // Labels should be sorted lexicographically
+        assert_eq!(
+            label_names,
+            vec![
+                "__name__",
+                "apple",
+                "env_name",
+                "instance_id",
+                "middle",
+                "revision_id",
+                "service_id",
+                "zebra"
+            ]
+        );
+    }
+}

runtimes/go/appruntime/infrasdk/metrics/prometheus/prometheus.go

@@ -7,6 +7,8 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"slices"
+	"strings"
 	"time"
 
 	"github.com/golang/protobuf/proto"
@@ -78,6 +80,10 @@ func (x *Exporter) getMetricData(now time.Time, collected []metrics.CollectedMet
 		copy(labels, baseLabels)
 		labels[len(baseLabels)] = &prompb.Label{Name: "__name__", Value: metricName}
 		labels[len(baseLabels)+1] = &prompb.Label{Name: "service", Value: x.svcs[svcIdx]}
+		// Sort labels lexicographically by name, as required by some Prometheus implementations.
+		slices.SortFunc(labels, func(a, b *prompb.Label) int {
+			return strings.Compare(a.Name, b.Name)
+		})
 		data = append(data, &prompb.TimeSeries{
 			Labels: labels,
 			Samples: []*prompb.Sample{
@@ -166,6 +172,10 @@ func (x *Exporter) getSysMetrics(now time.Time) []*prompb.TimeSeries {
 			Name:  "__name__",
 			Value: metricName,
 		}
+		// Sort labels lexicographically by name, as required by some Prometheus implementations.
+		slices.SortFunc(labels, func(a, b *prompb.Label) int {
+			return strings.Compare(a.Name, b.Name)
+		})
 		return labels
 	}
 

runtimes/go/appruntime/infrasdk/metrics/prometheus/prometheus_test.go

@@ -107,14 +107,14 @@ func TestGetMetricData(t *testing.T) {
 			data: []*prompb.TimeSeries{
 				{
 					Labels: []*prompb.Label{
-						{
-							Name:  "key",
-							Value: "value",
-						},
 						{
 							Name:  "__name__",
 							Value: "test_labels",
 						},
+						{
+							Name:  "key",
+							Value: "value",
+						},
 						{
 							Name:  "service",
 							Value: "foo",