fix: enable native histogram metrics for all deployments (#372)

Author: ppcano

compose.grafana-local-stack.microservices.yaml

@@ -164,15 +164,16 @@ services:
       - "3100:3100"
 
   prometheus:
-    image: prom/prometheus:v3.2.1
+    image: prom/prometheus:v3.9.1
     container_name: prometheus
     labels:
       - "service.type=instrumentation"
+    volumes:
+      - ./deployments/docker-compose/grafana-local-stack/prometheus.yaml:/etc/prometheus.yaml
     command:
       - --web.enable-remote-write-receiver
       - --enable-feature=exemplar-storage
-      - --enable-feature=native-histograms
-      - --config.file=/etc/prometheus/prometheus.yml
+      - --config.file=/etc/prometheus.yaml
     ports:
       - "9090:9090"
 

compose.grafana-local-stack.monolithic.yaml

@@ -87,15 +87,16 @@ services:
       - "3100:3100"
 
   prometheus:
-    image: prom/prometheus:v3.2.1
+    image: prom/prometheus:v3.9.1
     container_name: prometheus
     labels:
       - "service.type=instrumentation"
+    volumes:
+      - ./deployments/docker-compose/grafana-local-stack/prometheus.yaml:/etc/prometheus.yaml
     command:
       - --web.enable-remote-write-receiver
       - --enable-feature=exemplar-storage
-      - --enable-feature=native-histograms
-      - --config.file=/etc/prometheus/prometheus.yml
+      - --config.file=/etc/prometheus.yaml
     ports:
       - "9090:9090"
 

deployments/docker-compose/grafana-cloud/config.alloy

@@ -71,11 +71,24 @@ discovery.relabel "app_containers" {
   targets = discovery.docker.all_containers.targets
 }
 
-// Metrics
+// Use explicit remote_write to enable native histograms
+// (grafana_cloud.stack.receivers.metrics doesn't support send_native_histograms )
+prometheus.remote_write "grafana_cloud" {
+  endpoint {
+    url = grafana_cloud.stack.receivers.info["hmInstancePromUrl"] + "/api/prom/push"
+    basic_auth {
+      username = grafana_cloud.stack.receivers.info["hmInstancePromId"]
+      password = env("GRAFANA_CLOUD_TOKEN")
+    }
+    send_native_histograms = true
+  }
+}
+
 prometheus.scrape "app_containers" {
   scrape_interval = "10s"
+  scrape_native_histograms = true
   targets = concat(discovery.relabel.app_containers.output, discovery.relabel.database_observability_postgres_quickpizza.output)
-  forward_to = [grafana_cloud.stack.receivers.metrics]
+  forward_to = [prometheus.remote_write.grafana_cloud.receiver]
 }
 // Logs
 loki.source.docker "app_containers" {

deployments/docker-compose/grafana-local-stack/config.alloy

@@ -99,6 +99,7 @@ prometheus.remote_write "local" {
 }
 prometheus.scrape "app_containers" {
   scrape_interval = "10s"
+  scrape_native_histograms = true
   targets = concat(discovery.relabel.app_containers.output, discovery.relabel.postgres.output)
   forward_to = [prometheus.remote_write.local.receiver]
 }

deployments/docker-compose/grafana-local-stack/prometheus.yaml

@@ -0,0 +1,6 @@
+---
+global:
+  scrape_native_histograms: true
+storage:
+  tsdb:
+    out_of_order_time_window: 10m

deployments/kubernetes/cloud-testing/config/config.alloy

@@ -61,10 +61,23 @@ discovery.relabel "application_pods" {
   targets = discovery.kubernetes.application_pods.targets
 }
 
+// Use explicit remote_write to enable native histograms
+// (grafana_cloud.stack.receivers.metrics doesn't support send_native_histograms )
+prometheus.remote_write "grafana_cloud" {
+  endpoint {
+    url = grafana_cloud.stack.receivers.info["hmInstancePromUrl"] + "/api/prom/push"
+    basic_auth {
+      username = grafana_cloud.stack.receivers.info["hmInstancePromId"]
+      password = env("GRAFANA_CLOUD_TOKEN")
+    }
+    send_native_histograms = true
+  }
+}
 // Metrics: application pods
 prometheus.scrape "application_pods" {
   scrape_interval = "10s"
-  forward_to = [grafana_cloud.stack.receivers.metrics]
+  scrape_native_histograms = true
+  forward_to = [prometheus.remote_write.grafana_cloud.receiver]
   targets = discovery.relabel.application_pods.output
 }
 

deployments/kubernetes/cloud/config/config.alloy

@@ -67,11 +67,23 @@ discovery.relabel "application_pods" {
   targets = discovery.kubernetes.application_pods.targets
 }
 
-
+// Use explicit remote_write to enable native histograms
+// (grafana_cloud.stack.receivers.metrics doesn't support send_native_histograms )
+prometheus.remote_write "grafana_cloud" {
+  endpoint {
+    url = grafana_cloud.stack.receivers.info["hmInstancePromUrl"] + "/api/prom/push"
+    basic_auth {
+      username = grafana_cloud.stack.receivers.info["hmInstancePromId"]
+      password = env("GRAFANA_CLOUD_TOKEN")
+    }
+    send_native_histograms = true
+  }
+}
 // Metrics: application pods
 prometheus.scrape "application_pods" {
   scrape_interval = "10s"
-  forward_to = [grafana_cloud.stack.receivers.metrics]
+  scrape_native_histograms = true
+  forward_to = [prometheus.remote_write.grafana_cloud.receiver]
   targets = discovery.relabel.application_pods.output
 }
 

deployments/terraform/alloy/config.alloy

@@ -80,12 +80,25 @@ discovery.relabel "application_pods" {
   targets = discovery.kubernetes.application_pods.targets
 }
 
+// Use explicit remote_write to enable native histograms
+// (grafana_cloud.stack.receivers.metrics doesn't support send_native_histograms )
+prometheus.remote_write "grafana_cloud" {
+  endpoint {
+    url = grafana_cloud.stack.receivers.info["hmInstancePromUrl"] + "/api/prom/push"
+    basic_auth {
+      username = grafana_cloud.stack.receivers.info["hmInstancePromId"]
+      password = env("GRAFANA_CLOUD_TOKEN")
+    }
+    send_native_histograms = true
+  }
+}
 
 // Metrics: application pods
 prometheus.scrape "application_pods" {
   scrape_interval = "10s"
+  scrape_native_histograms = true
+  forward_to = [prometheus.remote_write.grafana_cloud.receiver]
   targets = concat(discovery.relabel.application_pods.output, discovery.relabel.database_observability_postgres_quickpizza.output)
-  forward_to = [grafana_cloud.stack.receivers.metrics]
 }
 
 // Logs: application pods