Enable profiling in pull mode. Enable profiling in k8s and docker microservice setups

Author: ppcano

alloy/cloud-microservices.alloy

@@ -11,6 +11,7 @@ grafana_cloud.stack "receivers" {
   token = env("GRAFANA_CLOUD_TOKEN")
 }
 
+// discover containers running QuickPizza
 discovery.docker "application_containers" {
   host = "unix:///var/run/docker.sock"
   filter {
@@ -19,6 +20,7 @@ discovery.docker "application_containers" {
   }
 }
 
+// set expected labels
 discovery.relabel "application_containers" {
   rule {
     target_label = "job"
@@ -40,6 +42,11 @@ discovery.relabel "application_containers" {
     target_label = "service_namespace"
     replacement = "quickpizza"
   }
+  // the `namespace` label is for visualizing Profiles in Application Observability
+  rule {
+    target_label = "namespace"
+    replacement = "quickpizza"
+  }
   rule {
     target_label = "service_name"
     source_labels = [
@@ -50,11 +57,12 @@ discovery.relabel "application_containers" {
   }
   targets = discovery.docker.application_containers.targets
 }
+
 // Metrics
 prometheus.scrape "application_containers" {
   scrape_interval = "10s"
-  forward_to = [grafana_cloud.stack.receivers.metrics]
   targets = discovery.relabel.application_containers.output
+  forward_to = [grafana_cloud.stack.receivers.metrics]
 }
 // Logs
 loki.source.docker "application_containers" {
@@ -63,13 +71,12 @@ loki.source.docker "application_containers" {
   forward_to = [grafana_cloud.stack.receivers.logs]
 }
 
-// Profiling
-pyroscope.receive_http "default" {
-   http {
-       listen_address = "0.0.0.0"
-       listen_port = 9999
-   }
-   forward_to = [grafana_cloud.stack.receivers.profiles]
+// Profiling Pull Mode
+pyroscope.scrape "application_containers" {
+  // https://grafana.com/docs/pyroscope/latest/configure-client/grafana-alloy/go_pull/
+  scrape_interval = "30s"
+  targets = discovery.relabel.application_containers.output
+  forward_to = [grafana_cloud.stack.receivers.profiles]
 }
 
 // Traces

cmd/main.go

@@ -27,10 +27,11 @@ func main() {
 		Level: logging.GetLogLevel(),
 	})))
 
-	// Enable profiling middleware if Pyroscope is enabled
+	// Profiling in pull mode is enabled by default.
+	// If QUICKPIZZA_PYROSCOPE_ENDPOINT is set, profiling in push mode will be enabled.
 	profilingConfig, profilingEnabled := envPyroscopeConfig()
 	if profilingEnabled {
-		slog.Debug("enabling Pyroscope profiling")
+		slog.Debug("enabling Pyroscope profiling in push mode")
 
 		runtime.SetMutexProfileFraction(5)
 		runtime.SetBlockProfileRate(5)

docker-compose-cloud.microservices.yaml

@@ -1,6 +1,5 @@
 x-quickpizza-env-common: &quickpizza-env-common
   QUICKPIZZA_OTLP_ENDPOINT: http://alloy:4318
-  QUICKPIZZA_PYROSCOPE_ENDPOINT: http://alloy:9999
   QUICKPIZZA_TRUST_CLIENT_TRACEID: true
   QUICKPIZZA_ALL_SERVICES: 0
   QUICKPIZZA_CATALOG_ENDPOINT: http://catalog:3333

kubernetes/cloud-dev/config/config.alloy

@@ -45,6 +45,14 @@ discovery.relabel "application_pods" {
       "__meta_kubernetes_pod_label_app_k8s_io_name",
     ]
   }
+  // the `namespace` label is for visualizing Profiles in Application Observability
+  rule {
+    target_label = "namespace"
+    source_labels = [
+      // quickpizza
+      "__meta_kubernetes_pod_label_app_k8s_io_name",
+    ]
+  }
   rule {
     target_label = "service_name"
     source_labels = [
@@ -67,14 +75,13 @@ loki.source.kubernetes "application_pods" {
   targets = discovery.relabel.application_pods.output
 }
 
-// TODO: not working
-// pyroscope.receive_http "default" {
-   // forward_to = [grafana_cloud.stack.receivers.profiles]
-   // http {
-       // listen_address = "127.0.0.1"
-       // listen_port = 9999
-   // }
-// }
+// Profiling Pull Mode
+pyroscope.scrape "application_pods" {
+  // https://grafana.com/docs/pyroscope/latest/configure-client/grafana-alloy/go_pull/
+  scrape_interval = "30s"
+  targets = discovery.relabel.application_pods.output
+  forward_to = [grafana_cloud.stack.receivers.profiles]
+}
 
 // Traces
 otelcol.receiver.otlp "default" {

kubernetes/cloud-dev/kustomization.yaml

@@ -28,8 +28,6 @@ configMapGenerator:
     literals:
       # Enable tracing
       - QUICKPIZZA_OTLP_ENDPOINT=http://grafana-alloy:4318
-      # TODO: pyroscope not working
-      - QUICKPIZZA_PYROSCOPE_ENDPOINT=http://grafana-alloy:9999
   - name: alloy-config
     files:
       - ./config/config.alloy

pkg/http/http.go

@@ -15,6 +15,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/http/httputil"
+	_ "net/http/pprof"
 	"net/url"
 	"os"
 	"slices"
@@ -259,6 +260,9 @@ func NewServer(profiling bool, traceInstaller *TraceInstaller) *Server {
 		}).Handler,
 	)
 
+	// Enable Profiling Pull Mode
+	router.Mount("/debug/pprof/", http.DefaultServeMux)
+
 	if profiling {
 		router.Use(k6.LabelsFromBaggageHandler)
 	}