H-5328: Add Pyroscope continuous profiling infrastructure to AWS observability stack (#7787)

Co-authored-by: Copilot <[email protected]>

Author: TimDiekmann

infra/terraform/hash/observability/alloy/config.tf

@@ -7,9 +7,15 @@ resource "aws_s3_object" "alloy_config" {
 
   # Grafana Alloy configuration
   content = templatefile("${path.module}/templates/alloy-config.alloy.tpl", {
-    region          = var.region
-    mimir_http_dns  = var.mimir_http_dns
-    mimir_http_port = var.mimir_http_port
+    environment           = terraform.workspace
+    region                = var.region
+    profile_port_internal = local.profile_port_internal
+    profile_port_external = local.profile_port_external
+    mimir_http_dns        = var.mimir_http_dns
+    mimir_http_port       = var.mimir_http_port
+    pyroscope_http_dns    = var.pyroscope_http_dns
+    pyroscope_http_port   = var.pyroscope_http_port
+
   })
 
   content_type = "text/plain"

infra/terraform/hash/observability/alloy/main.tf

@@ -5,11 +5,15 @@ locals {
   prefix       = "${var.prefix}-${local.service_name}"
 
   # Port definitions for Alloy
-  http_port = 5000 # Alloy HTTP API for metrics scraping
+  http_port             = 5000 # Alloy HTTP API for metrics scraping
+  profile_port_internal = 4040
+  profile_port_external = 4042
 
   # Port names for Service Connect
-  http_port_name = "${local.service_name}-http" # HTTP API
+  http_port_name    = "${local.service_name}-http" # HTTP API
+  profile_port_name = "${local.service_name}-profile"
 
   # DNS names for Service Connect
-  http_port_dns = "${local.http_port_name}.${var.service_discovery_namespace_name}"
+  http_port_dns    = "${local.http_port_name}.${var.service_discovery_namespace_name}"
+  profile_port_dns = "${local.profile_port_name}.${var.service_discovery_namespace_name}"
 }

infra/terraform/hash/observability/alloy/outputs.tf

@@ -2,7 +2,15 @@ output "http_port" {
   description = "Port number for Grafana Alloy HTTP API"
   value       = local.http_port
 }
+output "profile_port_internal" {
+  description = "Port number for Grafana Alloy Profile API"
+  value       = local.profile_port_internal
+}
 
+output "profile_port_external" {
+  description = "Port number for Grafana Alloy Profile API"
+  value       = local.profile_port_external
+}
 output "http_port_name" {
   description = "Port name for Service Connect"
   value       = local.http_port_name
@@ -12,3 +20,17 @@ output "http_port_dns" {
   description = "Service Connect DNS name for Grafana Alloy metrics endpoint"
   value       = local.http_port_dns
 }
+
+# Target groups for load balancer attachment
+
+# Internal target groups (service-to-service communication)
+output "profile_internal_target_group_arn" {
+  description = "Internal profile target group ARN"
+  value       = aws_lb_target_group.profile_internal.arn
+}
+
+# External target groups (client applications)
+output "profile_external_target_group_arn" {
+  description = "External profile target group ARN"
+  value       = aws_lb_target_group.profile_external.arn
+}

infra/terraform/hash/observability/alloy/security_groups.tf

@@ -13,6 +13,22 @@ resource "aws_security_group" "alloy" {
     cidr_blocks = [var.vpc.cidr_block]
   }
 
+  # Allow inbound HTTP for profile ingestion
+  ingress {
+    from_port   = local.profile_port_internal
+    to_port     = local.profile_port_internal
+    protocol    = "tcp"
+    description = "Grafana Alloy profile ingestion endpoint"
+    cidr_blocks = [var.vpc.cidr_block]
+  }
+  ingress {
+    from_port   = local.profile_port_external
+    to_port     = local.profile_port_external
+    protocol    = "tcp"
+    description = "Grafana Alloy profile ingestion endpoint"
+    cidr_blocks = [var.vpc.cidr_block]
+  }
+
   # Allow outbound HTTPS for CloudWatch API calls and S3 config download
   egress {
     from_port   = 443
@@ -48,6 +64,15 @@ resource "aws_security_group" "alloy" {
     cidr_blocks = [var.vpc.cidr_block]
   }
 
+  # Allow outbound HTTP to Pyroscope for profile forwarding
+  egress {
+    from_port   = var.pyroscope_http_port
+    to_port     = var.pyroscope_http_port
+    protocol    = "tcp"
+    description = "HTTP to Pyroscope for profile forwarding"
+    cidr_blocks = [var.vpc.cidr_block]
+  }
+
   tags = {
     Name    = "${local.prefix}-sg"
     Purpose = "Grafana Alloy security group"

infra/terraform/hash/observability/alloy/service.tf

@@ -91,10 +91,22 @@ resource "aws_ecs_task_definition" "alloy" {
       ], var.ssl_config.environment_vars)
 
       portMappings = [
+        # Internal ports with names for Service Connect
         {
           name          = local.http_port_name
           containerPort = local.http_port
           protocol      = "tcp"
+        },
+        {
+          name          = local.profile_port_name
+          containerPort = local.profile_port_internal
+          protocol      = "tcp"
+        },
+
+        # External ports without names (ALB only)
+        {
+          containerPort = local.profile_port_external
+          protocol      = "tcp"
         }
       ]
 
@@ -162,6 +174,28 @@ resource "aws_ecs_service" "alloy" {
         port = local.http_port
       }
     }
+
+    service {
+      port_name = local.profile_port_name
+
+      client_alias {
+        port = local.profile_port_internal
+      }
+    }
+  }
+
+  # Internal ALB target groups
+  load_balancer {
+    target_group_arn = aws_lb_target_group.profile_internal.arn
+    container_name   = "alloy"
+    container_port   = local.profile_port_internal
+  }
+
+  # External ALB target groups
+  load_balancer {
+    target_group_arn = aws_lb_target_group.profile_external.arn
+    container_name   = "alloy"
+    container_port   = local.profile_port_external
   }
 
   tags = {

infra/terraform/hash/observability/alloy/target_groups.tf

@@ -0,0 +1,51 @@
+# Internal HTTP target group for profiles
+resource "aws_lb_target_group" "profile_internal" {
+  name        = "${var.prefix}-profile-int"
+  port        = local.profile_port_internal
+  protocol    = "HTTP"
+  target_type = "ip"
+  vpc_id      = var.vpc.id
+
+  health_check {
+    enabled             = true
+    healthy_threshold   = 2
+    interval            = 30
+    matcher             = "200"
+    path                = "/"
+    port                = tostring(local.http_port)
+    protocol            = "HTTP"
+    timeout             = 5
+    unhealthy_threshold = 2
+  }
+
+  tags = {
+    Name    = "${var.prefix}-profile-int"
+    Purpose = "OpenTelemetry Collector internal profile target group"
+  }
+}
+
+# External HTTP target group for profiles
+resource "aws_lb_target_group" "profile_external" {
+  name        = "${var.prefix}-profile-ext"
+  port        = local.profile_port_external
+  protocol    = "HTTP"
+  target_type = "ip"
+  vpc_id      = var.vpc.id
+
+  health_check {
+    enabled             = true
+    healthy_threshold   = 2
+    interval            = 30
+    matcher             = "200"
+    path                = "/"
+    port                = tostring(local.http_port)
+    protocol            = "HTTP"
+    timeout             = 5
+    unhealthy_threshold = 2
+  }
+
+  tags = {
+    Name    = "${var.prefix}-profile-ext"
+    Purpose = "OpenTelemetry Collector external profile target group"
+  }
+}

infra/terraform/hash/observability/alloy/templates/alloy-config.alloy.tpl

@@ -275,3 +275,41 @@ otelcol.exporter.otlphttp "mimir" {
     }
   }
 }
+
+pyroscope.receive_http "profiles_internal" {
+  http {
+    listen_address = "0.0.0.0"
+    listen_port    = ${profile_port_internal}
+  }
+
+  forward_to = [pyroscope.write.pyroscope_internal.receiver]
+}
+
+pyroscope.write "pyroscope_internal" {
+  endpoint {
+    url = "http://${pyroscope_http_dns}:${pyroscope_http_port}"
+  }
+
+  external_labels = {
+    "env" = "${environment}",
+  }
+}
+
+pyroscope.receive_http "profiles_external" {
+  http {
+    listen_address = "0.0.0.0"
+    listen_port    = ${profile_port_external}
+  }
+
+  forward_to = [pyroscope.write.pyroscope_external.receiver]
+}
+
+pyroscope.write "pyroscope_external" {
+  endpoint {
+    url = "http://${pyroscope_http_dns}:${pyroscope_http_port}"
+  }
+
+  external_labels = {
+    "env" = "external",
+  }
+}

infra/terraform/hash/observability/alloy/variables.tf

@@ -54,3 +54,13 @@ variable "mimir_http_port" {
   type        = number
   description = "Mimir HTTP API port number"
 }
+
+variable "pyroscope_http_dns" {
+  type        = string
+  description = "Pyroscope HTTP API DNS name for metrics forwarding"
+}
+
+variable "pyroscope_http_port" {
+  type        = number
+  description = "Pyroscope HTTP API port number"
+}

infra/terraform/hash/observability/grafana/config.tf

@@ -3,10 +3,11 @@
 # Configuration hash for task definition versioning
 locals {
   config_hash = sha256(jsonencode({
-    grafana_config   = aws_s3_object.grafana_config.content
-    tempo_datasource = aws_s3_object.grafana_tempo_datasource.content
-    loki_datasource  = aws_s3_object.grafana_loki_datasource.content
-    mimi_datasource  = aws_s3_object.grafana_mimir_datasource.content
+    grafana_config       = aws_s3_object.grafana_config.content
+    tempo_datasource     = aws_s3_object.grafana_tempo_datasource.content
+    loki_datasource      = aws_s3_object.grafana_loki_datasource.content
+    mimir_datasource     = aws_s3_object.grafana_mimir_datasource.content
+    pyroscope_datasource = aws_s3_object.grafana_pyroscope_datasource.content
   }))
 }
 
@@ -79,3 +80,19 @@ resource "aws_s3_object" "grafana_mimir_datasource" {
     Service = "grafana"
   }
 }
+
+# Pyroscope datasource provisioning
+resource "aws_s3_object" "grafana_pyroscope_datasource" {
+  bucket = var.config_bucket.id
+  key    = "grafana/provisioning/datasources/pyroscope.yaml"
+  content = templatefile("${path.module}/templates/provisioning/datasources/pyroscope.yaml.tpl", {
+    pyroscope_http_dns  = var.pyroscope_http_dns
+    pyroscope_http_port = var.pyroscope_http_port
+  })
+  content_type = "application/x-yaml"
+
+  tags = {
+    Purpose = "Grafana Pyroscope Datasource"
+    Service = "grafana"
+  }
+}

infra/terraform/hash/observability/grafana/security_groups.tf

@@ -58,6 +58,15 @@ resource "aws_security_group" "grafana" {
     cidr_blocks = [var.vpc.cidr_block]
   }
 
+  # Allow outbound HTTP for Pyroscope API
+  egress {
+    from_port   = var.pyroscope_http_port
+    to_port     = var.pyroscope_http_port
+    protocol    = "tcp"
+    description = "Pyroscope API access"
+    cidr_blocks = [var.vpc.cidr_block]
+  }
+
   # Allow outbound PostgreSQL
   egress {
     from_port   = var.grafana_database_port