Separate conan traffic to dedicated internal ALB (#1868)

Co-authored-by: Claude <[email protected]>

Author: mattgodbolt

terraform/alb.tf

@@ -16,6 +16,24 @@ resource "aws_alb" "GccExplorerApp" {
   }
 }
 
+resource "aws_alb" "InternalServices" {
+  idle_timeout = 60
+  internal     = false
+  name         = "InternalServices"
+  security_groups = [
+    aws_security_group.InternalServicesAlb.id
+  ]
+  subnets = local.all_subnet_ids
+
+  enable_deletion_protection = false
+
+  access_logs {
+    bucket  = aws_s3_bucket.compiler-explorer-logs.bucket
+    prefix  = "elb-internal"
+    enabled = true
+  }
+}
+
 resource "aws_alb_listener" "compiler-explorer-alb-listen-http" {
   lifecycle {
     # Ignore changes to the default_action since it's managed by blue-green deployment
@@ -239,7 +257,7 @@ resource "aws_alb_listener" "ceconan-alb-listen-http" {
     target_group_arn = aws_alb_target_group.conan.arn
   }
 
-  load_balancer_arn = aws_alb.GccExplorerApp.arn
+  load_balancer_arn = aws_alb.InternalServices.arn
   port              = 1080
   protocol          = "HTTP"
 }
@@ -249,7 +267,7 @@ resource "aws_alb_listener" "ceconan-alb-listen-https" {
     type             = "forward"
     target_group_arn = aws_alb_target_group.conan.arn
   }
-  load_balancer_arn = aws_alb.GccExplorerApp.arn
+  load_balancer_arn = aws_alb.InternalServices.arn
   port              = 1443
   protocol          = "HTTPS"
   ssl_policy        = "ELBSecurityPolicy-2015-05"

terraform/cloudfront_conan.tf

@@ -4,8 +4,8 @@ resource "aws_cloudfront_distribution" "conan-compiler-explorer-com" {
     origin_id   = "S3-compiler-explorer"
   }
   origin {
-    domain_name = aws_alb.GccExplorerApp.dns_name
-    origin_id   = "GccExplorerApp"
+    domain_name = aws_alb.InternalServices.dns_name
+    origin_id   = "InternalServices"
     custom_origin_config {
       http_port                = 1080
       https_port               = 1443
@@ -77,7 +77,7 @@ resource "aws_cloudfront_distribution" "conan-compiler-explorer-com" {
         "*"
       ]
     }
-    target_origin_id       = "GccExplorerApp"
+    target_origin_id       = "InternalServices"
     viewer_protocol_policy = "redirect-to-https"
     compress               = true
   }

terraform/s3.tf

@@ -109,8 +109,9 @@ resource "aws_s3_bucket_lifecycle_configuration" "compiler-explorer-logs" {
   dynamic "rule" {
     # Keep only one month of these logs (See the privacy policy in the compiler explorer project)
     for_each = {
-      cloudfront = "cloudfront"
-      elb        = "elb"
+      cloudfront   = "cloudfront"
+      elb          = "elb"
+      elb-internal = "elb-internal"
     }
     content {
       id     = "delete_${rule.value}_per_log_policy"
@@ -202,9 +203,12 @@ data "aws_iam_policy_document" "compiler-explorer-logs-s3-policy" {
       identifiers = ["arn:aws:iam::127311923021:root"]
       type        = "AWS"
     }
-    sid       = "Allow ELB to write logs"
-    actions   = ["s3:PutObject"]
-    resources = ["${aws_s3_bucket.compiler-explorer-logs.arn}/elb/*"]
+    sid     = "Allow ELB to write logs"
+    actions = ["s3:PutObject"]
+    resources = [
+      "${aws_s3_bucket.compiler-explorer-logs.arn}/elb/*",
+      "${aws_s3_bucket.compiler-explorer-logs.arn}/elb-internal/*"
+    ]
   }
 }
 

terraform/security.tf

@@ -67,9 +67,9 @@ resource "aws_security_group_rule" "CE_ConanHttpFromAlb" {
   type                     = "ingress"
   from_port                = 1080
   to_port                  = 1080
-  source_security_group_id = aws_security_group.CompilerExplorerAlb.id
+  source_security_group_id = aws_security_group.InternalServicesAlb.id
   protocol                 = "tcp"
-  description              = "Allow HTTP access from the ALB"
+  description              = "Allow HTTP access from the internal services ALB"
 }
 
 resource "aws_security_group_rule" "CE_HttpsFromAlb" {
@@ -87,9 +87,9 @@ resource "aws_security_group_rule" "CE_ConanHttpsFromAlb" {
   type                     = "ingress"
   from_port                = 1443
   to_port                  = 1443
-  source_security_group_id = aws_security_group.CompilerExplorerAlb.id
+  source_security_group_id = aws_security_group.InternalServicesAlb.id
   protocol                 = "tcp"
-  description              = "Allow HTTPS access from the ALB"
+  description              = "Allow HTTPS access from the internal services ALB"
 }
 
 resource "aws_security_group" "CompilerExplorerAlb" {
@@ -101,6 +101,15 @@ resource "aws_security_group" "CompilerExplorerAlb" {
   }
 }
 
+resource "aws_security_group" "InternalServicesAlb" {
+  vpc_id      = module.ce_network.vpc.id
+  name        = "internal-services-alb-sg"
+  description = "Security group for internal services load balancer (conan, etc)"
+  tags = {
+    Name = "InternalServicesLoadBalancer"
+  }
+}
+
 resource "aws_security_group_rule" "ALB_HttpsFromAnywhere" {
   security_group_id = aws_security_group.CompilerExplorerAlb.id
   type              = "ingress"
@@ -112,19 +121,46 @@ resource "aws_security_group_rule" "ALB_HttpsFromAnywhere" {
   description       = "Allow HTTPS access from anywhere"
 }
 
-resource "aws_security_group_rule" "ALB_ConanHttpsFromAnywhere" {
+resource "aws_security_group_rule" "ALB_EgressToAnywhere" {
   security_group_id = aws_security_group.CompilerExplorerAlb.id
+  type              = "egress"
+  from_port         = 0
+  to_port           = 65535
+  cidr_blocks       = ["0.0.0.0/0"]
+  ipv6_cidr_blocks  = ["::/0"]
+  protocol          = "-1"
+  description       = "Allow egress to anywhere"
+}
+
+# TODO: Consider restricting these ports to CloudFront IP ranges only for better security.
+# Can use AWS managed prefix list: prefix_list_ids = ["pl-3b927c52"] instead of cidr_blocks.
+# Ports 1080/1443 are only used by conan via CloudFront, so there's no legitimate direct access.
+# Check with CE team before tightening to ensure no one is using direct ALB access for debugging.
+
+resource "aws_security_group_rule" "InternalALB_ConanHttpsFromAnywhere" {
+  security_group_id = aws_security_group.InternalServicesAlb.id
   type              = "ingress"
   from_port         = 1443
   to_port           = 1443
   cidr_blocks       = ["0.0.0.0/0"]
   ipv6_cidr_blocks  = ["::/0"]
   protocol          = "tcp"
-  description       = "Allow HTTPS access from anywhere"
+  description       = "Allow HTTPS access from anywhere (port 1443 for conan)"
 }
 
-resource "aws_security_group_rule" "ALB_EgressToAnywhere" {
-  security_group_id = aws_security_group.CompilerExplorerAlb.id
+resource "aws_security_group_rule" "InternalALB_ConanHttpFromAnywhere" {
+  security_group_id = aws_security_group.InternalServicesAlb.id
+  type              = "ingress"
+  from_port         = 1080
+  to_port           = 1080
+  cidr_blocks       = ["0.0.0.0/0"]
+  ipv6_cidr_blocks  = ["::/0"]
+  protocol          = "tcp"
+  description       = "Allow HTTP access from anywhere (port 1080 for conan)"
+}
+
+resource "aws_security_group_rule" "InternalALB_EgressToAnywhere" {
+  security_group_id = aws_security_group.InternalServicesAlb.id
   type              = "egress"
   from_port         = 0
   to_port           = 65535