Add autoscaling host agent module to terraform blueprint

Author: haasonsaas

deploy/terraform/README.md

@@ -38,6 +38,10 @@ This sample configuration provisions the minimum AWS infrastructure required to
 
 The bootstrap script installs Docker, pulls the Nimbus images, and writes the `.env` using the outputs produced by this Terraform stack.
 
+## Host agent autoscaling
+
+This stack also creates an EC2 Auto Scaling group that runs the Nimbus host agent. Provide `agent_ami`, `agent_instance_type`, and `agent_desired_capacity` in `terraform.tfvars` to scale eval capacity on demand. The `bootstrap/bootstrap-agent.sh` script installs Nimbus, primes Firecracker assets, and registers the agent service via systemd.
+
 ## File layout
 
 ```

deploy/terraform/bootstrap/bootstrap-agent.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+LOG="/var/log/bootstrap-nimbus-agent.log"
+exec > >(tee -a "$LOG") 2>&1
+
+echo "[Nimbus agent bootstrap] Installing dependencies..."
+
+sudo yum update -y
+sudo yum install -y docker git
+sudo systemctl enable docker
+sudo systemctl start docker
+
+if ! command -v uv >/dev/null 2>&1; then
+  curl -LsSf https://astral.sh/uv/install.sh | sh
+  export PATH="$HOME/.local/bin:$PATH"
+fi
+
+echo "[Nimbus agent bootstrap] Fetching Nimbus..."
+cd /opt
+if [ ! -d "nimbus" ]; then
+  sudo git clone https://github.com/evalops/nimbus.git
+fi
+cd nimbus
+
+sudo chown -R ec2-user:ec2-user /opt/nimbus
+
+cat <<'ENV' > /opt/nimbus/agent.env
+NIMBUS_AGENT_ID=
+NIMBUS_CONTROL_PLANE_URL=https://nimbus.example.com
+NIMBUS_CONTROL_PLANE_TOKEN=
+NIMBUS_KERNEL_IMAGE=/opt/nimbus/artifacts/kernel
+NIMBUS_ROOTFS_IMAGE=/opt/nimbus/artifacts/rootfs.ext4
+ENV
+
+sudo su - ec2-user <<'EOS'
+set -euo pipefail
+cd /opt/nimbus
+uv venv .venv
+source .venv/bin/activate
+uv pip install -e .
+mkdir -p artifacts
+python scripts/setup_firecracker_assets.py artifacts
+
+cat agent.env | xargs -0
+
+cat <<'SERVICE' > nimbus-agent.service
+[Unit]
+Description=Nimbus Host Agent
+After=network-online.target docker.service
+Requires=docker.service
+
+[Service]
+Type=simple
+EnvironmentFile=/opt/nimbus/agent.env
+WorkingDirectory=/opt/nimbus
+ExecStart=/opt/nimbus/.venv/bin/python -m nimbus.host_agent
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+SERVICE
+
+sudo mv nimbus-agent.service /etc/systemd/system/nimbus-agent.service
+sudo systemctl daemon-reload
+sudo systemctl enable nimbus-agent.service
+sudo systemctl start nimbus-agent.service
+EOS
+
+echo "[Nimbus agent bootstrap] Completed"

deploy/terraform/main.tf

@@ -87,6 +87,30 @@ resource "aws_security_group" "control_plane" {
   }
 }
 
+resource "aws_security_group" "host_agent" {
+  name        = "nimbus-host-agent"
+  description = "Access for host agents"
+  vpc_id      = aws_vpc.nimbus.id
+
+  ingress {
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = [var.admin_cidr]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "nimbus-host-agent"
+  }
+}
+
 resource "aws_instance" "control_plane" {
   ami                    = var.control_plane_ami
   instance_type          = var.control_plane_instance_type
@@ -140,6 +164,43 @@ resource "aws_lb_listener" "https" {
   }
 }
 
+resource "aws_launch_template" "host_agent" {
+  name_prefix   = "nimbus-agent"
+  image_id      = var.agent_ami
+  instance_type = var.agent_instance_type
+  key_name      = var.ssh_key_name
+  security_group_names = [aws_security_group.host_agent.name]
+  user_data            = base64encode(file("${path.module}/bootstrap/bootstrap-agent.sh"))
+
+  tag_specifications {
+    resource_type = "instance"
+    tags = {
+      Name = "nimbus-host-agent"
+    }
+  }
+}
+
+resource "aws_autoscaling_group" "host_agents" {
+  name                      = "nimbus-host-agents"
+  desired_capacity          = var.agent_desired_capacity
+  max_size                  = var.agent_desired_capacity + 1
+  min_size                  = 1
+  vpc_zone_identifier       = [for subnet in aws_subnet.public : subnet.id]
+  health_check_grace_period = 60
+  health_check_type         = "EC2"
+
+  launch_template {
+    id      = aws_launch_template.host_agent.id
+    version = "$Latest"
+  }
+
+  tag {
+    key                 = "Name"
+    value               = "nimbus-host-agent"
+    propagate_at_launch = true
+  }
+}
+
 output "control_plane_public_ip" {
   value = aws_instance.control_plane.public_ip
 }

deploy/terraform/variables.tf

@@ -48,3 +48,20 @@ variable "admin_cidr" {
   description = "CIDR range allowed to SSH"
   default     = "0.0.0.0/0"
 }
+
+variable "agent_ami" {
+  type        = string
+  description = "AMI for host agent autoscaling group"
+}
+
+variable "agent_instance_type" {
+  type        = string
+  description = "EC2 instance type for host agents"
+  default     = "t3.large"
+}
+
+variable "agent_desired_capacity" {
+  type        = number
+  description = "Desired host agent count"
+  default     = 2
+}

docs/onboarding.md

@@ -68,8 +68,8 @@ New to Nimbus? This quick-start itinerary walks you from zero to your first succ
 
 ## 5. Production hardening checklist
 
-- ✅ Provision Redis/Postgres from your infra provider (RDS, CloudSQL, etc.) and point `.env`.
 - ✅ Stand up the control plane via Terraform or Helm (templates in `deploy/`).
+- ✅ Scale host agents with the Terraform autoscaling group (see `deploy/terraform`).
 - ✅ Configure TLS (terminate at a reverse proxy or load balancer).
 - ✅ Schedule nightly `nimbus.cli.report overview` runs and archive the JSON for auditing.
 - ✅ Enable OpenTelemetry export to your tracing backend.