Skip to content

Commit e0ce677

Browse files
authored
feat(terraform): add modular linux ci stack and network module (#87)
* feat(terraform): add modular linux ci infrastructure Amp-Thread-ID: https://ampcode.com/threads/T-019cd19d-83ae-72b7-bce5-b2656e93c0aa * fix(terraform): default ci env to ubuntu and ignore local tfvars Amp-Thread-ID: https://ampcode.com/threads/T-019cd19d-83ae-72b7-bce5-b2656e93c0aa * refactor(terraform): simplify ci env inputs and defaults Amp-Thread-ID: https://ampcode.com/threads/T-019cd19d-83ae-72b7-bce5-b2656e93c0aa * fix(terraform): bootstrap cleanroom agent and scope kms decrypt Amp-Thread-ID: https://ampcode.com/threads/T-019cd215-92ff-71a8-ae09-c5a770c2761d * fix(ci): avoid interactive sudo prompt in firecracker e2e Amp-Thread-ID: https://ampcode.com/threads/T-019cd215-92ff-71a8-ae09-c5a770c2761d * fix(ci): pin terraform mise tool to 1.12.2 Amp-Thread-ID: https://ampcode.com/threads/T-019cd215-92ff-71a8-ae09-c5a770c2761d * fix(terraform): enable nested virtualization for linux ci host Amp-Thread-ID: https://ampcode.com/threads/T-019cd215-92ff-71a8-ae09-c5a770c2761d * fix(terraform): require deploy key parameter for ci bootstrap Amp-Thread-ID: https://ampcode.com/threads/T-019cd215-92ff-71a8-ae09-c5a770c2761d
1 parent 259c144 commit e0ce677

21 files changed

+1292
-2
lines changed

.buildkite/pipeline.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,5 +39,7 @@ steps:
3939
env:
4040
CLEANROOM_KERNEL_IMAGE: /var/lib/buildkite-agent/.local/share/cleanroom/images/vmlinux.bin
4141
CLEANROOM_FIRECRACKER_BINARY: /usr/local/bin/firecracker
42+
CLEANROOM_PRIVILEGED_MODE: helper
43+
CLEANROOM_PRIVILEGED_HELPER_PATH: /usr/local/sbin/cleanroom-root-helper
4244
agents:
4345
queue: cleanroom

.gitignore

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,9 @@
11
/dist
22
/release-extra
3+
4+
# Local Terraform environment values (keep *.example committed)
5+
/infra/terraform/envs/ci/terraform.tfvars
6+
/infra/terraform/envs/ci/.terraform/
7+
/infra/terraform/envs/ci/*.tfstate
8+
/infra/terraform/envs/ci/*.tfstate.*
9+
/infra/terraform/envs/ci/tfplan*

.mise.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
[tools]
22
go = "1.23"
3+
terraform = "1.12.2"
34
shellcheck = "latest"
45
hyperfine = "latest"
56
svu = "3"
@@ -13,7 +14,7 @@ run = "go test ./..."
1314

1415
[tasks.lint-shell]
1516
description = "Lint shell scripts"
16-
run = "shellcheck -x scripts/cleanroom-root-helper.sh scripts/benchmark-tti.sh scripts/build-go.sh scripts/install-go.sh scripts/install.sh scripts/release.sh"
17+
run = "shellcheck -x scripts/bootstrap-buildkite-agent.sh scripts/cleanroom-root-helper.sh scripts/benchmark-tti.sh scripts/build-go.sh scripts/install-go.sh scripts/install.sh scripts/release.sh"
1718

1819
[tasks.test-full]
1920
description = "Run full Go test suite"

infra/terraform/envs/ci/.terraform.lock.hcl

Lines changed: 25 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

infra/terraform/envs/ci/README.md

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Terraform Env: CI
2+
3+
Composition root for cleanroom CI infrastructure:
4+
5+
- `../../modules/network` for VPC/subnet/NAT
6+
- `../../modules/linux-ci` for the private Linux host bootstrap
7+
8+
Default AMI behaviour:
9+
10+
- uses latest Ubuntu 24.04 AMI from SSM public parameter
11+
- set `ami_id` in `terraform.tfvars` if you want to pin an explicit AMI
12+
13+
Network behaviour:
14+
15+
- env creates its own VPC/subnets via `modules/network`
16+
- network CIDRs and AZ selection are fixed in env wiring (not user vars)
17+
18+
Bootstrap behaviour:
19+
20+
- defaults to `scripts/bootstrap-buildkite-agent.sh`, which installs and starts a Buildkite agent for the `cleanroom` queue
21+
- override `setup_script_path` if you need custom host bootstrap logic
22+
23+
## Usage
24+
25+
```bash
26+
cd infra/terraform/envs/ci
27+
cp terraform.tfvars.example terraform.tfvars
28+
mise x -- terraform init
29+
mise x -- terraform plan
30+
mise x -- terraform apply
31+
```
32+
33+
## Access
34+
35+
After apply, use outputs:
36+
37+
- `ssm_start_session_command`
38+
- `tailscale_ssh_pattern` (when tailscale auth key is configured)
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package ci_test
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"strings"
7+
"testing"
8+
)
9+
10+
func requireContains(t *testing.T, path string, want string) {
11+
t.Helper()
12+
13+
content, err := os.ReadFile(path)
14+
if err != nil {
15+
t.Fatalf("read %s: %v", path, err)
16+
}
17+
18+
if !strings.Contains(string(content), want) {
19+
t.Fatalf("expected %s to contain %q", path, want)
20+
}
21+
}
22+
23+
func requireNotContains(t *testing.T, path string, want string) {
24+
t.Helper()
25+
26+
content, err := os.ReadFile(path)
27+
if err != nil {
28+
t.Fatalf("read %s: %v", path, err)
29+
}
30+
31+
if strings.Contains(string(content), want) {
32+
t.Fatalf("expected %s not to contain %q", path, want)
33+
}
34+
}
35+
36+
func readVariableBlock(t *testing.T, path string, variableName string) string {
37+
t.Helper()
38+
39+
content, err := os.ReadFile(path)
40+
if err != nil {
41+
t.Fatalf("read %s: %v", path, err)
42+
}
43+
44+
marker := "variable \"" + variableName + "\" {"
45+
src := string(content)
46+
start := strings.Index(src, marker)
47+
if start == -1 {
48+
t.Fatalf("could not find variable %q in %s", variableName, path)
49+
}
50+
51+
braceDepth := 0
52+
for i := start; i < len(src); i++ {
53+
switch src[i] {
54+
case '{':
55+
braceDepth++
56+
case '}':
57+
braceDepth--
58+
if braceDepth == 0 {
59+
return src[start : i+1]
60+
}
61+
}
62+
}
63+
64+
t.Fatalf("unterminated variable block for %q in %s", variableName, path)
65+
return ""
66+
}
67+
68+
func TestLinuxCiDefaultsUseBootstrapScript(t *testing.T) {
69+
t.Helper()
70+
71+
requireContains(t, "variables.tf", "default = \"scripts/bootstrap-buildkite-agent.sh\"")
72+
requireContains(t, filepath.Join("..", "..", "modules", "linux-ci", "variables.tf"), "default = \"scripts/bootstrap-buildkite-agent.sh\"")
73+
requireContains(t, "terraform.tfvars.example", "setup_script_path = \"scripts/bootstrap-buildkite-agent.sh\"")
74+
}
75+
76+
func TestBootstrapScriptConfiguresBuildkiteAgent(t *testing.T) {
77+
t.Helper()
78+
79+
scriptPath := filepath.Join("..", "..", "..", "..", "scripts", "bootstrap-buildkite-agent.sh")
80+
requireContains(t, scriptPath, "buildkite-agent start")
81+
requireContains(t, scriptPath, "BUILDKITE_TOKEN_PARAM")
82+
}
83+
84+
func TestUserDataInstallsAwsCliWithoutAptAwscliDependency(t *testing.T) {
85+
t.Helper()
86+
87+
templatePath := filepath.Join("..", "..", "modules", "linux-ci", "templates", "user_data.sh.tftpl")
88+
requireNotContains(t, templatePath, "apt-get install -y git jq curl tar ca-certificates openssh-client awscli")
89+
requireContains(t, templatePath, "awscli-exe-linux")
90+
}
91+
92+
func TestLinuxCiEnablesNestedVirtualization(t *testing.T) {
93+
t.Helper()
94+
95+
moduleMainPath := filepath.Join("..", "..", "modules", "linux-ci", "main.tf")
96+
requireContains(t, moduleMainPath, "nested_virtualization = \"enabled\"")
97+
}
98+
99+
func TestGitDeployKeyIsRequired(t *testing.T) {
100+
t.Helper()
101+
102+
files := []string{
103+
"variables.tf",
104+
filepath.Join("..", "..", "modules", "linux-ci", "variables.tf"),
105+
}
106+
107+
for _, path := range files {
108+
block := readVariableBlock(t, path, "git_deploy_key_parameter_name")
109+
110+
if strings.Contains(block, "default") {
111+
t.Fatalf("expected git_deploy_key_parameter_name to have no default in %s", path)
112+
}
113+
114+
if !strings.Contains(block, "trimspace(var.git_deploy_key_parameter_name) != \"\"") {
115+
t.Fatalf("expected git_deploy_key_parameter_name validation in %s", path)
116+
}
117+
}
118+
}

infra/terraform/envs/ci/main.tf

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
data "aws_ssm_parameter" "ubuntu_ami" {
2+
name = var.ubuntu_ami_ssm_parameter_name
3+
}
4+
5+
locals {
6+
selected_ami_id = var.ami_id != "" ? var.ami_id : data.aws_ssm_parameter.ubuntu_ami.value
7+
8+
# This env owns a fixed minimal private network shape.
9+
network = {
10+
availability_zone = ""
11+
vpc_cidr = "10.42.0.0/24"
12+
public_subnet_cidr = "10.42.0.0/26"
13+
private_subnet_cidr = "10.42.0.64/26"
14+
}
15+
}
16+
17+
module "network" {
18+
source = "../../modules/network"
19+
20+
name_prefix = var.name_prefix
21+
availability_zone = local.network.availability_zone
22+
vpc_cidr = local.network.vpc_cidr
23+
public_subnet_cidr = local.network.public_subnet_cidr
24+
private_subnet_cidr = local.network.private_subnet_cidr
25+
tags = var.tags
26+
}
27+
28+
module "linux_ci" {
29+
source = "../../modules/linux-ci"
30+
31+
aws_region = var.aws_region
32+
name_prefix = var.name_prefix
33+
vpc_id = module.network.vpc_id
34+
subnet_id = module.network.private_subnet_id
35+
ami_id = local.selected_ami_id
36+
instance_type = var.instance_type
37+
root_volume_size_gib = var.root_volume_size_gib
38+
buildkite_token_parameter_name = var.buildkite_token_parameter_name
39+
tailscale_auth_key_parameter_name = var.tailscale_auth_key_parameter_name
40+
git_deploy_key_parameter_name = var.git_deploy_key_parameter_name
41+
repo_url = var.repo_url
42+
repo_ref = var.repo_ref
43+
setup_script_path = var.setup_script_path
44+
tailscale_version = var.tailscale_version
45+
tailscale_hostname_prefix = var.tailscale_hostname_prefix
46+
tailscale_advertise_tags = var.tailscale_advertise_tags
47+
tailscale_enable_ssh = var.tailscale_enable_ssh
48+
tailscale_accept_routes = var.tailscale_accept_routes
49+
tags = var.tags
50+
}

infra/terraform/envs/ci/outputs.tf

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
output "vpc_id" {
2+
description = "VPC ID created for ci environment."
3+
value = module.network.vpc_id
4+
}
5+
6+
output "public_subnet_id" {
7+
description = "Public subnet ID containing NAT gateway."
8+
value = module.network.public_subnet_id
9+
}
10+
11+
output "private_subnet_id" {
12+
description = "Private subnet ID containing linux-ci host."
13+
value = module.network.private_subnet_id
14+
}
15+
16+
output "instance_id" {
17+
description = "EC2 instance ID for linux-ci host."
18+
value = module.linux_ci.instance_id
19+
}
20+
21+
output "private_ip" {
22+
description = "Private IP address for linux-ci host."
23+
value = module.linux_ci.private_ip
24+
}
25+
26+
output "ssm_start_session_command" {
27+
description = "Command to open SSM session to linux-ci host."
28+
value = module.linux_ci.ssm_start_session_command
29+
}
30+
31+
output "tailscale_ssh_pattern" {
32+
description = "Tailscale SSH pattern when tailscale auth key is configured."
33+
value = module.linux_ci.tailscale_ssh_pattern
34+
}
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
aws_region = "ap-southeast-2"
2+
name_prefix = "cleanroom-ci"
3+
4+
instance_type = "m8i.large"
5+
6+
# Optional: pin AMI manually. Leave empty to use latest Ubuntu from SSM parameter.
7+
# ami_id = "ami-0123456789abcdef0"
8+
9+
buildkite_token_parameter_name = "/buildkite/agent-token"
10+
tailscale_auth_key_parameter_name = "/tailscale/authkey/ci"
11+
git_deploy_key_parameter_name = "/buildkite/cleanroom/deploy-key"
12+
13+
repo_url = "git@github.com:buildkite/cleanroom.git"
14+
repo_ref = "main"
15+
setup_script_path = "scripts/bootstrap-buildkite-agent.sh"
16+
17+
tailscale_hostname_prefix = "cleanroom-ci-linux"
18+
tailscale_advertise_tags = ""
19+
tailscale_enable_ssh = true
20+
tailscale_accept_routes = false
21+
22+
tags = {
23+
env = "ci"
24+
team = "platform"
25+
}

0 commit comments

Comments
 (0)