Skip to content

Commit f6b7e3d

Browse files
scaduscadu
committed
Split AMI build process
Build base/gold AMI that includes everything except Buildkite tooling (agent and related services). The goal is to speed up build process.
1 parent da61a51 commit f6b7e3d

File tree

9 files changed

+531
-191
lines changed

9 files changed

+531
-191
lines changed

‎.buildkite/pipeline.yaml‎

Lines changed: 114 additions & 117 deletions
Large diffs are not rendered by default.

‎.buildkite/steps/packer.sh‎

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ fi
88

99
os="${1:-linux}"
1010
arch="${2:-amd64}"
11+
variant="${3:-full}" # "full" (default) or "base"
1112
agent_binary="buildkite-agent-${os}-${arch}"
1213

1314
if [[ "$os" == "windows" ]]; then
@@ -17,20 +18,54 @@ fi
1718
mkdir -p "build/"
1819

1920
# Build a hash of packer files and the agent versions
21+
# Include variant in the hash so base and full images don’t clash
2022
packer_files_sha=$(find Makefile "packer/${os}" plugins/ -type f -print0 | xargs -0 sha256sum | awk '{print $1}' | sort | sha256sum | awk '{print $1}')
2123
internal_files_sha=$(find go.mod go.sum internal/ -type f -print0 | xargs -0 sha256sum | awk '{print $1}' | sort | sha256sum | awk '{print $1}')
2224
stable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/stable/latest/${agent_binary}.sha256")
2325
unstable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/unstable/latest/${agent_binary}.sha256")
2426
packer_hash=$(echo "$packer_files_sha" "$internal_files_sha" "$arch" "$stable_agent_sha" "$unstable_agent_sha" | sha256sum | awk '{print $1}')
2527

26-
echo "Packer image hash for ${os}/${arch} is ${packer_hash}"
27-
packer_file="packer-${packer_hash}-${os}-${arch}.output"
28+
echo "Packer image hash for ${os}/${arch} (${variant}) is ${packer_hash}"
29+
if [[ "${variant}" == "base" ]]; then
30+
packer_file="packer-${packer_hash}-${os}-${arch}-base.output"
31+
local_output="packer-base-${os}-${arch}.output"
32+
else
33+
packer_file="packer-${packer_hash}-${os}-${arch}.output"
34+
local_output="packer-${os}-${arch}.output"
35+
fi
2836

2937
# Only build packer image if one with the same hash doesn't exist, and we're not being forced
3038
if [[ -n "${PACKER_REBUILD:-}" ]] || ! aws s3 cp "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}" .; then
31-
make "packer-${os}-${arch}.output"
32-
aws s3 cp "packer-${os}-${arch}.output" "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}"
33-
mv "packer-${os}-${arch}.output" "${packer_file}"
39+
if [[ "${variant}" == "base" ]]; then
40+
make "packer-base-${os}-${arch}.output"
41+
else
42+
# Require a golden base AMI. Try metadata first, then S3 as fallback.
43+
base_ami_id="$(buildkite-agent meta-data get "${os}-base-${arch}-ami" || true)"
44+
45+
if [[ -z "$base_ami_id" ]]; then
46+
echo "Base AMI ID not found in metadata, checking S3 for latest base image..."
47+
48+
# Calculate hash for base image to find the S3 file
49+
base_packer_hash=$(echo "$packer_files_sha" "$internal_files_sha" "$arch" "$stable_agent_sha" "$unstable_agent_sha" "base" | sha256sum | awk '{print $1}')
50+
base_packer_file="packer-${base_packer_hash}-${os}-${arch}-base.output"
51+
52+
# Try to download and extract AMI ID from the base image packer output
53+
if aws s3 cp "s3://${BUILDKITE_AWS_STACK_BUCKET}/${base_packer_file}" "/tmp/${base_packer_file}" 2>/dev/null; then
54+
base_ami_id=$(grep -Eo "${AWS_REGION}: (ami-.+)$" "/tmp/${base_packer_file}" | awk '{print $2}')
55+
echo "Found base AMI ID from S3: $base_ami_id"
56+
rm -f "/tmp/${base_packer_file}"
57+
fi
58+
fi
59+
60+
if [[ -z "$base_ami_id" ]]; then
61+
echo "ERROR: No golden base AMI found for ${os}/${arch}. Ensure the corresponding base image step ran and uploaded the AMI ID." >&2
62+
exit 1
63+
fi
64+
65+
make "packer-${os}-${arch}.output" BASE_AMI_ID="$base_ami_id"
66+
fi
67+
aws s3 cp "${local_output}" "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}"
68+
mv "${local_output}" "${packer_file}"
3469
else
3570
echo "Skipping packer build, no changes"
3671
fi
@@ -39,4 +74,8 @@ fi
3974
image_id=$(grep -Eo "${AWS_REGION}: (ami-.+)$" "$packer_file" | awk '{print $2}')
4075
echo "AMI for ${AWS_REGION} is $image_id"
4176

42-
buildkite-agent meta-data set "${os}_${arch}_image_id" "$image_id"
77+
if [[ "${variant}" == "base" ]]; then
78+
buildkite-agent meta-data set "${os}-base-${arch}-ami" "$image_id"
79+
else
80+
buildkite-agent meta-data set "${os}_${arch}_image_id" "$image_id"
81+
fi

‎Makefile‎

Lines changed: 78 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ PACKER_VERSION ?= 1.11.2
77
PACKER_LINUX_FILES = $(exec find packer/linux)
88
PACKER_WINDOWS_FILES = $(exec find packer/windows)
99

10+
# Allow passing an existing golden base AMI into packer via `BASE_AMI_ID` env var
11+
override BASE_AMI_ID ?=
12+
1013
GO_VERSION ?= 1.23.6
1114

1215
FIXPERMS_FILES = go.mod go.sum $(exec find internal/fixperms)
@@ -84,9 +87,13 @@ build/aws-stack.yml:
8487
}' templates/aws-stack.yml | $(SED) "s/%v/$(VERSION)/" > $@
8588

8689
# -----------------------------------------
87-
# AMI creation with Packer
8890

89-
packer: packer-linux-amd64.output packer-linux-arm64.output packer-windows-amd64.output
91+
92+
# Full images depend on base images when available
93+
packer: packer-base-linux-amd64.output packer-base-linux-arm64.output packer-base-windows-amd64.output packer-linux-amd64.output packer-linux-arm64.output packer-windows-amd64.output
94+
95+
packer-fmt:
96+
docker run --rm -v "$(PWD):/src" -w /src "hashicorp/packer:full-$(PACKER_VERSION)" fmt -check -recursive packer/
9097

9198
build/mappings.yml: build/linux-amd64-ami.txt build/linux-arm64-ami.txt build/windows-amd64-ami.txt
9299
mkdir -p build
@@ -118,6 +125,7 @@ packer-linux-amd64.output: $(PACKER_LINUX_FILES) build/fix-perms-linux-amd64
118125
-var 'is_released=$(IS_RELEASED)' \
119126
-var 'ami_public=$(AMI_PUBLIC)' \
120127
-var 'ami_users=$(AMI_USERS_LIST)' \
128+
-var 'base_ami_id=$(BASE_AMI_ID)' \
121129
buildkite-ami.pkr.hcl | tee $@
122130

123131
build/linux-arm64-ami.txt: packer-linux-arm64.output env-AWS_REGION
@@ -155,6 +163,7 @@ packer-linux-arm64.output: $(PACKER_LINUX_FILES) build/fix-perms-linux-arm64
155163
-var 'agent_version=$(CURRENT_AGENT_VERSION_LINUX)' \
156164
-var 'ami_public=$(AMI_PUBLIC)' \
157165
-var 'ami_users=$(AMI_USERS_LIST)' \
166+
-var 'base_ami_id=$(BASE_AMI_ID)' \
158167
buildkite-ami.pkr.hcl | tee $@
159168

160169
build/windows-amd64-ami.txt: packer-windows-amd64.output env-AWS_REGION
@@ -184,8 +193,75 @@ packer-windows-amd64.output: $(PACKER_WINDOWS_FILES)
184193
-var 'agent_version=$(CURRENT_AGENT_VERSION_WINDOWS)' \
185194
-var 'ami_public=$(AMI_PUBLIC)' \
186195
-var 'ami_users=$(AMI_USERS_LIST)' \
196+
-var 'base_ami_id=$(BASE_AMI_ID)' \
187197
buildkite-ami.pkr.hcl | tee $@
188198

199+
# -----------------------------------------
200+
# Base AMI creation
201+
202+
# Build base AMI for linux amd64
203+
packer-base-linux-amd64.output: $(PACKER_LINUX_FILES)
204+
docker run \
205+
-e AWS_DEFAULT_REGION \
206+
-e AWS_PROFILE \
207+
-e AWS_ACCESS_KEY_ID \
208+
-e AWS_SECRET_ACCESS_KEY \
209+
-e AWS_SESSION_TOKEN \
210+
-e PACKER_LOG \
211+
-v ${HOME}/.aws:/root/.aws \
212+
-v "$(PWD):/src" \
213+
--rm \
214+
-w /src/packer/linux \
215+
hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
216+
-var 'region=$(AWS_REGION)' \
217+
-var 'arch=x86_64' \
218+
-var 'instance_type=$(AMD64_INSTANCE_TYPE)' \
219+
-var 'build_number=$(BUILDKITE_BUILD_NUMBER)' \
220+
-var 'is_released=$(IS_RELEASED)' \
221+
base.pkr.hcl | tee $@
222+
223+
# Build base AMI for linux arm64
224+
packer-base-linux-arm64.output: $(PACKER_LINUX_FILES)
225+
docker run \
226+
-e AWS_DEFAULT_REGION \
227+
-e AWS_PROFILE \
228+
-e AWS_ACCESS_KEY_ID \
229+
-e AWS_SECRET_ACCESS_KEY \
230+
-e AWS_SESSION_TOKEN \
231+
-e PACKER_LOG \
232+
-v ${HOME}/.aws:/root/.aws \
233+
-v "$(PWD):/src" \
234+
--rm \
235+
-w /src/packer/linux \
236+
hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
237+
-var 'region=$(AWS_REGION)' \
238+
-var 'arch=arm64' \
239+
-var 'instance_type=$(ARM64_INSTANCE_TYPE)' \
240+
-var 'build_number=$(BUILDKITE_BUILD_NUMBER)' \
241+
-var 'is_released=$(IS_RELEASED)' \
242+
base.pkr.hcl | tee $@
243+
244+
# Build base AMI for windows amd64
245+
packer-base-windows-amd64.output: $(PACKER_WINDOWS_FILES)
246+
docker run \
247+
-e AWS_DEFAULT_REGION \
248+
-e AWS_PROFILE \
249+
-e AWS_ACCESS_KEY_ID \
250+
-e AWS_SECRET_ACCESS_KEY \
251+
-e AWS_SESSION_TOKEN \
252+
-e PACKER_LOG \
253+
-v ${HOME}/.aws:/root/.aws \
254+
-v "$(PWD):/src" \
255+
--rm \
256+
-w /src/packer/windows \
257+
hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
258+
-var 'region=$(AWS_REGION)' \
259+
-var 'arch=x86_64' \
260+
-var 'instance_type=$(WIN64_INSTANCE_TYPE)' \
261+
-var 'build_number=$(BUILDKITE_BUILD_NUMBER)' \
262+
-var 'is_released=$(IS_RELEASED)' \
263+
base.pkr.hcl | tee $@
264+
189265
# -----------------------------------------
190266
# fixperms
191267

‎packer/linux/base.pkr.hcl‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
packer {
2+
required_plugins {
3+
amazon = {
4+
source = "github.com/hashicorp/amazon"
5+
version = "~> 1"
6+
}
7+
}
8+
}
9+
10+
variable "arch" {
11+
type = string
12+
default = "x86_64"
13+
}
14+
15+
variable "instance_type" {
16+
type = string
17+
default = "m7a.xlarge"
18+
}
19+
20+
variable "region" {
21+
type = string
22+
default = "us-east-1"
23+
}
24+
25+
variable "build_number" {
26+
type = string
27+
default = "none"
28+
}
29+
30+
variable "is_released" {
31+
type = bool
32+
default = false
33+
}
34+
35+
# Latest minimal Amazon Linux 2023 image for the given arch
36+
data "amazon-ami" "al2023" {
37+
filters = {
38+
architecture = var.arch
39+
name = "al2023-ami-minimal-*"
40+
virtualization-type = "hvm"
41+
}
42+
most_recent = true
43+
owners = ["amazon"]
44+
region = var.region
45+
}
46+
47+
source "amazon-ebs" "buildkite-base-ami" {
48+
ami_description = "Buildkite Golden Base (Amazon Linux 2023 w/ docker)"
49+
ami_groups = ["all"]
50+
ami_name = "buildkite-base-linux-${var.arch}-${replace(timestamp(), ":", "-")}"
51+
instance_type = var.instance_type
52+
region = var.region
53+
source_ami = data.amazon-ami.al2023.id
54+
ssh_username = "ec2-user"
55+
ssh_clear_authorized_keys = true
56+
temporary_security_group_source_public_ip = true
57+
58+
launch_block_device_mappings {
59+
volume_type = "gp3"
60+
device_name = "/dev/xvda"
61+
volume_size = 10
62+
delete_on_termination = true
63+
}
64+
65+
metadata_options {
66+
http_endpoint = "enabled"
67+
http_tokens = "required"
68+
}
69+
imds_support = "v2.0"
70+
71+
tags = {
72+
Name = "buildkite-base-linux-${var.arch}"
73+
OSVersion = "Amazon Linux 2023"
74+
BuildNumber = var.build_number
75+
IsReleased = var.is_released
76+
SourceAMIID = data.amazon-ami.al2023.id
77+
Component = "buildkite-base"
78+
}
79+
}
80+
81+
build {
82+
sources = ["source.amazon-ebs.buildkite-base-ami"]
83+
84+
provisioner "file" {
85+
destination = "/tmp"
86+
source = "conf"
87+
}
88+
89+
provisioner "file" {
90+
destination = "/tmp/plugins"
91+
source = "../../plugins"
92+
}
93+
94+
provisioner "file" {
95+
destination = "/tmp/build"
96+
source = "../../build"
97+
}
98+
99+
# Essential utilities & updates
100+
provisioner "shell" {
101+
script = "scripts/install-utils.sh"
102+
}
103+
104+
# Docker engine
105+
provisioner "shell" {
106+
script = "scripts/install-docker.sh"
107+
}
108+
109+
# CloudWatch agent
110+
provisioner "shell" {
111+
script = "scripts/install-cloudwatch-agent.sh"
112+
}
113+
114+
# Session Manager plugin
115+
provisioner "shell" {
116+
script = "scripts/install-session-manager-plugin.sh"
117+
}
118+
119+
# Clean up
120+
provisioner "shell" {
121+
script = "scripts/cleanup.sh"
122+
}
123+
}

‎packer/linux/buildkite-ami.pkr.hcl‎

Lines changed: 16 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -60,24 +60,30 @@ data "amazon-ami" "al2023" {
6060
region = var.region
6161
}
6262

63+
# Optional override for building from a pre-baked “golden base” AMI
64+
variable "base_ami_id" {
65+
type = string
66+
default = ""
67+
}
68+
6369
source "amazon-ebs" "elastic-ci-stack-ami" {
6470
ami_description = "Buildkite Elastic Stack (Amazon Linux 2023 w/ docker)"
6571
ami_groups = var.ami_public ? ["all"] : []
6672
ami_users = var.ami_public ? [] : var.ami_users
6773
ami_name = "buildkite-stack-linux-${var.arch}-${replace(timestamp(), ":", "-")}"
6874
instance_type = var.instance_type
6975
region = var.region
70-
source_ami = data.amazon-ami.al2023.id
76+
source_ami = var.base_ami_id
7177
ssh_username = "ec2-user"
7278
ssh_clear_authorized_keys = true
7379
temporary_security_group_source_public_ip = true
7480

7581
metadata_options {
76-
http_endpoint = "enabled"
77-
http_tokens = "required"
82+
http_endpoint = "enabled"
83+
http_tokens = "required"
7884
http_put_response_hop_limit = 1
7985
}
80-
imds_support = "v2.0"
86+
imds_support = "v2.0"
8187

8288
launch_block_device_mappings {
8389
volume_type = "gp3"
@@ -91,13 +97,12 @@ source "amazon-ebs" "elastic-ci-stack-ami" {
9197
}
9298

9399
tags = {
94-
Name = "elastic-ci-stack-linux-${var.arch}"
95-
OSVersion = "Amazon Linux 2023"
96-
BuildNumber = var.build_number
97-
AgentVersion = var.agent_version
98-
IsReleased = var.is_released
99-
SourceAMIID = data.amazon-ami.al2023.id
100-
SourceAMIName = data.amazon-ami.al2023.name
100+
Name = "elastic-ci-stack-linux-${var.arch}"
101+
OSVersion = "Amazon Linux 2023"
102+
BuildNumber = var.build_number
103+
AgentVersion = var.agent_version
104+
IsReleased = var.is_released
105+
SourceAMIID = var.base_ami_id
101106
}
102107
}
103108

@@ -119,21 +124,6 @@ build {
119124
source = "../../build"
120125
}
121126

122-
provisioner "shell" {
123-
script = "scripts/install-utils.sh"
124-
}
125-
126-
provisioner "shell" {
127-
script = "scripts/install-cloudwatch-agent.sh"
128-
}
129-
130-
provisioner "shell" {
131-
script = "scripts/install-docker.sh"
132-
}
133-
134-
provisioner "shell" {
135-
script = "scripts/install-session-manager-plugin.sh"
136-
}
137127

138128
provisioner "shell" {
139129
script = "scripts/install-buildkite-agent.sh"

0 commit comments

Comments
 (0)