Merge branch 'main' into SUP-4545-log-group-retention-policies

Author: petetomasik

.buildkite/pipeline.yaml

@@ -8,6 +8,7 @@ fi
 
 os="${1:-linux}"
 arch="${2:-amd64}"
+variant="${3:-full}" # "full" (default) or "base"
 agent_binary="buildkite-agent-${os}-${arch}"
 
 if [[ "$os" == "windows" ]]; then
@@ -23,14 +24,48 @@ stable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/stable/latest
 unstable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/unstable/latest/${agent_binary}.sha256")
 packer_hash=$(echo "$packer_files_sha" "$internal_files_sha" "$arch" "$stable_agent_sha" "$unstable_agent_sha" | sha256sum | awk '{print $1}')
 
-echo "Packer image hash for ${os}/${arch} is ${packer_hash}"
-packer_file="packer-${packer_hash}-${os}-${arch}.output"
+# Include variant in the hash so base and full images don’t clash
+echo "Packer image hash for ${os}/${arch} (${variant}) is ${packer_hash}"
+if [[ "${variant}" == "base" ]]; then
+  packer_file="packer-${packer_hash}-${os}-${arch}-base.output"
+  local_output="packer-base-${os}-${arch}.output"
+else
+  packer_file="packer-${packer_hash}-${os}-${arch}.output"
+  local_output="packer-${os}-${arch}.output"
+fi
 
 # Only build packer image if one with the same hash doesn't exist, and we're not being forced
 if [[ -n "${PACKER_REBUILD:-}" ]] || ! aws s3 cp "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}" .; then
-  make "packer-${os}-${arch}.output"
-  aws s3 cp "packer-${os}-${arch}.output" "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}"
-  mv "packer-${os}-${arch}.output" "${packer_file}"
+  if [[ "${variant}" == "base" ]]; then
+    make "packer-base-${os}-${arch}.output"
+  else
+    # Require a golden base AMI. Try metadata first, then S3 as fallback.
+    base_ami_id="$(buildkite-agent meta-data get "${os}-base-${arch}-ami" || true)"
+
+    if [[ -z "$base_ami_id" ]]; then
+      echo "Base AMI ID not found in metadata, checking S3 for latest base image..."
+
+      # Calculate hash for base image to find the S3 file
+      base_packer_hash=$(echo "$packer_files_sha" "$internal_files_sha" "$arch" "$stable_agent_sha" "$unstable_agent_sha" "base" | sha256sum | awk '{print $1}')
+      base_packer_file="packer-${base_packer_hash}-${os}-${arch}-base.output"
+
+      # Try to download and extract AMI ID from the base image packer output
+      if aws s3 cp "s3://${BUILDKITE_AWS_STACK_BUCKET}/${base_packer_file}" "/tmp/${base_packer_file}" 2>/dev/null; then
+        base_ami_id=$(grep -Eo "${AWS_REGION}: (ami-.+)$" "/tmp/${base_packer_file}" | awk '{print $2}')
+        echo "Found base AMI ID from S3: $base_ami_id"
+        rm -f "/tmp/${base_packer_file}"
+      fi
+    fi
+
+    if [[ -z "$base_ami_id" ]]; then
+      echo "ERROR: No golden base AMI found for ${os}/${arch}. Ensure the corresponding base image step ran and uploaded the AMI ID." >&2
+      exit 1
+    fi
+
+    make "packer-${os}-${arch}.output" BASE_AMI_ID="$base_ami_id"
+  fi
+  aws s3 cp "${local_output}" "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}"
+  mv "${local_output}" "${packer_file}"
 else
   echo "Skipping packer build, no changes"
 fi
@@ -39,4 +74,8 @@ fi
 image_id=$(grep -Eo "${AWS_REGION}: (ami-.+)$" "$packer_file" | awk '{print $2}')
 echo "AMI for ${AWS_REGION} is $image_id"
 
-buildkite-agent meta-data set "${os}_${arch}_image_id" "$image_id"
+if [[ "${variant}" == "base" ]]; then
+  buildkite-agent meta-data set "${os}-base-${arch}-ami" "$image_id"
+else
+  buildkite-agent meta-data set "${os}_${arch}_image_id" "$image_id"
+fi

.buildkite/steps/packer.sh

@@ -7,6 +7,9 @@ PACKER_VERSION ?= 1.11.2
 PACKER_LINUX_FILES = $(exec find packer/linux)
 PACKER_WINDOWS_FILES = $(exec find packer/windows)
 
+# Allow passing an existing golden base AMI into packer via `BASE_AMI_ID` env var
+override BASE_AMI_ID ?=
+
 GO_VERSION ?= 1.23.6
 
 FIXPERMS_FILES = go.mod go.sum $(exec find internal/fixperms)
@@ -84,9 +87,13 @@ build/aws-stack.yml:
 	}' templates/aws-stack.yml | $(SED) "s/%v/$(VERSION)/" > $@
 
 # -----------------------------------------
-# AMI creation with Packer
 
-packer: packer-linux-amd64.output packer-linux-arm64.output packer-windows-amd64.output
+
+# Full images depend on base images when available
+packer: packer-base-linux-amd64.output packer-base-linux-arm64.output packer-base-windows-amd64.output packer-linux-amd64.output packer-linux-arm64.output packer-windows-amd64.output
+
+packer-fmt:
+	docker run --rm -v "$(PWD):/src" -w /src "hashicorp/packer:full-$(PACKER_VERSION)" fmt -check -recursive packer/
 
 build/mappings.yml: build/linux-amd64-ami.txt build/linux-arm64-ami.txt build/windows-amd64-ami.txt
 	mkdir -p build
@@ -118,6 +125,7 @@ packer-linux-amd64.output: $(PACKER_LINUX_FILES) build/fix-perms-linux-amd64
 			-var 'is_released=$(IS_RELEASED)' \
 			-var 'ami_public=$(AMI_PUBLIC)' \
 			-var 'ami_users=$(AMI_USERS_LIST)' \
+			-var 'base_ami_id=$(BASE_AMI_ID)' \
 			buildkite-ami.pkr.hcl | tee $@
 
 build/linux-arm64-ami.txt: packer-linux-arm64.output env-AWS_REGION
@@ -155,6 +163,7 @@ packer-linux-arm64.output: $(PACKER_LINUX_FILES) build/fix-perms-linux-arm64
 			-var 'agent_version=$(CURRENT_AGENT_VERSION_LINUX)' \
 			-var 'ami_public=$(AMI_PUBLIC)' \
 			-var 'ami_users=$(AMI_USERS_LIST)' \
+			-var 'base_ami_id=$(BASE_AMI_ID)' \
 			buildkite-ami.pkr.hcl | tee $@
 
 build/windows-amd64-ami.txt: packer-windows-amd64.output env-AWS_REGION
@@ -184,8 +193,75 @@ packer-windows-amd64.output: $(PACKER_WINDOWS_FILES)
 			-var 'agent_version=$(CURRENT_AGENT_VERSION_WINDOWS)' \
 			-var 'ami_public=$(AMI_PUBLIC)' \
 			-var 'ami_users=$(AMI_USERS_LIST)' \
+			-var 'base_ami_id=$(BASE_AMI_ID)' \
 			buildkite-ami.pkr.hcl | tee $@
 
+# -----------------------------------------
+# Base AMI creation
+
+# Build base AMI for linux amd64
+packer-base-linux-amd64.output: $(PACKER_LINUX_FILES)
+	docker run \
+		-e AWS_DEFAULT_REGION  \
+		-e AWS_PROFILE \
+		-e AWS_ACCESS_KEY_ID \
+		-e AWS_SECRET_ACCESS_KEY \
+		-e AWS_SESSION_TOKEN \
+		-e PACKER_LOG \
+		-v ${HOME}/.aws:/root/.aws \
+		-v "$(PWD):/src" \
+		--rm \
+		-w /src/packer/linux \
+		hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
+			-var 'region=$(AWS_REGION)' \
+			-var 'arch=x86_64' \
+			-var 'instance_type=$(AMD64_INSTANCE_TYPE)' \
+			-var 'build_number=$(BUILDKITE_BUILD_NUMBER)' \
+			-var 'is_released=$(IS_RELEASED)' \
+			base.pkr.hcl | tee $@
+
+# Build base AMI for linux arm64
+packer-base-linux-arm64.output: $(PACKER_LINUX_FILES)
+	docker run \
+		-e AWS_DEFAULT_REGION  \
+		-e AWS_PROFILE \
+		-e AWS_ACCESS_KEY_ID \
+		-e AWS_SECRET_ACCESS_KEY \
+		-e AWS_SESSION_TOKEN \
+		-e PACKER_LOG \
+		-v ${HOME}/.aws:/root/.aws \
+		-v "$(PWD):/src" \
+		--rm \
+		-w /src/packer/linux \
+		hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
+			-var 'region=$(AWS_REGION)' \
+			-var 'arch=arm64' \
+			-var 'instance_type=$(ARM64_INSTANCE_TYPE)' \
+			-var 'build_number=$(BUILDKITE_BUILD_NUMBER)' \
+			-var 'is_released=$(IS_RELEASED)' \
+			base.pkr.hcl | tee $@
+
+# Build base AMI for windows amd64
+packer-base-windows-amd64.output: $(PACKER_WINDOWS_FILES)
+	docker run \
+		-e AWS_DEFAULT_REGION  \
+		-e AWS_PROFILE \
+		-e AWS_ACCESS_KEY_ID \
+		-e AWS_SECRET_ACCESS_KEY \
+		-e AWS_SESSION_TOKEN \
+		-e PACKER_LOG \
+		-v ${HOME}/.aws:/root/.aws \
+		-v "$(PWD):/src" \
+		--rm \
+		-w /src/packer/windows \
+		hashicorp/packer:full-$(PACKER_VERSION) build -timestamp-ui \
+			-var 'region=$(AWS_REGION)' \
+			-var 'arch=x86_64' \
+			-var 'instance_type=$(WIN64_INSTANCE_TYPE)' \
+			-var 'build_number=$(BUILDKITE_BUILD_NUMBER)' \
+			-var 'is_released=$(IS_RELEASED)' \
+			base.pkr.hcl | tee $@
+
 # -----------------------------------------
 # fixperms
 

Makefile

@@ -114,14 +114,18 @@ AWS_PROFILE="some-profile" make create-stack
 aws-vault exec some-profile -- make create-stack
 ```
 
-If you need to build your own AMI (because you've changed something in the
-`packer` directory), run packer with AWS credentials in your shell environment.
+If you need to build your own AMIs (because you've changed something in the
+`packer` directory), run `make packer` with AWS credentials in your shell environment.
 
-By default, AMIs are built as private (only accessible to the AWS account that created them) for security. You can control AMI visibility and build location using these variables:
+The build process is now two steps:
+1.  **Base AMI:** A base AMI is created with the latest OS updates. This image is cached and only rebuilt when the underlying OS or configuration changes.
+2.  **Final AMI:** The final AMI is built on top of the base AMI, installing the Buildkite agent and other software. This makes final builds faster and more consistent.
 
-- **`AMI_PUBLIC`** - Set to `true` to make AMIs publicly accessible to all AWS users, or `false` (default) for private AMIs
-- **`AMI_USERS`** - Comma-separated list of AWS account IDs that should have access to private AMIs (ignored when `AMI_PUBLIC=true`)
-- **`AWS_REGION`** - AWS region where AMIs should be built (defaults to `us-east-1`)
+By default, AMIs are built as private. You can control AMI visibility and build location using these variables:
+
+- `AMI_PUBLIC` - Set to `true` to make AMIs publicly accessible to all AWS users, or `false` (default) for private AMIs.
+- `AMI_USERS` - A comma-separated list of AWS account IDs that should have access to private AMIs (ignored when `AMI_PUBLIC=true`).
+- `AWS_REGION` - The AWS region where AMIs should be built (defaults to `us-east-1`).
 
 ```bash
 # Build private AMIs (default - recommended for security)
@@ -137,8 +141,7 @@ make packer AMI_USERS="123456789012,987654321098,555666777888"
 make packer AMI_PUBLIC=false AMI_USERS="123456789012,987654321098" AWS_REGION=us-west-2
 ```
 
-This will boot and image three AWS EC2 instances in your account's `us-east-1`
-default VPC (or the region specified by `AWS_REGION`):
+This will create AMIs for the following platforms in your account's `us-east-1` region (or the region specified by `AWS_REGION`):
 
 - Linux (64-bit x86)
 - Linux (64-bit Arm)

README.md

@@ -0,0 +1,123 @@
+packer {
+  required_plugins {
+    amazon = {
+      source  = "github.com/hashicorp/amazon"
+      version = "~> 1"
+    }
+  }
+}
+
+variable "arch" {
+  type    = string
+  default = "x86_64"
+}
+
+variable "instance_type" {
+  type    = string
+  default = "m7a.xlarge"
+}
+
+variable "region" {
+  type    = string
+  default = "us-east-1"
+}
+
+variable "build_number" {
+  type    = string
+  default = "none"
+}
+
+variable "is_released" {
+  type    = bool
+  default = false
+}
+
+# Latest minimal Amazon Linux 2023 image for the given arch
+data "amazon-ami" "al2023" {
+  filters = {
+    architecture        = var.arch
+    name                = "al2023-ami-minimal-*"
+    virtualization-type = "hvm"
+  }
+  most_recent = true
+  owners      = ["amazon"]
+  region      = var.region
+}
+
+source "amazon-ebs" "buildkite-base-ami" {
+  ami_description                           = "Buildkite Golden Base (Amazon Linux 2023 w/ docker)"
+  ami_groups                                = ["all"]
+  ami_name                                  = "buildkite-base-linux-${var.arch}-${replace(timestamp(), ":", "-")}"
+  instance_type                             = var.instance_type
+  region                                    = var.region
+  source_ami                                = data.amazon-ami.al2023.id
+  ssh_username                              = "ec2-user"
+  ssh_clear_authorized_keys                 = true
+  temporary_security_group_source_public_ip = true
+
+  launch_block_device_mappings {
+    volume_type           = "gp3"
+    device_name           = "/dev/xvda"
+    volume_size           = 10
+    delete_on_termination = true
+  }
+
+  metadata_options {
+    http_endpoint = "enabled"
+    http_tokens   = "required"
+  }
+  imds_support = "v2.0"
+
+  tags = {
+    Name        = "buildkite-base-linux-${var.arch}"
+    OSVersion   = "Amazon Linux 2023"
+    BuildNumber = var.build_number
+    IsReleased  = var.is_released
+    SourceAMIID = data.amazon-ami.al2023.id
+    Component   = "buildkite-base"
+  }
+}
+
+build {
+  sources = ["source.amazon-ebs.buildkite-base-ami"]
+
+  provisioner "file" {
+    destination = "/tmp"
+    source      = "conf"
+  }
+
+  provisioner "file" {
+    destination = "/tmp/plugins"
+    source      = "../../plugins"
+  }
+
+  provisioner "file" {
+    destination = "/tmp/build"
+    source      = "../../build"
+  }
+
+  # Essential utilities & updates
+  provisioner "shell" {
+    script = "scripts/install-utils.sh"
+  }
+
+  # Docker engine
+  provisioner "shell" {
+    script = "scripts/install-docker.sh"
+  }
+
+  # CloudWatch agent
+  provisioner "shell" {
+    script = "scripts/install-cloudwatch-agent.sh"
+  }
+
+  # Session Manager plugin
+  provisioner "shell" {
+    script = "scripts/install-session-manager-plugin.sh"
+  }
+
+  # Clean up
+  provisioner "shell" {
+    script = "scripts/cleanup.sh"
+  }
+}