Fix: base AMI lookup

In #1593 we
aimed to simplify base AMI build logic, however we did miss fetching the
base AMI ID from S3.
This is needed when base AMI is not rebuild within the same pipeline
build, so it's not available in the meta-data.

Author: scadu

.buildkite/steps/packer.sh

@@ -30,11 +30,23 @@ fi
 if [[ "${variant}" == "base" ]]; then
   make "packer-base-${os}-${arch}.output"
 else
-  # Require a golden base AMI from metadata
+  # Try metadata first, then S3 fallback for base AMI ID
   base_ami_id="$(buildkite-agent meta-data get "${os}-base-${arch}-ami" || true)"
 
   if [[ -z "$base_ami_id" ]]; then
-    echo "ERROR: No golden base AMI found for ${os}/${arch}. Ensure the corresponding base image step ran and uploaded the AMI ID." >&2
+    echo "Base AMI ID not found in metadata, checking S3 for latest base image..."
+
+    # Try to fetch the latest base AMI output from S3
+    latest_base_file="packer-base-${os}-${arch}-latest.output"
+    if aws s3 cp "s3://${BUILDKITE_AWS_STACK_BUCKET}/${latest_base_file}" "/tmp/${latest_base_file}" 2>/dev/null; then
+      base_ami_id=$(grep -Eo "${AWS_REGION}: (ami-.+)$" "/tmp/${latest_base_file}" | awk '{print $2}')
+      echo "Found base AMI ID from S3: $base_ami_id"
+      rm -f "/tmp/${latest_base_file}"
+    fi
+  fi
+
+  if [[ -z "$base_ami_id" ]]; then
+    echo "ERROR: No golden base AMI found for ${os}/${arch}. Ensure a base image has been built from main branch." >&2
     exit 1
   fi
 
@@ -43,6 +55,14 @@ fi
 
 # Upload to S3 with timestamped filename
 aws s3 cp "${local_output}" "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}"
+
+# For base images on main branch, also upload as "latest"
+if [[ "${variant}" == "base" && "${BUILDKITE_BRANCH:-}" == "main" ]]; then
+  latest_file="packer-base-${os}-${arch}-latest.output"
+  aws s3 cp "${local_output}" "s3://${BUILDKITE_AWS_STACK_BUCKET}/${latest_file}"
+  echo "Updated latest base AMI pointer for ${os}/${arch}"
+fi
+
 mv "${local_output}" "${packer_file}"
 
 # Get the image id from the packer build output for later steps

packer/linux/.trigger-base-build

@@ -1 +1 @@
-# Last updated: 2025-09-18 12:56:38 UTC
+# Last updated: 2025-09-19 07:10:59 UTC

packer/windows/.trigger-base-build

@@ -1 +1 @@
-# Last updated: 2025-09-18 12:56:38 UTC
+# Last updated: 2025-09-19 07:10:59 UTC