Merge remote-tracking branch 'origin/master' into keithduncan/ssm-with-ssh

Author: keithduncan

.buildkite/pipeline.yml

@@ -27,25 +27,25 @@ steps:
       - "lint"
       - "bats-tests"
 
-  - id: "windows-launch"
+  - id: "launch-windows"
     name: ":cloudformation: :windows: Launch"
     command: .buildkite/steps/launch.sh windows
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     artifact_paths: "build/aws-stack.yml"
     depends_on: "packer-windows"
 
-  - id: "windows-test"
+  - id: "test-windows"
     name: ":cloudformation: :windows: Test"
     command: "docker info"
     timeout_in_minutes: 5
     agents:
-      stack: "buildkite-aws-stack-test-windows-${BUILDKITE_BUILD_NUMBER}"
-      queue: "testqueue-windows-${BUILDKITE_BUILD_NUMBER}"
-    depends_on: "windows-launch"
+      stack: "buildkite-aws-stack-test-windows-amd64-${BUILDKITE_BUILD_NUMBER}"
+      queue: "testqueue-windows-amd64-${BUILDKITE_BUILD_NUMBER}"
+    depends_on: "launch-windows"
 
-  - id: "packer-linux"
-    name: ":packer: :linux:"
+  - id: "packer-linux-amd64"
+    name: ":packer: :linux: AMD64"
     command: .buildkite/steps/packer.sh linux
     timeout_in_minutes: 60
     retry: { automatic: { limit: 3 } }
@@ -55,22 +55,50 @@ steps:
       - "lint"
       - "bats-tests"
 
-  - id: "linux-launch"
-    name: ":cloudformation: :linux: Launch"
+  - id: "launch-linux-amd64"
+    name: ":cloudformation: :linux: AMD64 Launch"
     command: .buildkite/steps/launch.sh linux
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     artifact_paths: "build/aws-stack.yml"
-    depends_on: "packer-linux"
+    depends_on: "packer-linux-amd64"
 
-  - id: "linux-test"
-    name: ":cloudformation: :linux: Test"
+  - id: "test-linux-amd64"
+    name: ":cloudformation: :linux: AMD64 Test"
     command: "goss validate --format documentation"
     timeout_in_minutes: 5
     agents:
-      stack: "buildkite-aws-stack-test-linux-${BUILDKITE_BUILD_NUMBER}"
-      queue: "testqueue-linux-${BUILDKITE_BUILD_NUMBER}"
-    depends_on: "linux-launch"
+      stack: "buildkite-aws-stack-test-linux-amd64-${BUILDKITE_BUILD_NUMBER}"
+      queue: "testqueue-linux-amd64-${BUILDKITE_BUILD_NUMBER}"
+    depends_on: "launch-linux-amd64"
+
+  - id: "packer-linux-arm64"
+    name: ":packer: :linux: ARM64"
+    command: .buildkite/steps/packer.sh linux arm64
+    timeout_in_minutes: 60
+    retry: { automatic: { limit: 3 } }
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    depends_on:
+      - "lint"
+      - "bats-tests"
+
+  - id: "launch-linux-arm64"
+    name: ":cloudformation: :linux: ARM64 Launch"
+    command: .buildkite/steps/launch.sh linux arm64
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    artifact_paths: "build/aws-stack.yml"
+    depends_on: "packer-linux-arm64"
+
+  - id: "test-linux-arm64"
+    name: ":cloudformation: :linux: ARM64 Test"
+    command: "goss validate --format documentation"
+    timeout_in_minutes: 5
+    agents:
+      stack: "buildkite-aws-stack-test-linux-arm64-${BUILDKITE_BUILD_NUMBER}"
+      queue: "testqueue-linux-arm64-${BUILDKITE_BUILD_NUMBER}"
+    depends_on: "launch-linux-arm64"
 
   - id: "copy-ami"
     name: ":cloudformation: 🚚 🌎"
@@ -79,8 +107,9 @@ steps:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     artifact_paths: "build/mappings.yml"
     depends_on:
-      - "linux-test"
-      - "windows-test"
+      - "test-linux-amd64"
+      - "test-linux-arm64"
+      - "test-windows"
 
   - id: "publish"
     name: ":cloudformation: :rocket:"

.buildkite/steps/cleanup.sh

@@ -25,8 +25,9 @@ delete_test_stack() {
 }
 
 if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]] ; then
-  delete_test_stack "windows"
-  delete_test_stack "linux"
+  delete_test_stack "windows-amd64"
+  delete_test_stack "linux-amd64"
+  delete_test_stack "linux-arm64"
 fi
 
 if [[ $OSTYPE =~ ^darwin ]] ; then
@@ -44,15 +45,15 @@ aws s3api list-buckets \
   --output text \
   --query "$(printf 'Buckets[?CreationDate<`%s`].[Name]' "$cutoff_date" )" \
   | xargs -n1 \
-  | grep -E 'buildkite-aws-stack-test-(\d+-)?managedsecrets' \
+  | grep -E 'buildkite-aws-stack-test-.*-managedsecretsbucket' \
   | xargs -n1 -t -I% aws s3 rb s3://% --force
 
 echo "--- Deleting old cloudformation stacks"
 aws cloudformation describe-stacks \
   --output text \
   --query "$(printf 'Stacks[?CreationTime<`%s`].[StackName]' "$cutoff_date" )" \
   | xargs -n1 \
-  | grep -E 'buildkite-aws-stack-test-\d+' \
+  | grep -E 'buildkite-aws-stack-test-(linux|windows)-(amd64|arm64)-[[:digit:]]+' \
   | xargs -n1 -t -I% aws cloudformation delete-stack --stack-name "%"
 
 echo "--- Deleting old packer builders"

.buildkite/steps/copy.sh

@@ -91,16 +91,18 @@ IMAGES=(
 )
 
 # Configuration
-linux_source_image_id="${1:-}"
-windows_source_image_id="${2:-}"
+linux_amd64_source_image_id="${1:-}"
+linux_arm64_source_image_id="${1:-}"
+windows_amd64_source_image_id="${2:-}"
 
 source_region="${AWS_REGION}"
 mapping_file="build/mappings.yml"
 
 # Read the source images from meta-data if no arguments are provided
 if [ $# -eq 0 ] ; then
-    linux_source_image_id=$(buildkite-agent meta-data get "linux_image_id")
-    windows_source_image_id=$(buildkite-agent meta-data get "windows_image_id")
+    linux_amd64_source_image_id=$(buildkite-agent meta-data get "linux_amd64_image_id")
+    linux_arm64_source_image_id=$(buildkite-agent meta-data get "linux_arm64_image_id")
+    windows_amd64_source_image_id=$(buildkite-agent meta-data get "windows_amd64_image_id")
 fi
 
 # If we're not on the master branch or a tag build skip the copy
@@ -110,15 +112,16 @@ if [[ $BUILDKITE_BRANCH != "master" ]] && [[ "$BUILDKITE_TAG" != "$BUILDKITE_BRA
   cat << EOF > "$mapping_file"
 Mappings:
   AWSRegion2AMI:
-    ${AWS_REGION} : { linux: $linux_source_image_id, windows: $windows_source_image_id }
+    ${AWS_REGION} : { linuxamd64: $linux_amd64_source_image_id, linuxarm64: $linux_arm64_source_image_id, windows: $windows_amd64_source_image_id }
 EOF
   exit 0
 fi
 
-s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s.yml" \
+s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s-%s.yml" \
   "${BUILDKITE_AWS_STACK_BUCKET}" \
-  "${linux_source_image_id}" \
-  "${windows_source_image_id}" \
+  "${linux_amd64_source_image_id}" \
+  "${linux_arm64_source_image_id}" \
+  "${windows_amd64_source_image_id}" \
   "${BUILDKITE_BRANCH}")
 
 # Check if there is a previously copy in the cache bucket
@@ -128,19 +131,24 @@ if aws s3 cp "${s3_mappings_cache}" "$mapping_file" ; then
 fi
 
 # Get the image names to copy to other regions
-linux_source_image_name=$(get_image_name "$linux_source_image_id" "$source_region")
-windows_source_image_name=$(get_image_name "$windows_source_image_id" "$source_region")
+linux_amd64_source_image_name=$(get_image_name "$linux_amd64_source_image_id" "$source_region")
+linux_arm64_source_image_name=$(get_image_name "$linux_arm64_source_image_id" "$source_region")
+windows_amd64_source_image_name=$(get_image_name "$windows_amd64_source_image_id" "$source_region")
 
 # Copy to all other regions
+# shellcheck disable=SC2048
 for region in ${ALL_REGIONS[*]}; do
   if [[ $region != "$source_region" ]] ; then
-    echo "--- Copying :linux: $linux_source_image_id to $region" >&2
-    IMAGES+=("$(copy_ami_to_region "$linux_source_image_id" "$source_region" "$region" "${linux_source_image_name}-${region}")")
+    echo "--- :linux: Copying Linux AMD64 $linux_amd64_source_image_id to $region" >&2
+    IMAGES+=("$(copy_ami_to_region "$linux_amd64_source_image_id" "$source_region" "$region" "${linux_amd64_source_image_name}-${region}")")
 
-    echo "--- Copying :windows: $windows_source_image_id to $region" >&2
-    IMAGES+=("$(copy_ami_to_region "$windows_source_image_id" "$source_region" "$region" "${windows_source_image_name}-${region}")")
+    echo "--- :linux: Copying Linux ARM64 $linux_arm64_source_image_id to $region" >&2
+    IMAGES+=("$(copy_ami_to_region "$linux_arm64_source_image_id" "$source_region" "$region" "${linux_arm64_source_image_name}-${region}")")
+
+    echo "--- :windows: Copying Windows AMD64 $windows_amd64_source_image_id to $region" >&2
+    IMAGES+=("$(copy_ami_to_region "$windows_amd64_source_image_id" "$source_region" "$region" "${windows_amd64_source_image_name}-${region}")")
   else
-    IMAGES+=("$linux_source_image_id" "$windows_source_image_id")
+    IMAGES+=("$linux_amd64_source_image_id" "$linux_arm64_source_image_id" "$windows_amd64_source_image_id")
   fi
 done
 
@@ -152,32 +160,41 @@ Mappings:
 EOF
 
 echo "--- Waiting for AMIs to become available"  >&2
-
+# shellcheck disable=SC2048
 for region in ${ALL_REGIONS[*]}; do
-  linux_image_id="${IMAGES[0]}"
-  windows_image_id="${IMAGES[1]}"
+  linux_amd64_image_id="${IMAGES[0]}"
+  linux_arm64_image_id="${IMAGES[1]}"
+  windows_amd64_image_id="${IMAGES[2]}"
 
-  wait_for_ami_to_be_available "$linux_image_id" "$region" >&2
+  wait_for_ami_to_be_available "$linux_amd64_image_id" "$region" >&2
 
   # Make the linux AMI public if it's not the source image
-  if [[ $linux_image_id != "$linux_source_image_id" ]] ; then
-    echo "Making :linux: ${linux_image_id} public" >&2
-    make_ami_public "$linux_image_id" "$region"
+  if [[ $linux_amd64_image_id != "$linux_amd64_source_image_id" ]] ; then
+    echo ":linux: Making Linux AMD64 ${linux_amd64_image_id} public" >&2
+    make_ami_public "$linux_amd64_image_id" "$region"
+  fi
+
+  wait_for_ami_to_be_available "$linux_arm64_image_id" "$region" >&2
+
+  # Make the linux ARM AMI public if it's not the source image
+  if [[ $linux_arm64_image_id != "$linux_arm64_source_image_id" ]] ; then
+    echo ":linux: Making Linux ARM64 ${linux_arm64_image_id} public" >&2
+    make_ami_public "$linux_arm64_image_id" "$region"
   fi
 
-  wait_for_ami_to_be_available "$windows_image_id" "$region" >&2
+  wait_for_ami_to_be_available "$windows_amd64_image_id" "$region" >&2
 
   # Make the windows AMI public if it's not the source image
-  if [[ $windows_image_id != "$windows_source_image_id" ]] ; then
-    echo "Making :windows: ${windows_image_id} public" >&2
-    make_ami_public "$windows_image_id" "$region"
+  if [[ $windows_amd64_image_id != "$windows_amd64_source_image_id" ]] ; then
+    echo ":windows: Making Windows AMD64 ${windows_amd64_image_id} public" >&2
+    make_ami_public "$windows_amd64_image_id" "$region"
   fi
 
   # Write yaml to file
-  echo "    $region : { linux: $linux_image_id, windows: $windows_image_id }"  >> "$mapping_file"
+  echo "    $region : { linuxamd64: $linux_amd64_image_id, linuxarm64: $linux_arm64_image_id, windows: $windows_amd64_image_id }"  >> "$mapping_file"
 
   # Shift off the processed images
-  IMAGES=("${IMAGES[@]:2}")
+  IMAGES=("${IMAGES[@]:3}")
 done
 
 echo "--- Uploading mapping to s3 cache"

.buildkite/steps/launch.sh

@@ -2,8 +2,9 @@
 set -eu
 
 os="${1:-linux}"
-stack_name="buildkite-aws-stack-test-${os}-${BUILDKITE_BUILD_NUMBER}"
-stack_queue_name="testqueue-${os}-${BUILDKITE_BUILD_NUMBER}"
+arch="${2:-amd64}"
+stack_name="buildkite-aws-stack-test-${os}-${arch}-${BUILDKITE_BUILD_NUMBER}"
+stack_queue_name="testqueue-${os}-${arch}-${BUILDKITE_BUILD_NUMBER}"
 
 # download parfait binary
 wget -N https://github.com/lox/parfait/releases/download/v1.1.3/parfait_linux_amd64
@@ -15,8 +16,8 @@ subnets=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$vpc_id" --quer
 subnet_ids=$(awk '{print $1}' <<< "$subnets" | tr ' ' ',' | tr '\n' ',' | sed 's/,$//')
 az_ids=$(awk '{print $2}' <<< "$subnets" | tr ' ' ',' | tr '\n' ',' | sed 's/,$//')
 
-image_id=$(buildkite-agent meta-data get "${os}_image_id")
-echo "Using AMI $image_id for $os"
+image_id=$(buildkite-agent meta-data get "${os}_${arch}_image_id")
+echo "Using AMI $image_id for $os/$arch"
 
 instance_type="t3.nano"
 instance_disk="10"
@@ -26,6 +27,10 @@ if [[ "$os" == "windows" ]] ; then
   instance_disk="100"
 fi
 
+if [[ "$arch" == "arm64" ]] ; then
+  instance_type="m6g.large"
+fi
+
 cat << EOF > config.json
 [
   {
@@ -88,7 +93,7 @@ cat << EOF > config.json
 EOF
 
 echo "--- Building templates"
-make "mappings-for-${os}-image" build/aws-stack.yml "IMAGE_ID=$image_id"
+make "mappings-for-${os}-${arch}-image" build/aws-stack.yml "IMAGE_ID=$image_id"
 
 echo "--- Validating templates"
 make validate

.buildkite/steps/packer.sh

@@ -7,7 +7,8 @@ if [[ -z "${BUILDKITE_AWS_STACK_BUCKET}" ]] ; then
 fi
 
 os="${1:-linux}"
-agent_binary="buildkite-agent-${os}-amd64"
+arch="${2:-amd64}"
+agent_binary="buildkite-agent-${os}-${arch}"
 
 if [[ "$os" == "windows" ]] ; then
   agent_binary+=".exe"
@@ -19,16 +20,16 @@ mkdir -p "build/"
 packer_files_sha=$(find Makefile "packer/${os}" plugins/ -type f -print0 | xargs -0 sha1sum | awk '{print $1}' | sort | sha1sum | awk '{print $1}')
 stable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/stable/latest/${agent_binary}.sha256")
 unstable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/unstable/latest/${agent_binary}.sha256")
-packer_hash=$(echo "$packer_files_sha" "$stable_agent_sha" "$unstable_agent_sha" | sha1sum | awk '{print $1}')
+packer_hash=$(echo "$packer_files_sha" "$arch" "$stable_agent_sha" "$unstable_agent_sha" | sha1sum | awk '{print $1}')
 
-echo "Packer image hash for ${os} is ${packer_hash}"
-packer_file="packer-${packer_hash}-${os}.output"
+echo "Packer image hash for ${os}/${arch} is ${packer_hash}"
+packer_file="packer-${packer_hash}-${os}-${arch}.output"
 
 # Only build packer image if one with the same hash doesn't exist, and we're not being forced
 if [[ -n "${PACKER_REBUILD:-}" ]] || ! aws s3 cp "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}" . ; then
-  make "packer-${os}.output"
-  aws s3 cp "packer-${os}.output" "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}"
-  mv "packer-${os}.output" "${packer_file}"
+  make "packer-${os}-${arch}.output"
+  aws s3 cp "packer-${os}-${arch}.output" "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}"
+  mv "packer-${os}-${arch}.output" "${packer_file}"
 else
   echo "Skipping packer build, no changes"
 fi
@@ -37,4 +38,4 @@ fi
 image_id=$(grep -Eo "${AWS_REGION}: (ami-.+)$" "$packer_file" | awk '{print $2}')
 echo "AMI for ${AWS_REGION} is $image_id"
 
-buildkite-agent meta-data set "${os}_image_id" "$image_id"
+buildkite-agent meta-data set "${os}_${arch}_image_id" "$image_id"