Merge branch 'master' into patch-2

Author: nitrocode

.buildkite/pipeline.yml

@@ -16,6 +16,63 @@ steps:
         run: unit-tests
         config: docker-compose.unit-tests.yml
 
+  - id: "s3secrets-helper-linux-amd64"
+    name: ":golang: :linux: s3secrets-helper-linux-amd64"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    plugins:
+      docker#v3.7.0:
+        image: "golang:1.15"
+        mount-checkout: false
+        volumes:
+          - "./build:/build:rw"
+          - "./plugins/secrets/s3secrets-helper:/s3secrets-helper:ro"
+        workdir: /s3secrets-helper
+        environment:
+          - "GOOS=linux"
+          - "GOARCH=amd64"
+        command: ["go", "build", "-o", "/build/s3secrets-helper-linux-amd64"]
+    artifact_paths:
+      - build/s3secrets-helper-linux-amd64
+
+  - id: "s3secrets-helper-linux-arm64"
+    name: ":golang: :linux: s3secrets-helper-linux-arm64"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    plugins:
+      docker#v3.7.0:
+        image: "golang:1.15"
+        mount-checkout: false
+        volumes:
+          - "./build:/build:rw"
+          - "./plugins/secrets/s3secrets-helper:/s3secrets-helper:ro"
+        workdir: /s3secrets-helper
+        environment:
+          - "GOOS=linux"
+          - "GOARCH=arm64"
+        command: ["go", "build", "-o", "/build/s3secrets-helper-linux-arm64"]
+    artifact_paths:
+      - build/s3secrets-helper-linux-arm64
+
+  - id: "s3secrets-helper-windows-amd64"
+    name: ":golang: :windows: s3secrets-helper-windows-amd64.exe"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    plugins:
+      docker#v3.7.0:
+        image: "golang:1.15"
+        mount-checkout: false
+        volumes:
+          - "./build:/build:rw"
+          - "./plugins/secrets/s3secrets-helper:/s3secrets-helper:ro"
+        workdir: /s3secrets-helper
+        environment:
+          - "GOOS=windows"
+          - "GOARCH=amd64"
+        command: ["go", "build", "-o", "/build/s3secrets-helper-windows-amd64.exe"]
+    artifact_paths:
+      - build/s3secrets-helper-windows-amd64.exe
+
   - id: "packer-windows"
     name: ":packer: :windows:"
     command: .buildkite/steps/packer.sh windows
@@ -26,26 +83,27 @@ steps:
     depends_on:
       - "lint"
       - "bats-tests"
+      - "s3secrets-helper-windows-amd64"
 
-  - id: "windows-launch"
+  - id: "launch-windows"
     name: ":cloudformation: :windows: Launch"
     command: .buildkite/steps/launch.sh windows
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     artifact_paths: "build/aws-stack.yml"
     depends_on: "packer-windows"
 
-  - id: "windows-test"
+  - id: "test-windows"
     name: ":cloudformation: :windows: Test"
     command: "docker info"
     timeout_in_minutes: 5
     agents:
-      stack: "buildkite-aws-stack-test-windows-${BUILDKITE_BUILD_NUMBER}"
-      queue: "testqueue-windows-${BUILDKITE_BUILD_NUMBER}"
-    depends_on: "windows-launch"
+      stack: "buildkite-aws-stack-test-windows-amd64-${BUILDKITE_BUILD_NUMBER}"
+      queue: "testqueue-windows-amd64-${BUILDKITE_BUILD_NUMBER}"
+    depends_on: "launch-windows"
 
-  - id: "packer-linux"
-    name: ":packer: :linux:"
+  - id: "packer-linux-amd64"
+    name: ":packer: :linux: AMD64"
     command: .buildkite/steps/packer.sh linux
     timeout_in_minutes: 60
     retry: { automatic: { limit: 3 } }
@@ -54,23 +112,53 @@ steps:
     depends_on:
       - "lint"
       - "bats-tests"
+      - "s3secrets-helper-linux-amd64"
 
-  - id: "linux-launch"
-    name: ":cloudformation: :linux: Launch"
+  - id: "launch-linux-amd64"
+    name: ":cloudformation: :linux: AMD64 Launch"
     command: .buildkite/steps/launch.sh linux
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     artifact_paths: "build/aws-stack.yml"
-    depends_on: "packer-linux"
+    depends_on: "packer-linux-amd64"
+
+  - id: "test-linux-amd64"
+    name: ":cloudformation: :linux: AMD64 Test"
+    command: "goss validate --format documentation"
+    timeout_in_minutes: 5
+    agents:
+      stack: "buildkite-aws-stack-test-linux-amd64-${BUILDKITE_BUILD_NUMBER}"
+      queue: "testqueue-linux-amd64-${BUILDKITE_BUILD_NUMBER}"
+    depends_on: "launch-linux-amd64"
+
+  - id: "packer-linux-arm64"
+    name: ":packer: :linux: ARM64"
+    command: .buildkite/steps/packer.sh linux arm64
+    timeout_in_minutes: 60
+    retry: { automatic: { limit: 3 } }
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    depends_on:
+      - "lint"
+      - "bats-tests"
+      - "s3secrets-helper-linux-arm64"
+
+  - id: "launch-linux-arm64"
+    name: ":cloudformation: :linux: ARM64 Launch"
+    command: .buildkite/steps/launch.sh linux arm64
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    artifact_paths: "build/aws-stack.yml"
+    depends_on: "packer-linux-arm64"
 
-  - id: "linux-test"
-    name: ":cloudformation: :linux: Test"
+  - id: "test-linux-arm64"
+    name: ":cloudformation: :linux: ARM64 Test"
     command: "goss validate --format documentation"
     timeout_in_minutes: 5
     agents:
-      stack: "buildkite-aws-stack-test-linux-${BUILDKITE_BUILD_NUMBER}"
-      queue: "testqueue-linux-${BUILDKITE_BUILD_NUMBER}"
-    depends_on: "linux-launch"
+      stack: "buildkite-aws-stack-test-linux-arm64-${BUILDKITE_BUILD_NUMBER}"
+      queue: "testqueue-linux-arm64-${BUILDKITE_BUILD_NUMBER}"
+    depends_on: "launch-linux-arm64"
 
   - id: "copy-ami"
     name: ":cloudformation: 🚚 🌎"
@@ -79,8 +167,9 @@ steps:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     artifact_paths: "build/mappings.yml"
     depends_on:
-      - "linux-test"
-      - "windows-test"
+      - "test-linux-amd64"
+      - "test-linux-arm64"
+      - "test-windows"
 
   - id: "publish"
     name: ":cloudformation: :rocket:"

.buildkite/steps/cleanup.sh

@@ -25,8 +25,9 @@ delete_test_stack() {
 }
 
 if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]] ; then
-  delete_test_stack "windows"
-  delete_test_stack "linux"
+  delete_test_stack "windows-amd64"
+  delete_test_stack "linux-amd64"
+  delete_test_stack "linux-arm64"
 fi
 
 if [[ $OSTYPE =~ ^darwin ]] ; then
@@ -44,15 +45,15 @@ aws s3api list-buckets \
   --output text \
   --query "$(printf 'Buckets[?CreationDate<`%s`].[Name]' "$cutoff_date" )" \
   | xargs -n1 \
-  | grep -E 'buildkite-aws-stack-test-(\d+-)?managedsecrets' \
+  | grep -E 'buildkite-aws-stack-test-.*-managedsecretsbucket' \
   | xargs -n1 -t -I% aws s3 rb s3://% --force
 
 echo "--- Deleting old cloudformation stacks"
 aws cloudformation describe-stacks \
   --output text \
   --query "$(printf 'Stacks[?CreationTime<`%s`].[StackName]' "$cutoff_date" )" \
   | xargs -n1 \
-  | grep -E 'buildkite-aws-stack-test-\d+' \
+  | grep -E 'buildkite-aws-stack-test-(linux|windows)-(amd64|arm64)-[[:digit:]]+' \
   | xargs -n1 -t -I% aws cloudformation delete-stack --stack-name "%"
 
 echo "--- Deleting old packer builders"

.buildkite/steps/copy.sh

@@ -91,16 +91,18 @@ IMAGES=(
 )
 
 # Configuration
-linux_source_image_id="${1:-}"
-windows_source_image_id="${2:-}"
+linux_amd64_source_image_id="${1:-}"
+linux_arm64_source_image_id="${1:-}"
+windows_amd64_source_image_id="${2:-}"
 
 source_region="${AWS_REGION}"
 mapping_file="build/mappings.yml"
 
 # Read the source images from meta-data if no arguments are provided
 if [ $# -eq 0 ] ; then
-    linux_source_image_id=$(buildkite-agent meta-data get "linux_image_id")
-    windows_source_image_id=$(buildkite-agent meta-data get "windows_image_id")
+    linux_amd64_source_image_id=$(buildkite-agent meta-data get "linux_amd64_image_id")
+    linux_arm64_source_image_id=$(buildkite-agent meta-data get "linux_arm64_image_id")
+    windows_amd64_source_image_id=$(buildkite-agent meta-data get "windows_amd64_image_id")
 fi
 
 # If we're not on the master branch or a tag build skip the copy
@@ -110,15 +112,16 @@ if [[ $BUILDKITE_BRANCH != "master" ]] && [[ "$BUILDKITE_TAG" != "$BUILDKITE_BRA
   cat << EOF > "$mapping_file"
 Mappings:
   AWSRegion2AMI:
-    ${AWS_REGION} : { linux: $linux_source_image_id, windows: $windows_source_image_id }
+    ${AWS_REGION} : { linuxamd64: $linux_amd64_source_image_id, linuxarm64: $linux_arm64_source_image_id, windows: $windows_amd64_source_image_id }
 EOF
   exit 0
 fi
 
-s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s.yml" \
+s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s-%s.yml" \
   "${BUILDKITE_AWS_STACK_BUCKET}" \
-  "${linux_source_image_id}" \
-  "${windows_source_image_id}" \
+  "${linux_amd64_source_image_id}" \
+  "${linux_arm64_source_image_id}" \
+  "${windows_amd64_source_image_id}" \
   "${BUILDKITE_BRANCH}")
 
 # Check if there is a previously copy in the cache bucket
@@ -128,19 +131,23 @@ if aws s3 cp "${s3_mappings_cache}" "$mapping_file" ; then
 fi
 
 # Get the image names to copy to other regions
-linux_source_image_name=$(get_image_name "$linux_source_image_id" "$source_region")
-windows_source_image_name=$(get_image_name "$windows_source_image_id" "$source_region")
+linux_amd64_source_image_name=$(get_image_name "$linux_amd64_source_image_id" "$source_region")
+linux_arm64_source_image_name=$(get_image_name "$linux_arm64_source_image_id" "$source_region")
+windows_amd64_source_image_name=$(get_image_name "$windows_amd64_source_image_id" "$source_region")
 
 # Copy to all other regions
 for region in ${ALL_REGIONS[*]}; do
   if [[ $region != "$source_region" ]] ; then
-    echo "--- Copying :linux: $linux_source_image_id to $region" >&2
-    IMAGES+=("$(copy_ami_to_region "$linux_source_image_id" "$source_region" "$region" "${linux_source_image_name}-${region}")")
+    echo "--- :linux: Copying Linux AMD64 $linux_amd64_source_image_id to $region" >&2
+    IMAGES+=("$(copy_ami_to_region "$linux_amd64_source_image_id" "$source_region" "$region" "${linux_amd64_source_image_name}-${region}")")
 
-    echo "--- Copying :windows: $windows_source_image_id to $region" >&2
-    IMAGES+=("$(copy_ami_to_region "$windows_source_image_id" "$source_region" "$region" "${windows_source_image_name}-${region}")")
+    echo "--- :linux: Copying Linux ARM64 $linux_arm64_source_image_id to $region" >&2
+    IMAGES+=("$(copy_ami_to_region "$linux_arm64_source_image_id" "$source_region" "$region" "${linux_arm64_source_image_name}-${region}")")
+
+    echo "--- :windows: Copying Windows AMD64 $windows_amd64_source_image_id to $region" >&2
+    IMAGES+=("$(copy_ami_to_region "$windows_amd64_source_image_id" "$source_region" "$region" "${windows_amd64_source_image_name}-${region}")")
   else
-    IMAGES+=("$linux_source_image_id" "$windows_source_image_id")
+    IMAGES+=("$linux_amd64_source_image_id" "$linux_arm64_source_image_id" "$windows_amd64_source_image_id")
   fi
 done
 
@@ -154,30 +161,39 @@ EOF
 echo "--- Waiting for AMIs to become available"  >&2
 
 for region in ${ALL_REGIONS[*]}; do
-  linux_image_id="${IMAGES[0]}"
-  windows_image_id="${IMAGES[1]}"
+  linux_amd64_image_id="${IMAGES[0]}"
+  linux_arm64_image_id="${IMAGES[1]}"
+  windows_amd64_image_id="${IMAGES[2]}"
 
-  wait_for_ami_to_be_available "$linux_image_id" "$region" >&2
+  wait_for_ami_to_be_available "$linux_amd64_image_id" "$region" >&2
 
   # Make the linux AMI public if it's not the source image
-  if [[ $linux_image_id != "$linux_source_image_id" ]] ; then
-    echo "Making :linux: ${linux_image_id} public" >&2
-    make_ami_public "$linux_image_id" "$region"
+  if [[ $linux_amd64_image_id != "$linux_amd64_source_image_id" ]] ; then
+    echo ":linux: Making Linux AMD64 ${linux_amd64_image_id} public" >&2
+    make_ami_public "$linux_amd64_image_id" "$region"
+  fi
+
+  wait_for_ami_to_be_available "$linux_arm64_image_id" "$region" >&2
+
+  # Make the linux ARM AMI public if it's not the source image
+  if [[ $linux_arm64_image_id != "$linux_arm64_source_image_id" ]] ; then
+    echo ":linux: Making Linux ARM64 ${linux_arm64_image_id} public" >&2
+    make_ami_public "$linux_arm64_image_id" "$region"
   fi
 
-  wait_for_ami_to_be_available "$windows_image_id" "$region" >&2
+  wait_for_ami_to_be_available "$windows_amd64_image_id" "$region" >&2
 
   # Make the windows AMI public if it's not the source image
-  if [[ $windows_image_id != "$windows_source_image_id" ]] ; then
-    echo "Making :windows: ${windows_image_id} public" >&2
-    make_ami_public "$windows_image_id" "$region"
+  if [[ $windows_amd64_image_id != "$windows_amd64_source_image_id" ]] ; then
+    echo ":windows: Making Windows AMD64 ${windows_amd64_image_id} public" >&2
+    make_ami_public "$windows_amd64_image_id" "$region"
   fi
 
   # Write yaml to file
-  echo "    $region : { linux: $linux_image_id, windows: $windows_image_id }"  >> "$mapping_file"
+  echo "    $region : { linuxamd64: $linux_amd64_image_id, linuxarm64: $linux_arm64_image_id, windows: $windows_amd64_image_id }"  >> "$mapping_file"
 
   # Shift off the processed images
-  IMAGES=("${IMAGES[@]:2}")
+  IMAGES=("${IMAGES[@]:3}")
 done
 
 echo "--- Uploading mapping to s3 cache"

.buildkite/steps/launch.sh

@@ -2,8 +2,9 @@
 set -eu
 
 os="${1:-linux}"
-stack_name="buildkite-aws-stack-test-${os}-${BUILDKITE_BUILD_NUMBER}"
-stack_queue_name="testqueue-${os}-${BUILDKITE_BUILD_NUMBER}"
+arch="${2:-amd64}"
+stack_name="buildkite-aws-stack-test-${os}-${arch}-${BUILDKITE_BUILD_NUMBER}"
+stack_queue_name="testqueue-${os}-${arch}-${BUILDKITE_BUILD_NUMBER}"
 
 # download parfait binary
 wget -N https://github.com/lox/parfait/releases/download/v1.1.3/parfait_linux_amd64
@@ -15,8 +16,8 @@ subnets=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$vpc_id" --quer
 subnet_ids=$(awk '{print $1}' <<< "$subnets" | tr ' ' ',' | tr '\n' ',' | sed 's/,$//')
 az_ids=$(awk '{print $2}' <<< "$subnets" | tr ' ' ',' | tr '\n' ',' | sed 's/,$//')
 
-image_id=$(buildkite-agent meta-data get "${os}_image_id")
-echo "Using AMI $image_id for $os"
+image_id=$(buildkite-agent meta-data get "${os}_${arch}_image_id")
+echo "Using AMI $image_id for $os/$arch"
 
 instance_type="t3.nano"
 instance_disk="10"
@@ -26,6 +27,10 @@ if [[ "$os" == "windows" ]] ; then
   instance_disk="100"
 fi
 
+if [[ "$arch" == "arm64" ]] ; then
+  instance_type="m6g.large"
+fi
+
 cat << EOF > config.json
 [
   {
@@ -88,7 +93,7 @@ cat << EOF > config.json
 EOF
 
 echo "--- Building templates"
-make "mappings-for-${os}-image" build/aws-stack.yml "IMAGE_ID=$image_id"
+make "mappings-for-${os}-${arch}-image" build/aws-stack.yml "IMAGE_ID=$image_id"
 
 echo "--- Validating templates"
 make validate