Merge pull request #569 from buildkite/get-windows-changes-working-in-ci

Get windows changes working in ci

Author: Lachlan Donald

.buildkite/pipeline.yml

@@ -1,7 +1,3 @@
-env:
-  AWS_STACK_NAME: "buildkite-aws-stack-test-${BUILDKITE_BUILD_NUMBER}"
-  AWS_STACK_QUEUE_NAME: "testqueue-${BUILDKITE_BUILD_NUMBER}"
-
 steps:
   - name: ":bash: Lint"
     command: .buildkite/steps/lint.sh
@@ -17,25 +13,46 @@ steps:
         config: docker-compose.unit-tests.yml
 
   - wait
-  - name: ":packer: Build"
-    command: .buildkite/steps/packer.sh
+  - name: ":packer: :windows:"
+    command: .buildkite/steps/packer.sh windows
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+
+  - wait
+  - name: ":cloudformation: :windows: Launch"
+    command: .buildkite/steps/launch.sh windows
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
+    artifact_paths: "build/aws-stack.yml"
+
+  - wait
+  - name: ":cloudformation: :windows: Test"
+    command: "docker info"
+    timeout_in_minutes: 5
+    agents:
+      stack: "buildkite-aws-stack-test-windows-${BUILDKITE_BUILD_NUMBER}"
+      queue: "testqueue-windows-${BUILDKITE_BUILD_NUMBER}"
+
+  - wait
+  - name: ":packer: :linux:"
+    command: .buildkite/steps/packer.sh linux
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
 
   - wait
-  - name: ":cloudformation: Launch"
-    command: .buildkite/steps/test.sh
+  - name: ":cloudformation: :linux: Launch"
+    command: .buildkite/steps/launch.sh linux
     agents:
       queue: "${BUILDKITE_AGENT_META_DATA_QUEUE}"
     artifact_paths: "build/aws-stack.yml"
 
   - wait
-  - name: ":buildkite: Server Specs"
+  - name: ":cloudformation: :linux: Test"
     command: "goss validate --format documentation"
     timeout_in_minutes: 5
     agents:
-      stack: "buildkite-aws-stack-test-${BUILDKITE_BUILD_NUMBER}"
-      queue: "testqueue-${BUILDKITE_BUILD_NUMBER}"
+      stack: "buildkite-aws-stack-test-linux-${BUILDKITE_BUILD_NUMBER}"
+      queue: "testqueue-linux-${BUILDKITE_BUILD_NUMBER}"
 
   - wait
   - name: ":cloudformation: 🚚 🌎"

.buildkite/steps/cleanup.sh

@@ -2,23 +2,31 @@
 # shellcheck disable=SC2016
 set -uxo pipefail
 
-if [[ -n "${AWS_STACK_NAME:-}" ]] ; then
+delete_test_stack() {
+  local os="$1"
+  local stack_name; stack_name="buildkite-aws-stack-test-${os}-${BUILDKITE_BUILD_NUMBER}"
+
   secrets_bucket=$(aws cloudformation describe-stacks \
-    --stack-name "${AWS_STACK_NAME}" \
+    --stack-name "${stack_name}" \
     --query 'Stacks[0].Outputs[?OutputKey==`ManagedSecretsBucket`].OutputValue' \
     --output text)
 
   secrets_logging_bucket=$(aws cloudformation describe-stacks \
-    --stack-name "${AWS_STACK_NAME}" \
+    --stack-name "${stack_name}" \
     --query 'Stacks[0].Outputs[?OutputKey==`ManagedSecretsLoggingBucket`].OutputValue' \
     --output text)
 
-  echo "--- Deleting stack $AWS_STACK_NAME"
-  aws cloudformation delete-stack --stack-name "$AWS_STACK_NAME"
+  echo "--- Deleting stack $stack_name"
+  aws cloudformation delete-stack --stack-name "$stack_name"
 
-  echo "--- Deleting buckets for $AWS_STACK_NAME"
+  echo "--- Deleting buckets for $stack_name"
   aws s3 rb "s3://${secrets_bucket}" --force
   aws s3 rb "s3://${secrets_logging_bucket}" --force
+}
+
+if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]] ; then
+  delete_test_stack "windows"
+  delete_test_stack "linux"
 fi
 
 if [[ $OSTYPE =~ ^darwin ]] ; then

.buildkite/steps/copy.sh

@@ -2,7 +2,7 @@
 set -euo pipefail
 
 # Copies an AMI to all other regions and outputs a build/mappings.yml file
-# Local Usage: .buildkite/steps/copy.sh <ami_id>
+# Local Usage: .buildkite/steps/copy.sh <linux_ami_id> <windows_ami_id>
 
 copy_ami_to_region() {
   local source_image_id="$1"
@@ -38,6 +38,17 @@ wait_for_ami_to_be_available() {
   done
 }
 
+get_image_name() {
+  local image_id="$1"
+  local region="$2"
+
+  aws ec2 describe-images \
+  --image-ids "$image_id" \
+  --output text \
+  --region "$region" \
+  --query 'Images[*].Name'
+}
+
 make_ami_public() {
   local image_id="$1"
   local region="$2"
@@ -76,13 +87,16 @@ IMAGES=(
 )
 
 # Configuration
-source_image_id="${1:-}"
+linux_source_image_id="${1:-}"
+windows_source_image_id="${2:-}"
+
 source_region="${AWS_REGION}"
 mapping_file="build/mappings.yml"
 
-# Read the source_image_id from meta-data if empty
-if [[ -z "$source_image_id" ]] ; then
-  source_image_id=$(buildkite-agent meta-data get image_id)
+# Read the source images from meta-data if no arguments are provided
+if [ $# -eq 0 ] ; then
+    linux_source_image_id=$(buildkite-agent meta-data get "linux_image_id")
+    windows_source_image_id=$(buildkite-agent meta-data get "windows_image_id")
 fi
 
 # If we're not on the master branch or a tag build skip the copy
@@ -92,33 +106,37 @@ if [[ $BUILDKITE_BRANCH != "master" ]] && [[ "$BUILDKITE_TAG" != "$BUILDKITE_BRA
   cat << EOF > "$mapping_file"
 Mappings:
   AWSRegion2AMI:
-    ${AWS_REGION} : { AMI: $source_image_id }
+    ${AWS_REGION} : { linux: $linux_source_image_id, windows: $windows_source_image_id }
 EOF
   exit 0
 fi
 
-s3_mappings_cache="s3://${BUILDKITE_AWS_STACK_BUCKET}/mappings-${source_image_id}-${BUILDKITE_BRANCH}.yml"
+s3_mappings_cache=$(printf "s3://%s/mappings-%s-%s-%s.yml" \
+  "${BUILDKITE_AWS_STACK_BUCKET}" \
+  "${linux_source_image_id}" \
+  "${windows_source_image_id}" \
+  "${BUILDKITE_BRANCH}")
 
 # Check if there is a previously copy in the cache bucket
 if aws s3 cp "${s3_mappings_cache}" "$mapping_file" ; then
   echo "--- Skipping AMI copy, was previously copied"
   exit 0
 fi
 
-# Get the image name to copy to other amis
-source_image_name=$(aws ec2 describe-images \
-  --image-ids "$source_image_id" \
-  --output text \
-  --region "$source_region" \
-  --query 'Images[*].Name')
+# Get the image names to copy to other regions
+linux_source_image_name=$(get_image_name "$linux_source_image_id" "$source_region")
+windows_source_image_name=$(get_image_name "$windows_source_image_id" "$source_region")
 
 # Copy to all other regions
 for region in ${ALL_REGIONS[*]}; do
   if [[ $region != "$source_region" ]] ; then
-    echo "--- Copying $source_image_id to $region" >&2
-    IMAGES+=("$(copy_ami_to_region "$source_image_id" "$source_region" "$region" "${source_image_name}-${region}")")
+    echo "--- Copying :linux: $linux_source_image_id to $region" >&2
+    IMAGES+=("$(copy_ami_to_region "$linux_source_image_id" "$source_region" "$region" "${linux_source_image_name}-${region}")")
+
+    echo "--- Copying :windows: $windows_source_image_id to $region" >&2
+    IMAGES+=("$(copy_ami_to_region "$windows_source_image_id" "$source_region" "$region" "${windows_source_image_name}-${region}")")
   else
-    IMAGES+=("$source_image_id")
+    IMAGES+=("$linux_source_image_id" "$windows_source_image_id")
   fi
 done
 
@@ -130,20 +148,32 @@ Mappings:
 EOF
 
 echo "--- Waiting for AMIs to become available"  >&2
-for ((i=0; i<${#IMAGES[*]}; i++)); do
-  region="${ALL_REGIONS[i]}"
-  image_id="${IMAGES[i]}"
 
-  wait_for_ami_to_be_available "$image_id" "$region" >&2
+for region in ${ALL_REGIONS[*]}; do
+  linux_image_id="${IMAGES[0]}"
+  windows_image_id="${IMAGES[1]}"
+
+  wait_for_ami_to_be_available "$linux_image_id" "$region" >&2
 
-  # Make the AMI public if it's not the source image
-  if [[ $image_id != "$source_image_id" ]] ; then
-    echo "Making ${image_id} public" >&2
-    make_ami_public "$image_id" "$region"
+  # Make the linux AMI public if it's not the source image
+  if [[ $linux_image_id != "$linux_source_image_id" ]] ; then
+    echo "Making :linux: ${linux_image_id} public" >&2
+    make_ami_public "$linux_image_id" "$region"
+  fi
+
+  wait_for_ami_to_be_available "$windows_image_id" "$region" >&2
+
+  # Make the windows AMI public if it's not the source image
+  if [[ $windows_image_id != "$windows_source_image_id" ]] ; then
+    echo "Making :windows: ${windows_image_id} public" >&2
+    make_ami_public "$windows_image_id" "$region"
   fi
 
   # Write yaml to file
-  echo "    $region : { AMI: $image_id }"  >> "$mapping_file"
+  echo "    $region : { linux: $linux_image_id, windows: $windows_image_id }"  >> "$mapping_file"
+
+  # Shift off the processed images
+  IMAGES=("${IMAGES[@]:2}")
 done
 
 echo "--- Uploading mapping to s3 cache"

.buildkite/steps/test.sh renamed to .buildkite/steps/launch.sh

@@ -2,6 +2,10 @@
 # shellcheck disable=SC1117
 set -eu
 
+os="${1:-linux}"
+stack_name="buildkite-aws-stack-test-${os}-${BUILDKITE_BUILD_NUMBER}"
+stack_queue_name="testqueue-${os}-${BUILDKITE_BUILD_NUMBER}"
+
 # download parfait binary
 wget -N https://github.com/lox/parfait/releases/download/v1.1.3/parfait_linux_amd64
 mv parfait_linux_amd64 parfait
@@ -12,8 +16,16 @@ subnets=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$vpc_id" --quer
 subnet_ids=$(awk '{print $1}' <<< "$subnets" | tr ' ' ',' | tr '\n' ',' | sed 's/,$//')
 az_ids=$(awk '{print $2}' <<< "$subnets" | tr ' ' ',' | tr '\n' ',' | sed 's/,$//')
 
-image_id=$(buildkite-agent meta-data get image_id)
-echo "Using AMI $image_id"
+image_id=$(buildkite-agent meta-data get "${os}_image_id")
+echo "Using AMI $image_id for $os"
+
+instance_type="t3.nano"
+instance_disk="10"
+
+if [[ "$os" == "windows" ]] ; then
+  instance_type="m5.large"
+  instance_disk="100"
+fi
 
 cat << EOF > config.json
 [
@@ -23,15 +35,19 @@ cat << EOF > config.json
   },
   {
     "ParameterKey": "BuildkiteQueue",
-    "ParameterValue": "${AWS_STACK_QUEUE_NAME}"
+    "ParameterValue": "${stack_queue_name}"
   },
   {
     "ParameterKey": "KeyName",
-    "ParameterValue": "${AWS_KEYPAIR:-default}"
+    "ParameterValue": "${AWS_KEYPAIR:-aws-stack-test}"
   },
   {
     "ParameterKey": "InstanceType",
-    "ParameterValue": "t2.nano"
+    "ParameterValue": "${instance_type}"
+  },
+  {
+    "ParameterKey": "InstanceOperatingSystem",
+    "ParameterValue": "${os}"
   },
   {
     "ParameterKey": "VpcId",
@@ -59,7 +75,7 @@ cat << EOF > config.json
   },
   {
     "ParameterKey": "RootVolumeSize",
-    "ParameterValue": "10"
+    "ParameterValue": "${instance_disk}"
   },
   {
     "ParameterKey": "EnableDockerUserNamespaceRemap",
@@ -77,13 +93,13 @@ cat << EOF > config.json
 EOF
 
 echo "--- Building templates"
-make mappings-for-image build "IMAGE_ID=$image_id"
+make "mappings-for-${os}-image" build/aws-stack.yml "IMAGE_ID=$image_id"
 
 echo "--- Validating templates"
 make validate
 
-echo "--- Creating stack ${AWS_STACK_NAME}"
-make create-stack "STACK_NAME=$AWS_STACK_NAME"
+echo "--- Creating stack ${stack_name}"
+make create-stack "STACK_NAME=$stack_name"
 
 echo "+++ ⌛️ Waiting for update to complete"
-./parfait watch-stack "${AWS_STACK_NAME}"
+./parfait watch-stack "${stack_name}"

.buildkite/steps/packer.sh

@@ -6,22 +6,29 @@ if [[ -z "${BUILDKITE_AWS_STACK_BUCKET}" ]] ; then
   exit 1
 fi
 
-mkdir -p build
+os="${1:-linux}"
+agent_binary="buildkite-agent-${os}-amd64"
+
+if [[ "$os" == "windows" ]] ; then
+  agent_binary+=".exe"
+fi
+
+mkdir -p "build/"
 
 # Build a hash of packer files and the agent versions
-packer_files_sha=$(find packer/ plugins/ -type f -print0 | xargs -0 sha1sum | awk '{print $1}' | sort | sha1sum | awk '{print $1}')
-stable_agent_sha=$(curl -Lfs https://download.buildkite.com/agent/stable/latest/buildkite-agent-linux-amd64.sha256)
-unstable_agent_sha=$(curl -Lfs https://download.buildkite.com/agent/unstable/latest/buildkite-agent-linux-amd64.sha256)
+packer_files_sha=$(find "packer/${os}" plugins/ -type f -print0 | xargs -0 sha1sum | awk '{print $1}' | sort | sha1sum | awk '{print $1}')
+stable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/stable/latest/${agent_binary}.sha256")
+unstable_agent_sha=$(curl -Lfs "https://download.buildkite.com/agent/unstable/latest/${agent_binary}.sha256")
 packer_hash=$(echo "$packer_files_sha" "$stable_agent_sha" "$unstable_agent_sha" | sha1sum | awk '{print $1}')
 
-echo "Packer image hash is $packer_hash"
-packer_file="${packer_hash}.packer"
+echo "Packer image hash for ${os} is ${packer_hash}"
+packer_file="packer-${packer_hash}-${os}.output"
 
 # Only build packer image if one with the same hash doesn't exist
 if ! aws s3 cp "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}" . ; then
-  make packer
-  aws s3 cp packer.output "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}"
-  mv packer.output "${packer_file}"
+  make "packer-${os}.output"
+  aws s3 cp "packer-${os}.output" "s3://${BUILDKITE_AWS_STACK_BUCKET}/${packer_file}"
+  mv "packer-${os}.output" "${packer_file}"
 else
   echo "Skipping packer build, no changes"
 fi
@@ -30,4 +37,4 @@ fi
 image_id=$(grep -Eo "${AWS_REGION}: (ami-.+)$" "$packer_file" | awk '{print $2}')
 echo "AMI for ${AWS_REGION} is $image_id"
 
-buildkite-agent meta-data set image_id "$image_id"
+buildkite-agent meta-data set "${os}_image_id" "$image_id"

.buildkite/steps/publish.sh

@@ -29,7 +29,7 @@ echo "--- Fetching latest git tags"
 git fetch --tags
 
 echo "--- Building :cloudformation: templates"
-make build
+make build/aws-stack.yml
 
 # Publish the top-level mappings only on when we see the most recent tag on master
 if is_latest_tag ; then

.gitignore

@@ -1,5 +1,5 @@
 build/
-packer.output
+packer*.output
 config.json
 vendor/
 node_modules/