Merge pull request #557 from buildkite/add-instance-storage

Optionally mount Instance Storage on boot

Author: keithduncan

README.md

@@ -51,6 +51,7 @@ CloudWatch Logs Agent | ✅ | ✅
 Per-Instance Bootstrap Script | ✅ | ✅ 
 🧑‍🔬 git-mirrors experiment | ✅ | ✅ 
 SSM Access | ✅ | ✅ 
+Instance Storage (NVMe) | ✅ | 
 SSH Access | ✅ | 
 Periodic authorized_keys Refresh | ✅ | 
 Periodic Instance Health Check | ✅ | 

packer/linux/buildkite-ami.json

@@ -65,6 +65,10 @@
       "type": "shell",
       "script": "scripts/install-session-manager-plugin.sh"
     },
+    {
+      "type": "shell",
+      "script": "scripts/install-nvme-cli.sh"
+    },
     {
       "type": "shell",
       "inline": [

packer/linux/conf/bin/bk-check-disk-space.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 DISK_MIN_AVAILABLE=${DISK_MIN_AVAILABLE:-5242880} # 5GB
 DISK_MIN_INODES=${DISK_MIN_INODES:-250000} # docker needs lots
 
-DOCKER_DIR="/var/lib/docker/"
+DOCKER_DIR="$(jq -r '."data-root" // "/var/lib/docker"' /etc/docker/daemon.json)"
 
 disk_avail=$(df -k --output=avail "$DOCKER_DIR" | tail -n1)
 

packer/linux/conf/bin/bk-configure-docker.sh

@@ -17,3 +17,8 @@ fi
 if [[ "${DOCKER_EXPERIMENTAL:-false}" == "true" ]] ; then
   cat <<< "$(jq '.experimental=true' /etc/docker/daemon.json)" > /etc/docker/daemon.json
 fi
+
+# Move docker root to the ephemeral device
+if [[ "${BUILDKITE_ENABLE_INSTANCE_STORAGE:-false}" == "true" ]] ; then
+  cat <<< "$(jq '."data-root"="/mnt/ephemeral/docker"' /etc/docker/daemon.json)" > /etc/docker/daemon.json
+fi

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -126,15 +126,32 @@ if [[ -n "${BUILDKITE_AGENT_TAGS:-}" ]] ; then
 fi
 
 # Enable git-mirrors
+BUILDKITE_AGENT_GIT_MIRRORS_PATH=""
 if [[ "${BUILDKITE_AGENT_ENABLE_GIT_MIRRORS_EXPERIMENT}" == "true" ]] ; then
   if [[ -z "$BUILDKITE_AGENT_EXPERIMENTS" ]] ; then
     BUILDKITE_AGENT_EXPERIMENTS="git-mirrors"
   else
     BUILDKITE_AGENT_EXPERIMENTS+=",git-mirrors"
   fi
-  BUILDKITE_AGENT_GIT_MIRRORS_PATH="/var/lib/buildkite-agent/git-mirrors"
-else
-  BUILDKITE_AGENT_GIT_MIRRORS_PATH=""
+
+  if [ "${BUILDKITE_ENABLE_INSTANCE_STORAGE:-false}" == "true" ]
+  then
+    BUILDKITE_AGENT_GIT_MIRRORS_PATH="/mnt/ephemeral/git-mirrors"
+
+    mkdir -p "${BUILDKITE_AGENT_GIT_MIRRORS_PATH}"
+    chown buildkite-agent: "${BUILDKITE_AGENT_GIT_MIRRORS_PATH}"
+  else
+    BUILDKITE_AGENT_GIT_MIRRORS_PATH="/var/lib/buildkite-agent/git-mirrors"
+  fi
+fi
+
+BUILDKITE_AGENT_BUILD_PATH="/var/lib/buildkite-agent/builds"
+if [ "${BUILDKITE_ENABLE_INSTANCE_STORAGE:-false}" == "true" ]
+then
+  BUILDKITE_AGENT_BUILD_PATH="/mnt/ephemeral/builds"
+
+  mkdir -p "${BUILDKITE_AGENT_BUILD_PATH}"
+  chown buildkite-agent: "${BUILDKITE_AGENT_BUILD_PATH}"
 fi
 
 BUILDKITE_AGENT_TOKEN="$(aws ssm get-parameter --name "${BUILDKITE_AGENT_TOKEN_PATH}" --with-decryption --query Parameter.Value --output text)"
@@ -146,7 +163,7 @@ tags=$(IFS=, ; echo "${agent_metadata[*]}")
 tags-from-ec2-meta-data=true
 timestamp-lines=${BUILDKITE_AGENT_TIMESTAMP_LINES}
 hooks-path=/etc/buildkite-agent/hooks
-build-path=/var/lib/buildkite-agent/builds
+build-path=${BUILDKITE_AGENT_BUILD_PATH}
 plugins-path=/var/lib/buildkite-agent/plugins
 git-mirrors-path="${BUILDKITE_AGENT_GIT_MIRRORS_PATH}"
 experiment="${BUILDKITE_AGENT_EXPERIMENTS}"

packer/linux/conf/bin/bk-mount-instance-storage.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+set -euo pipefail
+
+# Write to system console and to our log file
+# See https://alestic.com/2010/12/ec2-user-data-output/
+exec > >(tee -a /var/log/elastic-stack.log|logger -t user-data -s 2>/dev/console) 2>&1
+
+# Mount instance storage if we can
+# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html
+
+# Move docker root to the ephemeral device
+if [[ "${BUILDKITE_ENABLE_INSTANCE_STORAGE:-false}" != "true" ]] ; then
+  echo "Skipping mounting instance storage"
+  exit 0
+fi
+
+#shellcheck disable=SC2207
+devices=($(nvme list | grep "Amazon EC2 NVMe Instance Storage"| cut -f1 -d' '))
+
+if [ -z "${devices[*]}" ]
+then
+  echo "No Instance Storage NVMe drives to mount" >&2
+  exit 0
+fi
+
+if [[ "${#devices[@]}" -eq 1 ]] ; then
+  echo "Mounting instance storage device directly" >&2
+  logicalname="${devices[0]}"
+elif [[ "${#devices[@]}" -gt 1 ]] ; then
+  echo "Mounting instance storage devices using software RAID" >&2
+  logicalname=/dev/md0
+
+  mdadm \
+    --create "$logicalname" \
+    --level=0 \
+    -c256 \
+    --raid-devices="${#devices[@]}" "${devices[@]}"
+
+  echo "DEVICE ${devices[*]}" > /etc/mdadm.conf
+
+  mdadm --detail --scan >> /etc/mdadm.conf
+  blockdev --setra 65536 "$logicalname"
+fi
+
+# Make an ext4 file system, [-F]orce creation, don’t TRIM at fs creation time
+# (-E nodiscard)
+echo "Formatting $logicalname as ext4" >&2
+mkfs.ext4 -F -E nodiscard "$logicalname" > /dev/null
+
+devicemount=/mnt/ephemeral
+
+echo "Mounting $logicalname to $devicemount" >&2
+
+fs_type="ext4"
+mount_options="defaults,noatime"
+
+mkdir -p "$devicemount"
+mount -t "${fs_type}" -o "${mount_options}" "$logicalname" "$devicemount"
+
+if [ ! -f /etc/fstab.backup ]; then
+  cp -rP /etc/fstab /etc/fstab.backup
+  echo "$logicalname $devicemount    ${fs_type}  ${mount_options}  0 0" >> /etc/fstab
+fi

packer/linux/scripts/install-nvme-cli.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+sudo yum install -y nvme-cli

templates/aws-stack.yml

@@ -41,6 +41,7 @@ Metadata:
         - ImageId
         - ImageIdParameter
         - InstanceType
+        - EnableInstanceStorage
         - AgentsPerInstance
         - KeyName
         - SpotPrice
@@ -383,6 +384,14 @@ Parameters:
       - "false"
     Default: "false"
 
+  EnableInstanceStorage:
+    Type: String
+    Description: Mount available NVMe Instance Storage at /mnt/ephemeral
+    AllowedValues:
+      - "true"
+      - "false"
+    Default: "false"
+
   EnableCostAllocationTags:
     Type: String
     Description: Enables AWS Cost Allocation tags for all resources in the stack. See https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/cost-alloc-tags.html
@@ -952,8 +961,13 @@ Resources:
                   MIME-Version: 1.0
                   --==BOUNDARY==
                   Content-Type: text/cloud-boothook; charset="us-ascii"
+                  BUILDKITE_ENABLE_INSTANCE_STORAGE="${EnableInstanceStorage}" \
+                    /usr/local/bin/bk-mount-instance-storage.sh
+                  --==BOUNDARY==
+                  Content-Type: text/cloud-boothook; charset="us-ascii"
                   DOCKER_USERNS_REMAP=${EnableDockerUserNamespaceRemap} \
                   DOCKER_EXPERIMENTAL=${EnableDockerExperimental} \
+                  BUILDKITE_ENABLE_INSTANCE_STORAGE="${EnableInstanceStorage}" \
                     /usr/local/bin/bk-configure-docker.sh
                   --==BOUNDARY==
                   Content-Type: text/x-shellscript; charset="us-ascii"
@@ -971,6 +985,7 @@ Resources:
                   BUILDKITE_QUEUE="${BuildkiteQueue}" \
                   BUILDKITE_AGENT_ENABLE_GIT_MIRRORS_EXPERIMENT=${EnableAgentGitMirrorsExperiment} \
                   BUILDKITE_ELASTIC_BOOTSTRAP_SCRIPT="${BootstrapScriptUrl}" \
+                  BUILDKITE_ENABLE_INSTANCE_STORAGE="${EnableInstanceStorage}" \
                   BUILDKITE_AUTHORIZED_USERS_URL="${AuthorizedUsersUrl}" \
                   BUILDKITE_ECR_POLICY=${ECRAccessPolicy} \
                   BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB=${BuildkiteTerminateInstanceAfterJob} \