Merge branch 'master' into MaxInstanceLifetime

Author: nitrocode

goss.yaml

@@ -44,6 +44,10 @@ service:
     enabled: true
     running: true
 
+  lifecycled:
+    enabled: true
+    running: true
+
   sshd:
     enabled: true
     running: true
@@ -89,6 +93,9 @@ process:
   buildkite-agent:
     running: true
 
+  lifecycled:
+    running: true
+
   sshd:
     running: true
 

packer/linux/buildkite-ami.json

@@ -45,6 +45,10 @@
       "type": "shell",
       "script": "scripts/install-cloudwatch-agent.sh"
     },
+    {
+      "type": "shell",
+      "script": "scripts/install-lifecycled.sh"
+    },
     {
       "type": "shell",
       "script": "scripts/install-docker.sh"

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -134,25 +134,31 @@ if [[ "${BUILDKITE_AGENT_ENABLE_GIT_MIRRORS_EXPERIMENT}" == "true" ]] ; then
     BUILDKITE_AGENT_EXPERIMENTS+=",git-mirrors"
   fi
 
+  BUILDKITE_AGENT_GIT_MIRRORS_PATH="/var/lib/buildkite-agent/git-mirrors"
+  mkdir -p "${BUILDKITE_AGENT_GIT_MIRRORS_PATH}"
+
   if [ "${BUILDKITE_ENABLE_INSTANCE_STORAGE:-false}" == "true" ]
   then
-    BUILDKITE_AGENT_GIT_MIRRORS_PATH="/mnt/ephemeral/git-mirrors"
+    EPHEMERAL_GIT_MIRRORS_PATH="/mnt/ephemeral/git-mirrors"
+    mkdir -p "${EPHEMERAL_GIT_MIRRORS_PATH}"
 
-    mkdir -p "${BUILDKITE_AGENT_GIT_MIRRORS_PATH}"
-    chown buildkite-agent: "${BUILDKITE_AGENT_GIT_MIRRORS_PATH}"
-  else
-    BUILDKITE_AGENT_GIT_MIRRORS_PATH="/var/lib/buildkite-agent/git-mirrors"
+    mount -o bind "${EPHEMERAL_GIT_MIRRORS_PATH}" "${BUILDKITE_AGENT_GIT_MIRRORS_PATH}"
+    echo "${EPHEMERAL_GIT_MIRRORS_PATH} ${BUILDKITE_AGENT_GIT_MIRRORS_PATH} none defaults,bind 0 0" >>/etc/fstab
   fi
+
+  chown buildkite-agent: "${BUILDKITE_AGENT_GIT_MIRRORS_PATH}"
 fi
 
 BUILDKITE_AGENT_BUILD_PATH="/var/lib/buildkite-agent/builds"
+mkdir -p "${BUILDKITE_AGENT_BUILD_PATH}"
 if [ "${BUILDKITE_ENABLE_INSTANCE_STORAGE:-false}" == "true" ]
 then
-  BUILDKITE_AGENT_BUILD_PATH="/mnt/ephemeral/builds"
-
-  mkdir -p "${BUILDKITE_AGENT_BUILD_PATH}"
-  chown buildkite-agent: "${BUILDKITE_AGENT_BUILD_PATH}"
+  EPHEMERAL_BUILD_PATH="/mnt/ephemeral/builds"
+  mkdir -p "${EPHEMERAL_BUILD_PATH}"
+  mount -o bind "${EPHEMERAL_BUILD_PATH}" "${BUILDKITE_AGENT_BUILD_PATH}"
+  echo "${EPHEMERAL_BUILD_PATH} ${BUILDKITE_AGENT_BUILD_PATH} none defaults,bind 0 0" >>/etc/fstab
 fi
+chown buildkite-agent: "${BUILDKITE_AGENT_BUILD_PATH}"
 
 BUILDKITE_AGENT_TOKEN="$(aws ssm get-parameter --name "${BUILDKITE_AGENT_TOKEN_PATH}" --with-decryption --query Parameter.Value --output text)"
 
@@ -197,6 +203,15 @@ if [[ -n "${BUILDKITE_ELASTIC_BOOTSTRAP_SCRIPT}" ]] ; then
 	rm /tmp/elastic_bootstrap
 fi
 
+cat << EOF > /etc/lifecycled
+AWS_REGION=${AWS_REGION}
+LIFECYCLED_HANDLER=/usr/local/bin/stop-agent-gracefully
+LIFECYCLED_CLOUDWATCH_GROUP=/buildkite/lifecycled
+EOF
+
+systemctl enable lifecycled.service
+systemctl start lifecycled
+
 # wait for docker to start
 next_wait_time=0
 until docker ps || [ $next_wait_time -eq 5 ]; do

packer/linux/scripts/install-lifecycled.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -eu -o pipefail
+
+LIFECYCLED_VERSION=v3.2.0
+
+MACHINE=$(uname -m)
+
+case "${MACHINE}" in
+	x86_64)    ARCH=amd64;;
+	aarch64)   ARCH=arm64;;
+	*)         ARCH=unknown;;
+esac
+
+echo "Installing lifecycled ${LIFECYCLED_VERSION}..."
+
+sudo touch /etc/lifecycled
+sudo curl -Lf -o /usr/bin/lifecycled \
+	https://github.com/buildkite/lifecycled/releases/download/${LIFECYCLED_VERSION}/lifecycled-linux-${ARCH}
+sudo chmod +x /usr/bin/lifecycled
+sudo curl -Lf -o /etc/systemd/system/lifecycled.service \
+	https://raw.githubusercontent.com/buildkite/lifecycled/${LIFECYCLED_VERSION}/init/systemd/lifecycled.unit
+

packer/windows/buildkite-ami.json

@@ -44,6 +44,10 @@
       "type": "powershell",
       "script": "scripts/install-cloudwatch-agent.ps1"
     },
+    {
+      "type": "powershell",
+      "script": "scripts/install-lifecycled.ps1"
+    },
     {
       "type": "powershell",
       "script": "scripts/install-docker.ps1"

packer/windows/conf/bin/bk-install-elastic-stack.ps1

@@ -140,6 +140,10 @@ disconnect-after-job=${Env:BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB}
 "@
 $OFS=" "
 
+nssm set lifecycled AppEnvironmentExtra :AWS_REGION=$Env:AWS_REGION
+nssm set lifecycled AppEnvironmentExtra +LIFECYCLED_HANDLER="C:\buildkite-agent\bin\stop-agent-gracefully.ps1"
+Restart-Service lifecycled
+
 # wait for docker to start
 $next_wait_time=0
 do {

packer/windows/conf/cloudwatch-agent/amazon-cloudwatch-agent.json

@@ -22,6 +22,11 @@
             "file_path": "C:\\buildkite-agent\\buildkite-agent.log",
             "log_group_name": "/buildkite/buildkite-agent",
             "timestamp_format": "%Y-%m-%dT%H:%M:%S.%f"
+          },
+          {
+            "file_path": "C:\\lifecycled\\lifecycled.log",
+            "log_group_name": "/buildkite/lifecycled",
+            "timestamp_format": "%Y-%m-%dT%H:%M:%S.%f"
           }
         ]
       },

packer/windows/scripts/install-lifecycled.ps1

@@ -0,0 +1,19 @@
+# Stop script execution when a non-terminating error occurs
+$ErrorActionPreference = "Stop"
+
+$lifecycled_version = "v3.2.0"
+
+Write-Output "Installing lifecycled ${lifecycled_version}..."
+
+New-Item -ItemType directory -Path C:\lifecycled\bin
+
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+Invoke-WebRequest -OutFile C:\lifecycled\bin\lifecycled.exe https://github.com/buildkite/lifecycled/releases/download/${lifecycled_version}/lifecycled-windows-amd64.exe
+
+Write-Output "Configure lifecycled to run on startup..."
+nssm install lifecycled C:\lifecycled\bin\lifecycled.exe
+If ($lastexitcode -ne 0) { Exit $lastexitcode }
+nssm set lifecycled AppStdout C:\lifecycled\lifecycled.log
+If ($lastexitcode -ne 0) { Exit $lastexitcode }
+nssm set lifecycled AppStderr C:\lifecycled\lifecycled.log
+If ($lastexitcode -ne 0) { Exit $lastexitcode }

templates/aws-stack.yml

@@ -1054,6 +1054,8 @@ Resources:
         - OldestLaunchConfiguration
         - ClosestToNextInstanceHour
       MaxInstanceLifetime: !If [UseMaxInstanceLifetime, !Ref MaxInstanceLifetime, !Ref "AWS::NoValue"]
+      NewInstancesProtectedFromScaleIn: true
+
     CreationPolicy:
       ResourceSignal:
         Timeout: !If [ UseDefaultInstanceCreationTimeout, !If [ UseWindowsAgents, PT10M, PT5M ], !Ref InstanceCreationTimeout ]