Merge pull request #1567 from buildkite/SUP-4523/Adding-purge-builds

Add build purge option on disk full for Buildkite agents

Author: JoeColeman95

packer/linux/conf/bin/bk-check-disk-space.sh

@@ -15,7 +15,25 @@ if [[ $disk_avail -lt $DISK_MIN_AVAILABLE ]]; then
   disk_min_human=$(numfmt --to=iec-i --suffix=B --from-unit=1024 "${DISK_MIN_AVAILABLE}")
   disk_avail_human=$(numfmt --to=iec-i --suffix=B --from-unit=1024 "${disk_avail}")
   echo "Not enough disk space free: ${disk_avail_human} (${disk_avail}KB) available, cutoff is ${disk_min_human} (${DISK_MIN_AVAILABLE}KB) 🚨" >&2
-  exit 1
+
+  # Last resort for clearing space with build directory cleanup (if enabled)
+  if [[ "${BUILDKITE_PURGE_BUILDS_ON_DISK_FULL:-false}" == "true" ]]; then
+    echo "Purging all build directories to reclaim disk space..."
+    rm -rf "${BUILDKITE_AGENT_BUILD_PATH:-/var/lib/buildkite-agent/builds}"/*
+    disk_avail=$(df -k --output=avail "$DOCKER_DIR" | tail -n1)
+    disk_avail_human=$(numfmt --to=iec-i --suffix=B --from-unit=1024 "${disk_avail}")
+    echo "Disk space free after build purge: ${disk_avail_human} (${disk_avail}KB)"
+    if [[ $disk_avail -ge $DISK_MIN_AVAILABLE ]]; then
+      echo "Disk space sufficient after build purge. Continuing."
+      exit 0
+    else
+      echo "Disk health checks failed after build purge. Terminating agent." >&2
+      exit 1
+    fi
+  else
+    echo "Disk health checks failed. Build purge not enabled. Terminating agent." >&2
+    exit 1
+  fi
 fi
 
 inodes_avail=$(df -k --output=iavail "$DOCKER_DIR" | tail -n1)

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -135,7 +135,7 @@ set_always         "BUILDKITE_AGENTS_PER_INSTANCE" "$BUILDKITE_AGENTS_PER_INSTAN
 # also set via /etc/systemd/system/buildkite-agent.service.d/environment.conf
 set_always         "BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB" "$BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB"
 set_always         "BUILDKITE_TERMINATE_INSTANCE_ON_DISK_FULL" "$BUILDKITE_TERMINATE_INSTANCE_ON_DISK_FULL"
-
+set_always         "BUILDKITE_PURGE_BUILDS_ON_DISK_FULL" "$BUILDKITE_PURGE_BUILDS_ON_DISK_FULL"
 set_always         "BUILDKITE_ECR_POLICY" "${BUILDKITE_ECR_POLICY:-none}"
 set_always         "BUILDKITE_SECRETS_BUCKET" "$BUILDKITE_SECRETS_BUCKET"
 set_always         "BUILDKITE_SECRETS_BUCKET_REGION" "$BUILDKITE_SECRETS_BUCKET_REGION"

templates/aws-stack.yml

@@ -55,6 +55,7 @@ Metadata:
         - BuildkiteAgentCancelGracePeriod
         - BuildkiteAgentSignalGracePeriod
         - BuildkiteTerminateInstanceAfterJob
+        - BuildkitePurgeBuildsOnDiskFull
         - BuildkiteAdditionalSudoPermissions
         - BuildkiteWindowsAdministrator
         - BuildkiteAgentScalerServerlessARN
@@ -308,6 +309,14 @@ Parameters:
       - "false"
     Default: "false"
 
+  BuildkitePurgeBuildsOnDiskFull:
+    Description: Set to "true" to purge build directories as a last resort when disk space is critically low.
+    Type: String
+    AllowedValues:
+      - "true"
+      - "false"
+    Default: "false"
+
   BuildkiteAgentDisconnectAfterUptime:
     Description: >
       The maximum uptime in seconds before the agent stops accepting new jobs and shuts down
@@ -1800,6 +1809,7 @@ Resources:
                   BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB="${BuildkiteTerminateInstanceAfterJob}" \
                   BUILDKITE_AGENT_DISCONNECT_AFTER_UPTIME="${BuildkiteAgentDisconnectAfterUptime}" \
                   BUILDKITE_TERMINATE_INSTANCE_ON_DISK_FULL="${BuildkiteTerminateInstanceOnDiskFull}" \
+                  BUILDKITE_PURGE_BUILDS_ON_DISK_FULL="${BuildkitePurgeBuildsOnDiskFull}" \
                   BUILDKITE_ADDITIONAL_SUDO_PERMISSIONS="${BuildkiteAdditionalSudoPermissions}" \
                   AWS_DEFAULT_REGION="${AWS::Region}" \
                   SECRETS_PLUGIN_ENABLED="${EnableSecretsPlugin}" \