Allow minimum disk before cleanup to be customized

Author: Lachlan Donald

packer/linux/conf/bin/bk-check-disk-space.sh

@@ -1,25 +1,57 @@
 #!/bin/bash
 set -euo pipefail
 
-DISK_MIN_AVAILABLE=${DISK_MIN_AVAILABLE:-5242880} # 5GB
-DISK_MIN_INODES=${DISK_MIN_INODES:-250000} # docker needs lots
-
-DOCKER_DIR="/var/lib/docker/"
-
-disk_avail=$(df -k --output=avail "$DOCKER_DIR" | tail -n1)
-
-echo "Disk space free: $(df -k -h --output=avail "$DOCKER_DIR" | tail -n1 | sed -e 's/^[[:space:]]//')"
-
-if [[ $disk_avail -lt $DISK_MIN_AVAILABLE ]]; then
-  echo "Not enough disk space free, cutoff is ${DISK_MIN_AVAILABLE} 🚨" >&2
-  exit 1
+# Usage:
+# bk-check-disk-space.sh (min disk required) (min inodes required)
+# min disk required can be either an amount of bytes, a pattern like 10G
+# or 500M, or a percentage like 5%
+# min inodes must be a number, default to 250,000
+
+# Converts human-readable units like 1.43K and 120.3M to bytes
+dehumanize() {
+  awk '/[0-9][bB]?$/ {printf "%u\n", $1*1024}
+       /[tT][bB]?$/  {printf "%u\n", $1*(1024*1024*1024)}
+       /[gG][bB]?$/  {printf "%u\n", $1*(1024*1024)}
+       /[mM][bB]?$/  {printf "%u\n", $1*(1024)}
+       /[kK][bB]?$/  {printf "%u\n", $1*1}' <<< "$1"
+}
+
+min_available=${1:-5G}
+docker_dir="/var/lib/docker/"
+
+# First check the disk available
+
+disk_avail=$(df -k --output=avail "$docker_dir" | tail -n1)
+disk_avail_human=$(df -k -h --output=avail "$docker_dir" | tail -n1 | tr -d '[:space:]')
+disk_used_pct=$(df -k --output=pcent "$docker_dir" | tail -n1 | tr -d '[:space:]' | tr -d '%')
+disk_free_pct=$((100-disk_used_pct))
+
+printf "Disk space free: %s (%s%%)\\n" "$disk_avail_human" "$disk_free_pct"
+
+# Check if the min_available is a percentage
+if [[ $min_available =~ \%$ ]] ; then
+  if [[ $(echo "${disk_free_pct}<${min_available}" | sed 's/%//g' | bc) -gt 0 ]] ; then
+    echo "Not enough disk space free, cutoff is ${min_available} 🚨" >&2
+    exit 1
+  fi
+else
+  if [[ $disk_avail -lt $(dehumanize "$min_available") ]]; then
+    echo "Not enough disk space free, cutoff is ${min_available} 🚨" >&2
+    exit 1
+  fi
 fi
 
-inodes_avail=$(df -k --output=iavail "$DOCKER_DIR" | tail -n1)
+# Next check inodes, these can be exhausted by docker build operations
+
+inodes_min_available=${2:-250000}
+inodes_avail=$(df -k --output=iavail "$docker_dir" | tail -n1 | tr -d '[:space:]')
+inodes_avail_human=$(df -k -h --output=iavail "$docker_dir" | tail -n1 | tr -d '[:space:]')
+inodes_used_pct=$(df -k --output=ipcent "$docker_dir" | tail -n1 | tr -d '[:space:]' | tr -d '%')
+inodes_free_pct=$((100-inodes_used_pct))
 
-echo "Inodes free: $(df -k -h --output=iavail "$DOCKER_DIR" | tail -n1 | sed -e 's/^[[:space:]]//')"
+printf "Inodes free: %s (%s%%)\\n" "$inodes_avail_human" "$inodes_free_pct"
 
-if [[ $inodes_avail -lt $DISK_MIN_INODES ]]; then
-  echo "Not enough inodes free, cutoff is ${DISK_MIN_INODES} 🚨" >&2
+if [[ $inodes_avail -lt $inodes_min_available ]]; then
+  echo "Not enough inodes free, cutoff is ${inodes_min_available} 🚨" >&2
   exit 1
 fi

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -62,6 +62,12 @@ export PLUGINS_ENABLED="${PLUGINS_ENABLED[*]-}"
 export BUILDKITE_ECR_POLICY=${BUILDKITE_ECR_POLICY:-none}
 EOF
 
+# cron-env is sourced by crontab entries and low disk scripts
+cat << EOF > /var/lib/buildkite-agent/cron-env
+export DISK_MIN_AVAILABLE=$DISK_MIN_AVAILABLE
+export DOCKER_PRUNE_UNTIL=$DOCKER_PRUNE_UNTIL
+EOF
+
 if [[ "${BUILDKITE_AGENT_RELEASE}" == "edge" ]] ; then
 	echo "Downloading buildkite-agent edge..."
 	curl -Lsf -o /usr/bin/buildkite-agent-edge \

packer/linux/conf/buildkite-agent/hooks/environment

@@ -7,6 +7,11 @@ source ~/cfn-env
 echo "~~~ :llama: Setting up elastic stack environment ($BUILDKITE_STACK_VERSION)"
 cat ~/cfn-env
 
+if [[ -f ~/cron-env ]] ; then
+  # shellcheck source=/dev/null
+  source ~/cron-env
+fi
+
 echo "Checking docker"
 if ! docker ps ; then
   echo "^^^ +++"
@@ -17,13 +22,13 @@ if ! docker ps ; then
 fi
 
 echo "Checking disk space"
-if ! /usr/local/bin/bk-check-disk-space.sh ; then
+if ! /usr/local/bin/bk-check-disk-space.sh "${DISK_MIN_AVAILABLE:-5G}" ; then
 
   echo "Cleaning up docker resources older than ${DOCKER_PRUNE_UNTIL:-4h}"
   docker image prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL:-4h}"
 
   echo "Checking disk space again"
-  if ! /usr/local/bin/bk-check-disk-space.sh ; then
+  if ! /usr/local/bin/bk-check-disk-space.sh "${DISK_MIN_AVAILABLE:-5G}"; then
     echo "Disk health checks failed" >&2
     exit 1
   fi

packer/linux/conf/docker/cron.hourly/docker-gc

@@ -5,10 +5,16 @@ if [[ $EUID -eq 0 ]]; then
   exec >> /var/log/elastic-stack.log 2>&1 # Logs to elastic-stack.log
 fi
 
-DOCKER_PRUNE_UNTIL=${DOCKER_PRUNE_UNTIL:-4h}
+# Load config from file if it exists
+if [[ -f /var/lib/buildkite-agent/cron-env ]] ; then
+  # shellcheck source=/dev/null
+  source /var/lib/buildkite-agent/cron-env
+else
+  DOCKER_PRUNE_UNTIL=4h
+fi
 
 ## ------------------------------------------
 ## Prune stuff that doesn't affect cache hits
 
-docker network prune --force --filter "until=${DOCKER_PRUNE_UNTIL}"
-docker container prune --force --filter "until=${DOCKER_PRUNE_UNTIL}"
+docker network prune --force --filter "until=${!DOCKER_PRUNE_UNTIL}"
+docker container prune --force --filter "until=${!DOCKER_PRUNE_UNTIL}"

packer/linux/conf/docker/cron.hourly/docker-low-disk-gc

@@ -5,8 +5,6 @@ if [[ $EUID -eq 0 ]]; then
   exec >> /var/log/elastic-stack.log 2>&1 # Logs to elastic-stack.log
 fi
 
-DOCKER_PRUNE_UNTIL=${DOCKER_PRUNE_UNTIL:-1h}
-
 mark_instance_unhealthy() {
   # cancel any running buildkite builds
   killall -QUIT buildkite-agent || true
@@ -19,14 +17,22 @@ mark_instance_unhealthy() {
 
 trap mark_instance_unhealthy ERR
 
+# Load config from file if it exists
+if [[ -f /var/lib/buildkite-agent/cron-env ]] ; then
+  # shellcheck source=/dev/null
+  source /var/lib/buildkite-agent/cron-env
+else
+  DISK_MIN_AVAILABLE=5G
+fi
+
 ## -----------------------------------------------------------------
 ## Check disk, we only want to prune images/containers if we have to
 
-if ! /usr/local/bin/bk-check-disk-space.sh ; then
-  echo "Cleaning up docker resources older than ${DOCKER_PRUNE_UNTIL}"
-  docker image prune --all --force --filter "until=${DOCKER_PRUNE_UNTIL}"
+if ! /usr/local/bin/bk-check-disk-space.sh "$DISK_MIN_AVAILABLE" ; then
+  echo "Cleaning up docker resources older than 1h"
+  docker image prune --all --force --filter "until=1h"
 
-  if ! /usr/local/bin/bk-check-disk-space.sh ; then
+  if ! /usr/local/bin/bk-check-disk-space.sh "$DISK_MIN_AVAILABLE" ; then
     echo "Disk health checks failed" >&2
     exit 1
   fi

templates/aws-stack.yml

@@ -375,6 +375,16 @@ Parameters:
       - "false"
     Default: "false"
 
+  MinimumDiskAvailableBeforeCleanup:
+    Type: String
+    Description: Either a percentage (%) or absolute unit (B, MB, GB) of disk below which disk cleanup is run
+    Default: "2GB"
+
+  DockerPruneUntil:
+    Type: String
+    Description: How far back to prune docker networks images and containers on hourly cleanup
+    Default: "4h"
+
 Outputs:
   VpcId:
     Value:
@@ -857,6 +867,8 @@ Resources:
                   BUILDKITE_ECR_POLICY=${ECRAccessPolicy} \
                   BUILDKITE_TERMINATE_INSTANCE_AFTER_JOB=${BuildkiteTerminateInstanceAfterJob} \
                   BUILDKITE_ADDITIONAL_SUDO_PERMISSIONS=${BuildkiteAdditionalSudoPermissions} \
+                  DISK_MIN_AVAILABLE="${MinimumDiskAvailableBeforeCleanup}" \
+                  DOCKER_PRUNE_UNTIL="${DockerPruneUntil} \
                   AWS_DEFAULT_REGION=${AWS::Region} \
                   SECRETS_PLUGIN_ENABLED=${EnableSecretsPlugin} \
                   ECR_PLUGIN_ENABLED=${EnableECRPlugin} \