Add configurable Docker garbage collection (Windows/Linux)

goss.yaml

@@ -160,12 +160,6 @@ command:
   /usr/local/bin/docker-gc:
     exit-status: 0
 
-  systemctl is-enabled docker-low-disk-gc.timer:
-    exit-status: 0
-
-  /usr/local/bin/docker-low-disk-gc:
-    exit-status: 0
-
   # test that we can build a docker image
   docker buildx build -f tests/Dockerfile --progress=plain -t buildkite-postgres:latest tests:
     exit-status: 0

packer/linux/conf/bin/bk-configure-docker-gc.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+set -euo pipefail
+
+echo "Configuring docker cleanup"
+
+DOCKER_GC_SCHEDULE="${DOCKER_GC_SCHEDULE:-hourly}"
+DOCKER_GC_PRUNE_UNTIL="${DOCKER_GC_PRUNE_UNTIL:-4h}"
+DOCKER_GC_PRUNE_IMAGES="${DOCKER_GC_PRUNE_IMAGES:-false}"
+DOCKER_GC_PRUNE_VOLUMES="${DOCKER_GC_PRUNE_VOLUMES:-false}"
+
+if ! [[ "$DOCKER_GC_PRUNE_UNTIL" =~ ^[0-9]+[smhd]$ ]]; then
+  echo "Warning: time format not expected: $DOCKER_GC_PRUNE_UNTIL" >&2
+  echo "use format like 4h, 30m, 1d" >&2
+fi
+
+case "$DOCKER_GC_SCHEDULE" in
+hourly | daily | weekly | monthly) ;;
+*[0-9]*) ;;
+*)
+  echo "Warning: time format not expected - $DOCKER_GC_SCHEDULE" >&2
+  echo "use hourly, daily, weekly, monthly" >&2
+  ;;
+esac
+
+echo "Schedule: $DOCKER_GC_SCHEDULE"
+echo "Prune older than: $DOCKER_GC_PRUNE_UNTIL"
+echo "Cleaning all images: $DOCKER_GC_PRUNE_IMAGES"
+echo "Volumes: $DOCKER_GC_PRUNE_VOLUMES"
+
+cat >/usr/local/bin/docker-gc <<'EOF'
+#!/bin/bash
+set -euo pipefail
+
+if [[ $EUID -eq 0 ]]; then
+    exec >> /var/log/elastic-stack.log 2>&1
+fi
+
+echo "$(date): Docker cleanup starting"
+
+TIME_FILTER="--filter until=DOCKER_PRUNE_UNTIL_PLACEHOLDER"
+
+echo "Cleaning networks and containers"
+docker network prune --force
+docker container prune --force $TIME_FILTER
+
+if [[ "DOCKER_GC_PRUNE_IMAGES_PLACEHOLDER" == "true" ]]; then
+    echo "Cleaning all images"
+    docker image prune --all --force $TIME_FILTER
+else
+    echo "Cleaning dangling images only"
+    docker image prune --force $TIME_FILTER
+fi
+
+if [[ "DOCKER_GC_PRUNE_VOLUMES_PLACEHOLDER" == "true" ]]; then
+    echo "Cleaning volumes"
+    docker volume prune --force
+fi
+
+echo "Cleanup Done"
+EOF
+
+sed -i "s/DOCKER_PRUNE_UNTIL_PLACEHOLDER/$DOCKER_GC_PRUNE_UNTIL/g" /usr/local/bin/docker-gc
+sed -i "s/DOCKER_GC_PRUNE_IMAGES_PLACEHOLDER/$DOCKER_GC_PRUNE_IMAGES/g" /usr/local/bin/docker-gc
+sed -i "s/DOCKER_GC_PRUNE_VOLUMES_PLACEHOLDER/$DOCKER_GC_PRUNE_VOLUMES/g" /usr/local/bin/docker-gc
+
+chmod +x /usr/local/bin/docker-gc
+
+cat >/etc/systemd/system/docker-gc.timer <<EOF
+[Unit]
+Description=Docker GC Cleanup Timer
+Requires=docker-gc.service
+
+[Timer]
+Unit=docker-gc.service
+OnCalendar=${DOCKER_GC_SCHEDULE}
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+EOF
+
+cat >/etc/systemd/system/docker-gc.service <<EOF
+[Unit]
+Description=Docker GC Cleanup
+Wants=docker-gc.timer
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/docker-gc
+StandardOutput=journal
+StandardError=journal
+EOF
+
+echo "Enabling timer"
+systemctl daemon-reload || {
+  echo "Warning: systemctl daemon-reload failed, retrying in 5 seconds"
+  sleep 5
+  systemctl daemon-reload || {
+    echo "Error: systemctl daemon-reload failed twice, skipping timer setup"
+    exit 0
+  }
+}
+
+systemctl enable docker-gc.timer || {
+  echo "Warning: failed to enable docker-gc.timer"
+}
+
+systemctl start docker-gc.timer || {
+  echo "Warning: failed to start docker-gc.timer, will retry later"
+}
+
+echo "Docker GC Cleanup configured"
+echo "Schedule: $DOCKER_GC_SCHEDULE"
+echo "Prune older than: $DOCKER_GC_PRUNE_UNTIL"
+if [[ "$DOCKER_GC_PRUNE_IMAGES" == "true" ]]; then
+  echo "Will clean all images"
+else
+  echo "Will clean dangling images only"
+fi
+if [[ "$DOCKER_GC_PRUNE_VOLUMES" == "true" ]]; then
+  echo "Will clean volumes"
+else
+  echo "Volumes left alone"
+fi
+
+echo Restarting docker daemon...
+systemctl restart docker

packer/linux/conf/bin/bk-configure-docker.sh

@@ -95,11 +95,5 @@ else
   echo Instance storage not configured.
 fi
 
-echo Cleaning up docker images...
-systemctl start docker-low-disk-gc.service
-
-echo Enabling docker-gc timers...
-systemctl enable docker-gc.timer docker-low-disk-gc.timer
-
 echo Restarting docker daemon...
 systemctl restart docker

packer/linux/conf/bin/bk-install-elastic-stack.sh

@@ -143,6 +143,12 @@ set_always         "BUILDKITE_STACK_NAME" "$BUILDKITE_STACK_NAME"
 set_always         "BUILDKITE_STACK_VERSION" "$BUILDKITE_STACK_VERSION"
 set_always         "BUILDKITE_DOCKER_EXPERIMENTAL" "$DOCKER_EXPERIMENTAL"
 set_always         "DOCKER_USERNS_REMAP" "$DOCKER_USERNS_REMAP"
+set_always         "DOCKER_GC_SCHEDULE" "$DOCKER_GC_SCHEDULE"
+set_always         "DOCKER_GC_PRUNE_UNTIL" "$DOCKER_GC_PRUNE_UNTIL"
+set_always         "DOCKER_GC_PRUNE_IMAGES" "$DOCKER_GC_PRUNE_IMAGES"
+set_always         "DOCKER_GC_PRUNE_VOLUMES" "$DOCKER_GC_PRUNE_VOLUMES"
+set_always         "DISK_MIN_AVAILABLE" "$DISK_MIN_AVAILABLE"
+set_always         "DISK_MIN_INODES" "$DISK_MIN_INODES"
 set_always         "DOCKER_VERSION" "$DOCKER_VERSION"
 set_always         "PLUGINS_ENABLED" "${PLUGINS_ENABLED[*]-}"
 set_always         "BUILDKITE_ARTIFACTS_BUCKET" "$BUILDKITE_ARTIFACTS_BUCKET"
@@ -418,6 +424,17 @@ fi
 echo "Waited $next_wait_time times for docker to start. We will exit if it still has not started."
 check_docker
 
+echo "Configuring Docker garbage collection..."
+if [[ -f "/usr/local/bin/bk-configure-docker-gc.sh" ]]; then
+  if /usr/local/bin/bk-configure-docker-gc.sh; then
+    echo "Docker GC configuration completed successfully"
+  else
+    echo "Warning: Docker GC configuration failed, continuing with installation..."
+  fi
+else
+  echo "Warning: bk-configure-docker-gc.sh not found, skipping Docker GC configuration"
+fi
+
 echo Writing buildkite-agent systemd environment override...
 # also set in /var/lib/buildkite-agent/cfn-env so that it's shown in the job logs
 mkdir -p /etc/systemd/system/buildkite-agent.service.d

packer/linux/scripts/install-docker.sh

@@ -17,9 +17,11 @@ sudo mkdir -p /etc/docker
 sudo cp /tmp/conf/docker/daemon.json /etc/docker/daemon.json
 
 echo "Adding docker systemd timers..."
-sudo cp /tmp/conf/docker/scripts/* /usr/local/bin
+if ls /tmp/conf/docker/scripts/* >/dev/null 2>&1; then
+  sudo cp /tmp/conf/docker/scripts/* /usr/local/bin
+  sudo chmod +x /usr/local/bin/docker-*
+fi
 sudo cp /tmp/conf/docker/systemd/docker-* /etc/systemd/system
-sudo chmod +x /usr/local/bin/docker-*
 
 echo "Installing docker buildx..."
 DOCKER_CLI_DIR=/usr/libexec/docker/cli-plugins