From 762c81c9206ff2df19850afad5c24603b08a0598 Mon Sep 17 00:00:00 2001 From: Xiaoyu Date: Wed, 21 Jan 2026 21:24:13 +0000 Subject: [PATCH 1/3] clean up CI image disk space --- .azure_pipelines/job_templates/build-docker-image-template.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure_pipelines/job_templates/build-docker-image-template.yaml b/.azure_pipelines/job_templates/build-docker-image-template.yaml index 9d671c1329..3dd46ad34b 100644 --- a/.azure_pipelines/job_templates/build-docker-image-template.yaml +++ b/.azure_pipelines/job_templates/build-docker-image-template.yaml @@ -10,6 +10,8 @@ parameters: steps: - script: | docker login -u $(docker-username) -p $(docker-password) + docker system prune -af --volumes || true + docker builder prune -af || true docker build \ --build-arg BASE_IMAGE=${{ parameters.base_image }} \ --build-arg TENSORRT_VERSION=${{ parameters.trt_version }} \ From 47c592ecab16cc3560a9424491ca5b47cea4e57b Mon Sep 17 00:00:00 2001 From: Xiaoyu Date: Thu, 22 Jan 2026 07:19:55 +0000 Subject: [PATCH 2/3] test --- .../build-docker-image-template.yaml | 82 +++++++++++++++++-- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/.azure_pipelines/job_templates/build-docker-image-template.yaml b/.azure_pipelines/job_templates/build-docker-image-template.yaml index 3dd46ad34b..6cfb69f61a 100644 --- a/.azure_pipelines/job_templates/build-docker-image-template.yaml +++ b/.azure_pipelines/job_templates/build-docker-image-template.yaml @@ -9,20 +9,84 @@ parameters: steps: - script: | + set -euxo pipefail + + echo "=== BEFORE CLEAN: disk ===" + df -h + df -ih || true + + echo "=== BEFORE CLEAN: docker usage ===" + docker version + docker info --format 'DockerRootDir={{.DockerRootDir}}' + docker system df -v || true + + echo "=== BEFORE CLEAN: biggest dirs (/) ===" + sudo du -xhd1 / 2>/dev/null | sort -hr | head -n 30 || true + + echo "=== login ===" docker login -u $(docker-username) -p $(docker-password) + + echo "=== CLEAN ===" docker system prune -af --volumes || true docker builder prune -af || true + + echo "=== AFTER CLEAN: disk ===" + df -h + df -ih || true + + echo "=== AFTER CLEAN: docker usage ===" + docker system df -v || true + + echo "=== AFTER CLEAN: docker root breakdown ===" + DOCKER_ROOT="$(docker info --format '{{.DockerRootDir}}')" + sudo du -sh "$DOCKER_ROOT" 2>/dev/null || true + sudo du -xhd2 "$DOCKER_ROOT" 2>/dev/null | sort -hr | head -n 50 || true + + echo "=== AFTER CLEAN: buildx cache (if available) ===" + docker buildx ls || true + docker buildx du --verbose || true + + displayName: Preflight & Cleanup (find disk hogs) + +- script: | + set -euxo pipefail + + echo "=== BUILD ===" docker build \ - --build-arg BASE_IMAGE=${{ parameters.base_image }} \ - --build-arg TENSORRT_VERSION=${{ parameters.trt_version }} \ - --build-arg PYTHON_VERSION=${{ parameters.python_version }} \ - -t ${{ parameters.docker_image }} \ - -f $(Build.SourcesDirectory)/${{ parameters.dockerfile }} . + --build-arg BASE_IMAGE=${{ parameters.base_image }} \ + --build-arg TENSORRT_VERSION=${{ parameters.trt_version }} \ + --build-arg PYTHON_VERSION=${{ parameters.python_version }} \ + -t ${{ parameters.docker_image }} \ + -f $(Build.SourcesDirectory)/${{ parameters.dockerfile }} . + displayName: Build Docker Image - script: | - docker version - docker image ls - docker system df + set -euxo pipefail + + echo "=== AFTER BUILD: disk ===" df -h - displayName: Check Docker Images + df -ih || true + + echo "=== AFTER BUILD: images ===" + docker image ls + + echo "=== AFTER BUILD: docker usage ===" + docker system df -v || true + + echo "=== AFTER BUILD: docker root breakdown ===" + DOCKER_ROOT="$(docker info --format '{{.DockerRootDir}}')" + sudo du -sh "$DOCKER_ROOT" 2>/dev/null || true + sudo du -xhd2 "$DOCKER_ROOT" 2>/dev/null | sort -hr | head -n 50 || true + + echo "=== AFTER BUILD: workspace hogs ===" + echo "PIPELINE_WORKSPACE=$PIPELINE_WORKSPACE" + echo "BUILD_SOURCESDIRECTORY=$BUILD_SOURCESDIRECTORY" + du -sh "$PIPELINE_WORKSPACE" 2>/dev/null || true + du -sh "$BUILD_SOURCESDIRECTORY" 2>/dev/null || true + sudo find "$PIPELINE_WORKSPACE" -type f -printf '%s %p\n' 2>/dev/null | sort -nr | head -n 50 || true + + echo "=== AFTER BUILD: buildx cache (if available) ===" + docker buildx du --verbose || true + + displayName: Postcheck (what consumed disk) From 10b1f9de2bb8b9a719ab95b96168ac0982fdf72c Mon Sep 17 00:00:00 2001 From: Xiaoyu Date: Thu, 22 Jan 2026 07:50:06 +0000 Subject: [PATCH 3/3] move docker --- .../job_templates/build-docker-image-template.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.azure_pipelines/job_templates/build-docker-image-template.yaml b/.azure_pipelines/job_templates/build-docker-image-template.yaml index 6cfb69f61a..43462517e0 100644 --- a/.azure_pipelines/job_templates/build-docker-image-template.yaml +++ b/.azure_pipelines/job_templates/build-docker-image-template.yaml @@ -8,6 +8,20 @@ parameters: trt_version: '' steps: +- script: | + set -euxo pipefail + + echo "=== Move Docker data to larger disk ===" + # /mnt/storage has ~1TB available, while / is often low on space + if [ -d /mnt/storage ] && [ ! -f /etc/docker/daemon.json ]; then + sudo mkdir -p /mnt/storage/docker + echo '{"data-root": "/mnt/storage/docker"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + echo "Docker data-root moved to /mnt/storage/docker" + fi + + displayName: Move Docker to larger disk + - script: | set -euxo pipefail