Skip to content

Commit 088ab3c

Browse files
authored
ci: increase available disk space for GHA container image builds (#577)
* ci: increase available disk space for GHA container image builds This PR creates a LVM overlay, increasing the available disk space from previous 66GB to 82GB by default, and 106GB when building any amd/cude/pytorch/tensorflow image. * fixup from review, add intel
1 parent ea7f467 commit 088ab3c

File tree

3 files changed

+109
-6
lines changed

3 files changed

+109
-6
lines changed

.github/workflows/build-notebooks-TEMPLATE.yaml

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,38 @@ jobs:
3434
username: ${{ github.actor }}
3535
password: ${{ secrets.GITHUB_TOKEN }}
3636

37+
- name: Free up additional disk space
38+
# https://docs.github.com/en/actions/learn-github-actions/expressions
39+
if: "${{ contains(inputs.target, 'amd') || contains(inputs.target, 'cuda') || contains(inputs.target, 'intel') ||
40+
contains(inputs.target, 'pytorch') || contains(inputs.target, 'tensorflow') }}"
41+
run: |
42+
set -x
43+
44+
df -h
45+
46+
sudo rm -rf /usr/local/lib/android &
47+
sudo rm -rf /usr/local/share/boost &
48+
sudo rm -rf /usr/local/lib/node_modules &
49+
sudo rm -rf /usr/share/dotnet &
50+
sudo rm -rf /opt/ghc &
51+
sudo rm -rf /opt/hostedtoolcache/CodeQL &
52+
53+
sudo docker image prune --all --force &
54+
55+
wait
56+
57+
df -h
58+
59+
- name: Mount lvm overlay for podman builds
60+
run: |
61+
df -h
62+
free -h
63+
64+
bash ./ci/cached-builds/gha_lvm_overlay.bash
65+
66+
df -h
67+
free -h
68+
3769
# https://github.com/containers/buildah/issues/2521#issuecomment-884779112
3870
- name: Workaround https://github.com/containers/podman/issues/22152#issuecomment-2027705598
3971
run: sudo apt-get -qq remove podman crun
@@ -58,12 +90,10 @@ jobs:
5890
mkdir -p $HOME/.config/containers/
5991
cp ci/cached-builds/containers.conf $HOME/.config/containers/containers.conf
6092
cp ci/cached-builds/storage.conf $HOME/.config/containers/storage.conf
93+
6194
# should at least reset storage when touching storage.conf
62-
sudo mkdir -p /mnt/containers/
63-
sudo chown -R $USER:$USER /mnt/containers
6495
podman system reset --force
65-
# podman bug? need to create this _after_ doing the reset
66-
mkdir -p /mnt/containers/tmp
96+
mkdir -p $HOME/.local/share/containers/storage/tmp
6797
6898
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push
6999
- name: "push: make ${{ inputs.target }}"

ci/cached-builds/gha_lvm_overlay.bash

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/usr/bin/env bash
2+
set -Eeuo pipefail
3+
4+
# GitHub Actions runners have two disks, /dev/root and /dev/sda1.
5+
# We would like to be able to combine available disk space on both and use it for podman container builds.
6+
#
7+
# This script creates file-backed volumes on /dev/root and /dev/sda1, then creates ext4 over both, and mounts it for our use
8+
# https://github.com/easimon/maximize-build-space/blob/master/action.yml
9+
10+
root_reserve_mb=2048
11+
temp_reserve_mb=100
12+
swap_size_mb=4096
13+
14+
build_mount_path="${HOME}/.local/share/containers"
15+
build_mount_path_ownership="runner:runner"
16+
17+
pv_loop_path=/pv.img
18+
tmp_pv_loop_path=/mnt/tmp-pv.img
19+
overprovision_lvm=false
20+
21+
VG_NAME=buildvg
22+
23+
# github runners have an active swap file in /mnt/swapfile
24+
# we want to reuse the temp disk, so first unmount swap and clean the temp disk
25+
echo "Unmounting and removing swap file."
26+
sudo swapoff -a
27+
sudo rm -f /mnt/swapfile
28+
29+
echo "Creating LVM Volume."
30+
echo " Creating LVM PV on root fs."
31+
# create loop pv image on root fs
32+
ROOT_RESERVE_KB=$(expr ${root_reserve_mb} \* 1024)
33+
ROOT_FREE_KB=$(df --block-size=1024 --output=avail / | tail -1)
34+
ROOT_LVM_SIZE_KB=$(expr $ROOT_FREE_KB - $ROOT_RESERVE_KB)
35+
ROOT_LVM_SIZE_BYTES=$(expr $ROOT_LVM_SIZE_KB \* 1024)
36+
sudo touch "${pv_loop_path}" && sudo fallocate -z -l "${ROOT_LVM_SIZE_BYTES}" "${pv_loop_path}"
37+
export ROOT_LOOP_DEV=$(sudo losetup --find --show "${pv_loop_path}")
38+
sudo pvcreate -f "${ROOT_LOOP_DEV}"
39+
40+
# create pv on temp disk
41+
echo " Creating LVM PV on temp fs."
42+
TMP_RESERVE_KB=$(expr ${temp_reserve_mb} \* 1024)
43+
TMP_FREE_KB=$(df --block-size=1024 --output=avail /mnt | tail -1)
44+
TMP_LVM_SIZE_KB=$(expr $TMP_FREE_KB - $TMP_RESERVE_KB)
45+
TMP_LVM_SIZE_BYTES=$(expr $TMP_LVM_SIZE_KB \* 1024)
46+
sudo touch "${tmp_pv_loop_path}" && sudo fallocate -z -l "${TMP_LVM_SIZE_BYTES}" "${tmp_pv_loop_path}"
47+
export TMP_LOOP_DEV=$(sudo losetup --find --show "${tmp_pv_loop_path}")
48+
sudo pvcreate -f "${TMP_LOOP_DEV}"
49+
50+
# create volume group from these pvs
51+
sudo vgcreate "${VG_NAME}" "${TMP_LOOP_DEV}" "${ROOT_LOOP_DEV}"
52+
53+
echo "Recreating swap"
54+
# create and activate swap
55+
sudo lvcreate -L "${swap_size_mb}M" -n swap "${VG_NAME}"
56+
sudo mkswap "/dev/mapper/${VG_NAME}-swap"
57+
sudo swapon "/dev/mapper/${VG_NAME}-swap"
58+
59+
echo "Creating build volume"
60+
# create and mount build volume
61+
sudo lvcreate --type raid0 --stripes 2 --stripesize 4 --alloc anywhere --extents 100%FREE --name buildlv "${VG_NAME}"
62+
if [[ ${overprovision_lvm} == 'true' ]]; then
63+
sudo mkfs.ext4 -m0 "/dev/mapper/${VG_NAME}-buildlv"
64+
else
65+
sudo mkfs.ext4 -Enodiscard -m0 "/dev/mapper/${VG_NAME}-buildlv"
66+
fi
67+
sudo mount "/dev/mapper/${VG_NAME}-buildlv" "${build_mount_path}"
68+
sudo chown -R "${build_mount_path_ownership}" "${build_mount_path}"
69+
70+
# if build mount path is a parent of $GITHUB_WORKSPACE, and has been deleted, recreate it
71+
if [[ ! -d "${GITHUB_WORKSPACE}" ]]; then
72+
sudo mkdir -p "${GITHUB_WORKSPACE}"
73+
sudo chown -R "${WORKSPACE_OWNER}" "${GITHUB_WORKSPACE}"
74+
fi

ci/cached-builds/storage.conf

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,9 @@
22

33
[storage]
44
driver="overlay"
5-
rootless_storage_path="/mnt/containers"
65

76
[storage.options]
87
# https://www.redhat.com/sysadmin/faster-container-image-pulls
9-
pull_options = {enable_partial_images = "true", use_hard_links = "true", ostree_repos=""}
8+
pull_options = {enable_partial_images = "true", use_hard_links = "false", ostree_repos=""}
109

1110
[storage.options.overlay]

0 commit comments

Comments
 (0)