Skip to content

Commit 3ae6e8f

Browse files
rajachanrajachan
committed
Actions caching
Signed-off-by: Raghu Raja <[email protected]>
1 parent 8cdf492 commit 3ae6e8f

File tree

2 files changed

+140
-15
lines changed

2 files changed

+140
-15
lines changed

‎.github/workflows/distcheck.yaml‎

Lines changed: 78 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ env:
3838
# environment variable is also now needed, as of july 2024.
3939
# ref: https://github.com/actions/runner/issues/2906#issuecomment-2208546951
4040
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: 'true'
41+
42+
# APT package list for caching
43+
APT_CACHE_RESTORE_KEYS: |
44+
apt-cache-${{ runner.os }}-${{ github.job }}
4145
4246
concurrency:
4347
group: ${{ github.head_ref || github.run_id }}
@@ -190,19 +194,51 @@ jobs:
190194
sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 10
191195
fi
192196
197+
- name: Setup APT Cache for Distcheck
198+
id: apt-cache-distcheck
199+
uses: actions/cache@v3
200+
with:
201+
path: /var/cache/apt/archives
202+
key: apt-cache-${{ runner.os }}-distcheck-${{ hashFiles('**/workflows/distcheck.yaml') }}
203+
restore-keys: ${{ env.APT_CACHE_RESTORE_KEYS }}
204+
193205
- name: Install Base Dependencies
194206
run: |
195-
sudo apt-get update -y
207+
# Download packages without installing if they're not in cache
208+
sudo apt-get update -y || (sleep 10 && sudo apt-get update -y) || (sleep 30 && sudo apt-get update -y)
209+
sudo apt-get install -y --download-only ${{ env.APT_PACKAGES }} || true
210+
# Install from cache
196211
sudo apt-get install -y ${{ env.APT_PACKAGES }}
197212
213+
- name: Setup CUDA APT Cache
214+
id: cuda-apt-cache-distcheck
215+
if: matrix.sdk == 'cuda'
216+
uses: actions/cache@v3
217+
with:
218+
path: /var/cache/apt/archives
219+
key: cuda-apt-cache-distcheck-${{ runner.os }}-${{ hashFiles('**/workflows/distcheck.yaml') }}
220+
restore-keys: |
221+
cuda-apt-cache-${{ runner.os }}-
222+
198223
- name: Install CUDA SDK
199224
if: matrix.sdk == 'cuda'
200225
run: |
201-
sudo apt-get update -y && sudo apt-get install -y wget lsb-release
226+
# Install with retry logic
227+
sudo apt-get update -y || (sleep 10 && sudo apt-get update -y) || (sleep 30 && sudo apt-get update -y)
228+
sudo apt-get install -y wget lsb-release || (sleep 10 && sudo apt-get install -y wget lsb-release)
202229
repo="ubuntu$(lsb_release -r | cut -d':' -f2 | xargs | sed 's/[.]//g')"
203-
wget https://developer.download.nvidia.com/compute/cuda/repos/${repo}/$(uname -m)/cuda-keyring_1.1-1_all.deb
230+
for i in {1..3}; do
231+
if wget https://developer.download.nvidia.com/compute/cuda/repos/${repo}/$(uname -m)/cuda-keyring_1.1-1_all.deb; then
232+
break
233+
fi
234+
echo "Retrying wget in 10 seconds..."
235+
sleep 10
236+
done
204237
sudo dpkg -i cuda-keyring_1.1-1_all.deb
205-
sudo apt-get update -y
238+
sudo apt-get update -y || (sleep 10 && sudo apt-get update -y) || (sleep 30 && sudo apt-get update -y)
239+
# Download packages without installing if not in cache
240+
sudo apt-get install -y --download-only cuda-cudart-dev-12-6 cuda-crt-12-6 || true
241+
# Install from cache
206242
sudo apt-get install -y cuda-cudart-dev-12-6 cuda-crt-12-6
207243
208244
- name: Install lttng
@@ -281,24 +317,58 @@ jobs:
281317
- uses: actions/setup-python@v5
282318
with:
283319
python-version: '3.9'
320+
321+
- name: Setup APT Cache for CodeChecker
322+
id: apt-cache-codechecker
323+
uses: actions/cache@v3
324+
with:
325+
path: /var/cache/apt/archives
326+
key: apt-cache-${{ runner.os }}-codechecker-${{ hashFiles('**/workflows/distcheck.yaml') }}
327+
restore-keys: ${{ env.APT_CACHE_RESTORE_KEYS }}
284328

285329
- name: Install Base Dependencies
286330
run: |
287-
sudo apt-get update -y
331+
# Download packages without installing if they're not in cache
332+
sudo apt-get update -y || (sleep 10 && sudo apt-get update -y) || (sleep 30 && sudo apt-get update -y)
333+
sudo apt-get install -y --download-only ${{ env.APT_PACKAGES }} || true
334+
# Install from cache
288335
sudo apt-get install -y ${{ env.APT_PACKAGES }}
289336
337+
- name: Setup CUDA APT Cache
338+
id: cuda-apt-cache-codechecker
339+
if: matrix.sdk == 'cuda'
340+
uses: actions/cache@v3
341+
with:
342+
path: /var/cache/apt/archives
343+
key: cuda-apt-cache-codechecker-${{ runner.os }}-${{ hashFiles('**/workflows/distcheck.yaml') }}
344+
restore-keys: |
345+
cuda-apt-cache-${{ runner.os }}-
346+
290347
- name: Install CUDA SDK
291348
if: matrix.sdk == 'cuda'
292349
run: |
293-
sudo apt-get update -y && sudo apt-get install -y wget lsb-release
350+
# Install with retry logic
351+
sudo apt-get update -y || (sleep 10 && sudo apt-get update -y) || (sleep 30 && sudo apt-get update -y)
352+
sudo apt-get install -y wget lsb-release || (sleep 10 && sudo apt-get install -y wget lsb-release)
294353
repo="ubuntu$(lsb_release -r | cut -d':' -f2 | xargs | sed 's/[.]//g')"
295-
wget https://developer.download.nvidia.com/compute/cuda/repos/${repo}/$(uname -m)/cuda-keyring_1.1-1_all.deb
354+
for i in {1..3}; do
355+
if wget https://developer.download.nvidia.com/compute/cuda/repos/${repo}/$(uname -m)/cuda-keyring_1.1-1_all.deb; then
356+
break
357+
fi
358+
echo "Retrying wget in 10 seconds..."
359+
sleep 10
360+
done
296361
sudo dpkg -i cuda-keyring_1.1-1_all.deb
297-
sudo apt-get update -y
362+
sudo apt-get update -y || (sleep 10 && sudo apt-get update -y) || (sleep 30 && sudo apt-get update -y)
363+
# Download packages without installing if not in cache
364+
sudo apt-get install -y --download-only cuda-cudart-dev-12-6 cuda-crt-12-6 || true
365+
# Install from cache
298366
sudo apt-get install -y cuda-cudart-dev-12-6 cuda-crt-12-6
299367
300368
- name: Install cppcheck
301369
run: |
370+
sudo apt-get update -y || (sleep 10 && sudo apt-get update -y) || (sleep 30 && sudo apt-get update -y)
371+
sudo apt-get install -y --download-only cppcheck || true
302372
sudo apt-get install -y cppcheck
303373
304374
- name: Fetch and Install EFA Installer Dependencies

‎.github/workflows/tag-makedist.yaml‎

Lines changed: 62 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ on:
66
- release/**
77
- v**
88

9+
env:
10+
# For caching APT packages
11+
APT_CACHE_RESTORE_KEYS: |
12+
apt-cache-${{ runner.os }}-makedist
13+
914
jobs:
1015
amazonlinux:
1116
strategy:
@@ -29,27 +34,77 @@ jobs:
2934
container: ${{ matrix.container }}
3035
name: make dist for tag ${{ github.ref_name }}
3136
steps:
37+
- name: Setup YUM Cache
38+
uses: actions/cache@v3
39+
id: yum-cache
40+
with:
41+
path: /var/cache/yum
42+
key: yum-cache-${{ runner.os }}-${{ hashFiles('**/workflows/tag-makedist.yaml') }}
43+
restore-keys: |
44+
yum-cache-${{ runner.os }}-
45+
3246
- run: |
33-
yum -y update && yum -y install git tar util-linux findutils yum-utils
47+
# Configure YUM to keep cache
48+
echo 'keepcache=1' | tee -a /etc/yum.conf
49+
# Add retry logic
50+
for i in {1..3}; do
51+
if yum -y update && yum -y install git tar util-linux findutils yum-utils; then
52+
break
53+
fi
54+
echo "Retrying yum in 10 seconds..."
55+
sleep 10
56+
done
3457
- uses: actions/checkout@v4
3558
- name: Fetch and Install EFA Installer Dependencies
3659
run: |
37-
curl -O https://efa-installer.amazonaws.com/aws-efa-installer-${{ matrix.efainstaller }}.tar.gz
60+
# Add retry logic for curl
61+
for i in {1..3}; do
62+
if curl -O https://efa-installer.amazonaws.com/aws-efa-installer-${{ matrix.efainstaller }}.tar.gz; then
63+
break
64+
fi
65+
echo "Retrying curl in 10 seconds..."
66+
sleep 10
67+
done
3868
tar -xf aws-efa-installer-*.tar.gz
69+
# Install EFA RPMs from local directory to avoid network issues
3970
( cd aws-efa-installer/RPMS/${{ matrix.efainstallerdir }}/x86_64 ; find . | grep rpm$ | xargs yum -y localinstall )
4071
rm -rf aws-efa-installer*
4172
4273
- name: Install hwloc, utilities.
4374
run: |
44-
yum -y install hwloc-devel autoconf automake libtool gcc gcc-c++ git make
75+
# Add retry logic for package installation
76+
for i in {1..3}; do
77+
if yum -y install hwloc-devel autoconf automake libtool gcc gcc-c++ git make; then
78+
break
79+
fi
80+
echo "Retrying yum in 10 seconds..."
81+
sleep 10
82+
done
4583
4684
- name: Install CUDA
4785
run: |
48-
${{ matrix.configmanager }} --add-repo \
49-
http://developer.download.nvidia.com/compute/cuda/repos/${{ matrix.nvidiadistro }}/x86_64/cuda-${{ matrix.nvidiadistro }}.repo \
50-
--save
86+
# Add retry logic for repo config
87+
for i in {1..3}; do
88+
if ${{ matrix.configmanager }} --add-repo \
89+
http://developer.download.nvidia.com/compute/cuda/repos/${{ matrix.nvidiadistro }}/x86_64/cuda-${{ matrix.nvidiadistro }}.repo \
90+
--save; then
91+
break
92+
fi
93+
echo "Retrying repo add in 10 seconds..."
94+
sleep 10
95+
done
96+
97+
# Clean and refresh cache
5198
yum -y clean expire-cache
52-
yum -y install ${{ matrix.cudapackages }}
99+
100+
# Add retry logic for package installation
101+
for i in {1..3}; do
102+
if yum -y install ${{ matrix.cudapackages }}; then
103+
break
104+
fi
105+
echo "Retrying CUDA package installation in 10 seconds..."
106+
sleep 10
107+
done
53108
54109
- name: Call `autoreconf -ivf`
55110
run: |

0 commit comments

Comments
 (0)