Skip to content

Commit 9ef2aa8

Browse files
skylar-simoncelliskylar-simoncelli
authored
feat: cache validation + option cache wipe + prune stale cache files + misc fixes (#959)
* feat: validation + cache wipe option * fix: cleanup stale files in cache * fix: chain-specs pre-merge sha * fix: glob > find * fix: post merge save * fix: .ready * fix: awk > cut * fix: validate step deps * fix: more useful output
1 parent d6bc2d2 commit 9ef2aa8

File tree

1 file changed

+235
-23
lines changed

1 file changed

+235
-23
lines changed

‎.github/workflows/ci.yml

Lines changed: 235 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@ on:
1111
description: 'Commit SHA to build'
1212
required: true
1313
type: string
14+
wipe_cache:
15+
description: 'Wipe /tmp/rust-cache before build'
16+
required: false
17+
type: boolean
18+
default: false
1419

1520
env:
1621
AWS_REGION: "eu-central-1"
@@ -45,17 +50,81 @@ jobs:
4550
run: |
4651
set -e
4752
echo "--- Restoring Rust cache from host ---"
48-
if [ -d /tmp/rust-cache ]; then
49-
echo "Rust cache found. Restoring..."
53+
if [ -d /tmp/rust-cache ] && [ -f /tmp/rust-cache/.ready ]; then
54+
echo "Ready cache found. Restoring..."
5055
mkdir -p ~/.cargo/git ~/.cargo/registry ./target
5156
nix shell nixpkgs#rsync -c bash -c "
5257
rsync -a /tmp/rust-cache/cargo-git/ ~/.cargo/git/
5358
rsync -a /tmp/rust-cache/cargo-registry/ ~/.cargo/registry/
5459
rsync -a /tmp/rust-cache/target/ ./target/
5560
"
5661
else
57-
echo "No Rust cache found. Will be created after build."
62+
echo "No ready cache; skipping restore."
5863
fi
64+
- name: Validate Rust cache
65+
run: |
66+
set -euo pipefail
67+
CACHE=/tmp/rust-cache
68+
CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
69+
TARGET_DIR="${CARGO_TARGET_DIR:-$GITHUB_WORKSPACE/target}"
70+
[[ -d "$CACHE" ]] || { echo "No shared cache present."; exit 0; }
71+
72+
echo "--- Cache preflight ---"
73+
RUSTC="$(nix develop -c rustc -vV 2>/dev/null || rustc -vV)"
74+
LOCK_SHA="$(sha256sum Cargo.lock | cut -d' ' -f1)"
75+
MAN="$CACHE/.manifest"; READY="$CACHE/.ready"
76+
WANT="$(printf '%s\n%s\n' "$RUSTC" "$LOCK_SHA")"
77+
78+
BAD=0; REASONS=()
79+
80+
echo "[1/4] Checking manifest and ready marker..."
81+
if [[ ! -f "$READY" || ! -f "$MAN" ]]; then
82+
echo "manifest/ready missing"
83+
BAD=1; REASONS+=("manifest/ready missing")
84+
else
85+
HAVE="$(head -n 2 "$MAN" || true)"
86+
if [[ "$WANT" != "$HAVE" ]]; then
87+
echo "manifest/ready mismatch"
88+
BAD=1; REASONS+=("manifest mismatch")
89+
fi
90+
fi
91+
92+
echo "[2/4] Scanning for broken symlinks..."
93+
if find "$CARGO_HOME" "$TARGET_DIR" -xtype l -print -quit >/dev/null 2>&1; then
94+
echo "Broken symlink detected:"
95+
find "$CARGO_HOME" "$TARGET_DIR" -xtype l -print 2>/dev/null | head -5 || true
96+
BAD=1; REASONS+=("broken symlink(s)")
97+
fi
98+
99+
echo "[3/4] Scanning for zero-byte files..."
100+
if find "$CARGO_HOME" "$TARGET_DIR" -type f -size 0 -print -quit >/dev/null 2>&1; then
101+
echo "Zero-byte file detected:"
102+
find "$CARGO_HOME" "$TARGET_DIR" -type f -size 0 -print 2>/dev/null | head -5 || true
103+
BAD=1; REASONS+=("zero-byte file(s)")
104+
fi
105+
106+
echo "[4/4] Running cargo fetch integrity check..."
107+
if ! nix develop -c cargo fetch --locked -q; then
108+
echo "cargo fetch failed"
109+
BAD=1; REASONS+=("cargo fetch failed")
110+
fi
111+
112+
if [[ $BAD -eq 1 ]]; then
113+
echo "--- Purging suspect parts of cache ---"
114+
if [[ ${#REASONS[@]} -gt 1 ]]; then
115+
echo "Multiple issues (${REASONS[*]}): removing entire target/"
116+
rm -rf "$TARGET_DIR" || true
117+
else
118+
echo "Single issue (${REASONS[*]}): purging build dirs"
119+
find "$TARGET_DIR" -type d -name build -prune -exec rm -rf {} + || true
120+
fi
121+
echo "Clearing registry/src and git/checkouts..."
122+
rm -rf "$CARGO_HOME"/registry/src/* "$CARGO_HOME"/git/checkouts/* || true
123+
else
124+
echo "Cache looks healthy."
125+
fi
126+
127+
echo "--- Preflight summary: ${REASONS[*]:-OK} ---"
59128
- name: Release cache lock
60129
if: always()
61130
run: rmdir /tmp/rust-cache.lock
@@ -82,18 +151,21 @@ jobs:
82151
sleep 5
83152
done
84153
- name: Save Rust cache to host
85-
if: always()
86154
run: |
87155
set -e
88156
echo "--- Saving Rust cache to host cache ---"
89157
mkdir -p /tmp/rust-cache
90158
nix shell nixpkgs#rsync -c bash -c "
91-
rsync -a ~/.cargo/git/ /tmp/rust-cache/cargo-git/
92-
rsync -a ~/.cargo/registry/ /tmp/rust-cache/cargo-registry/
93-
rsync -a ./target/ /tmp/rust-cache/target/
159+
rsync -a --delete ~/.cargo/git/ /tmp/rust-cache/cargo-git/
160+
rsync -a --delete ~/.cargo/registry/ /tmp/rust-cache/cargo-registry/
161+
rsync -a --delete ./target/ /tmp/rust-cache/target/
94162
"
95163
echo "--- Verifying cache contents ---"
96164
ls -laR /tmp/rust-cache
165+
RUSTC="$(nix develop -c rustc -vV 2>/dev/null || rustc -vV)"
166+
LOCK_SHA="$(sha256sum Cargo.lock | cut -d' ' -f1)"
167+
printf '%s\n%s\n' "$RUSTC" "$LOCK_SHA" > /tmp/rust-cache/.manifest
168+
touch /tmp/rust-cache/.ready
97169
- name: Release cache lock
98170
if: always()
99171
run: rmdir /tmp/rust-cache.lock
@@ -225,7 +297,7 @@ jobs:
225297
- name: Upload chain spec artifacts to Kubernetes
226298
uses: ./.github/actions/deploy/upload-chain-specs
227299
with:
228-
sha: ${{ needs.build-linux-x86_64-pre-merge.outputs.sha }}
300+
sha: ${{ github.sha }}
229301
env:
230302
kubeconfig_base64: ${{ secrets.kubeconfig_base64 }}
231303
K8S_SERVER: ${{ secrets.K8S_SERVER }}
@@ -278,17 +350,81 @@ jobs:
278350
run: |
279351
set -e
280352
echo "--- Restoring Rust cache from host ---"
281-
if [ -d /tmp/rust-cache ]; then
282-
echo "Rust cache found. Restoring..."
353+
if [ -d /tmp/rust-cache ] && [ -f /tmp/rust-cache/.ready ]; then
354+
echo "Ready cache found. Restoring..."
283355
mkdir -p ~/.cargo/git ~/.cargo/registry ./target
284356
nix shell nixpkgs#rsync -c bash -c "
285357
rsync -a /tmp/rust-cache/cargo-git/ ~/.cargo/git/
286358
rsync -a /tmp/rust-cache/cargo-registry/ ~/.cargo/registry/
287359
rsync -a /tmp/rust-cache/target/ ./target/
288360
"
289361
else
290-
echo "No Rust cache found. Will be created after build."
362+
echo "No ready cache; skipping restore."
363+
fi
364+
- name: Validate Rust cache
365+
run: |
366+
set -euo pipefail
367+
CACHE=/tmp/rust-cache
368+
CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
369+
TARGET_DIR="${CARGO_TARGET_DIR:-$GITHUB_WORKSPACE/target}"
370+
[[ -d "$CACHE" ]] || { echo "No shared cache present."; exit 0; }
371+
372+
echo "--- Cache preflight ---"
373+
RUSTC="$(nix develop -c rustc -vV 2>/dev/null || rustc -vV)"
374+
LOCK_SHA="$(sha256sum Cargo.lock | cut -d' ' -f1)"
375+
MAN="$CACHE/.manifest"; READY="$CACHE/.ready"
376+
WANT="$(printf '%s\n%s\n' "$RUSTC" "$LOCK_SHA")"
377+
378+
BAD=0; REASONS=()
379+
380+
echo "[1/4] Checking manifest and ready marker..."
381+
if [[ ! -f "$READY" || ! -f "$MAN" ]]; then
382+
echo "manifest/ready missing"
383+
BAD=1; REASONS+=("manifest/ready missing")
384+
else
385+
HAVE="$(head -n 2 "$MAN" || true)"
386+
if [[ "$WANT" != "$HAVE" ]]; then
387+
echo "manifest/ready mismatch"
388+
BAD=1; REASONS+=("manifest mismatch")
389+
fi
390+
fi
391+
392+
echo "[2/4] Scanning for broken symlinks..."
393+
if find "$CARGO_HOME" "$TARGET_DIR" -xtype l -print -quit >/dev/null 2>&1; then
394+
echo "Broken symlink detected:"
395+
find "$CARGO_HOME" "$TARGET_DIR" -xtype l -print 2>/dev/null | head -5 || true
396+
BAD=1; REASONS+=("broken symlink(s)")
397+
fi
398+
399+
echo "[3/4] Scanning for zero-byte files..."
400+
if find "$CARGO_HOME" "$TARGET_DIR" -type f -size 0 -print -quit >/dev/null 2>&1; then
401+
echo "Zero-byte file detected:"
402+
find "$CARGO_HOME" "$TARGET_DIR" -type f -size 0 -print 2>/dev/null | head -5 || true
403+
BAD=1; REASONS+=("zero-byte file(s)")
404+
fi
405+
406+
echo "[4/4] Running cargo fetch integrity check..."
407+
if ! nix develop -c cargo fetch --locked -q; then
408+
echo "cargo fetch failed"
409+
BAD=1; REASONS+=("cargo fetch failed")
410+
fi
411+
412+
if [[ $BAD -eq 1 ]]; then
413+
echo "--- Purging suspect parts of cache ---"
414+
if [[ ${#REASONS[@]} -gt 1 ]]; then
415+
echo "Multiple issues (${REASONS[*]}): removing entire target/"
416+
rm -rf "$TARGET_DIR" || true
417+
else
418+
echo "Single issue (${REASONS[*]}): purging build dirs"
419+
find "$TARGET_DIR" -type d -name build -prune -exec rm -rf {} + || true
420+
fi
421+
echo "Clearing registry/src and git/checkouts..."
422+
rm -rf "$CARGO_HOME"/registry/src/* "$CARGO_HOME"/git/checkouts/* || true
423+
else
424+
echo "Cache looks healthy."
291425
fi
426+
427+
echo "--- Preflight summary: ${REASONS[*]:-OK} ---"
292428
- name: Release cache lock
293429
if: always()
294430
run: rmdir /tmp/rust-cache.lock
@@ -339,18 +475,21 @@ jobs:
339475
sleep 5
340476
done
341477
- name: Save Rust cache to host
342-
if: always()
343478
run: |
344479
set -e
345480
echo "--- Saving Rust cache to host cache ---"
346481
mkdir -p /tmp/rust-cache
347482
nix shell nixpkgs#rsync -c bash -c "
348-
rsync -a ~/.cargo/git/ /tmp/rust-cache/cargo-git/
349-
rsync -a ~/.cargo/registry/ /tmp/rust-cache/cargo-registry/
350-
rsync -a ./target/ /tmp/rust-cache/target/
483+
rsync -a --delete ~/.cargo/git/ /tmp/rust-cache/cargo-git/
484+
rsync -a --delete ~/.cargo/registry/ /tmp/rust-cache/cargo-registry/
485+
rsync -a --delete ./target/ /tmp/rust-cache/target/
351486
"
352487
echo "--- Verifying cache contents ---"
353488
ls -laR /tmp/rust-cache
489+
RUSTC="$(nix develop -c rustc -vV 2>/dev/null || rustc -vV)"
490+
LOCK_SHA="$(sha256sum Cargo.lock | awk '{print $1}')"
491+
printf '%s\n%s\n' "$RUSTC" "$LOCK_SHA" > /tmp/rust-cache/.manifest
492+
touch /tmp/rust-cache/.ready
354493
- name: Release cache lock
355494
if: always()
356495
run: rmdir /tmp/rust-cache.lock
@@ -538,6 +677,12 @@ jobs:
538677
fetch-depth: 0
539678
ref: ${{ inputs.sha }}
540679
clean: true
680+
- name: (Optional) Wipe shared Rust cache
681+
if: github.event_name == 'workflow_dispatch' && inputs.wipe_cache == true
682+
run: |
683+
until mkdir /tmp/rust-cache.lock 2>/dev/null; do sleep 2; done
684+
rm -rf /tmp/rust-cache/
685+
rmdir /tmp/rust-cache.lock || true
541686
- name: Acquire cache lock
542687
run: |
543688
until mkdir /tmp/rust-cache.lock 2>/dev/null; do
@@ -547,17 +692,81 @@ jobs:
547692
run: |
548693
set -e
549694
echo "--- Restoring Rust cache from host ---"
550-
if [ -d /tmp/rust-cache ]; then
551-
echo "Rust cache found. Restoring..."
695+
if [ -d /tmp/rust-cache ] && [ -f /tmp/rust-cache/.ready ]; then
696+
echo "Ready cache found. Restoring..."
552697
mkdir -p ~/.cargo/git ~/.cargo/registry ./target
553698
nix shell nixpkgs#rsync -c bash -c "
554699
rsync -a /tmp/rust-cache/cargo-git/ ~/.cargo/git/
555700
rsync -a /tmp/rust-cache/cargo-registry/ ~/.cargo/registry/
556701
rsync -a /tmp/rust-cache/target/ ./target/
557702
"
558703
else
559-
echo "No Rust cache found. Will be created after build."
704+
echo "No ready cache; skipping restore."
705+
fi
706+
- name: Validate Rust cache
707+
run: |
708+
set -euo pipefail
709+
CACHE=/tmp/rust-cache
710+
CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
711+
TARGET_DIR="${CARGO_TARGET_DIR:-$GITHUB_WORKSPACE/target}"
712+
[[ -d "$CACHE" ]] || { echo "No shared cache present."; exit 0; }
713+
714+
echo "--- Cache preflight ---"
715+
RUSTC="$(nix develop -c rustc -vV 2>/dev/null || rustc -vV)"
716+
LOCK_SHA="$(sha256sum Cargo.lock | cut -d' ' -f1)"
717+
MAN="$CACHE/.manifest"; READY="$CACHE/.ready"
718+
WANT="$(printf '%s\n%s\n' "$RUSTC" "$LOCK_SHA")"
719+
720+
BAD=0; REASONS=()
721+
722+
echo "[1/4] Checking manifest and ready marker..."
723+
if [[ ! -f "$READY" || ! -f "$MAN" ]]; then
724+
echo "manifest/ready missing"
725+
BAD=1; REASONS+=("manifest/ready missing")
726+
else
727+
HAVE="$(head -n 2 "$MAN" || true)"
728+
if [[ "$WANT" != "$HAVE" ]]; then
729+
echo "manifest/ready mismatch"
730+
BAD=1; REASONS+=("manifest mismatch")
731+
fi
560732
fi
733+
734+
echo "[2/4] Scanning for broken symlinks..."
735+
if find "$CARGO_HOME" "$TARGET_DIR" -xtype l -print -quit >/dev/null 2>&1; then
736+
echo "Broken symlink detected:"
737+
find "$CARGO_HOME" "$TARGET_DIR" -xtype l -print 2>/dev/null | head -5 || true
738+
BAD=1; REASONS+=("broken symlink(s)")
739+
fi
740+
741+
echo "[3/4] Scanning for zero-byte files..."
742+
if find "$CARGO_HOME" "$TARGET_DIR" -type f -size 0 -print -quit >/dev/null 2>&1; then
743+
echo "Zero-byte file detected:"
744+
find "$CARGO_HOME" "$TARGET_DIR" -type f -size 0 -print 2>/dev/null | head -5 || true
745+
BAD=1; REASONS+=("zero-byte file(s)")
746+
fi
747+
748+
echo "[4/4] Running cargo fetch integrity check..."
749+
if ! nix develop -c cargo fetch --locked -q; then
750+
echo "cargo fetch failed"
751+
BAD=1; REASONS+=("cargo fetch failed")
752+
fi
753+
754+
if [[ $BAD -eq 1 ]]; then
755+
echo "--- Purging suspect parts of cache ---"
756+
if [[ ${#REASONS[@]} -gt 1 ]]; then
757+
echo "Multiple issues (${REASONS[*]}): removing entire target/"
758+
rm -rf "$TARGET_DIR" || true
759+
else
760+
echo "Single issue (${REASONS[*]}): purging build dirs"
761+
find "$TARGET_DIR" -type d -name build -prune -exec rm -rf {} + || true
762+
fi
763+
echo "Clearing registry/src and git/checkouts..."
764+
rm -rf "$CARGO_HOME"/registry/src/* "$CARGO_HOME"/git/checkouts/* || true
765+
else
766+
echo "Cache looks healthy."
767+
fi
768+
769+
echo "--- Preflight summary: ${REASONS[*]:-OK} ---"
561770
- name: Release cache lock
562771
if: always()
563772
run: rmdir /tmp/rust-cache.lock
@@ -584,18 +793,21 @@ jobs:
584793
sleep 5
585794
done
586795
- name: Save Rust cache to host
587-
if: always()
588796
run: |
589797
set -e
590798
echo "--- Saving Rust cache to host cache ---"
591799
mkdir -p /tmp/rust-cache
592800
nix shell nixpkgs#rsync -c bash -c "
593-
rsync -a ~/.cargo/git/ /tmp/rust-cache/cargo-git/
594-
rsync -a ~/.cargo/registry/ /tmp/rust-cache/cargo-registry/
595-
rsync -a ./target/ /tmp/rust-cache/target/
801+
rsync -a --delete ~/.cargo/git/ /tmp/rust-cache/cargo-git/
802+
rsync -a --delete ~/.cargo/registry/ /tmp/rust-cache/cargo-registry/
803+
rsync -a --delete ./target/ /tmp/rust-cache/target/
596804
"
597805
echo "--- Verifying cache contents ---"
598806
ls -laR /tmp/rust-cache
807+
RUSTC="$(nix develop -c rustc -vV 2>/dev/null || rustc -vV)"
808+
LOCK_SHA="$(sha256sum Cargo.lock | awk '{print $1}')"
809+
printf '%s\n%s\n' "$RUSTC" "$LOCK_SHA" > /tmp/rust-cache/.manifest
810+
touch /tmp/rust-cache/.ready
599811
- name: Release cache lock
600812
if: always()
601813
run: rmdir /tmp/rust-cache.lock
@@ -696,4 +908,4 @@ jobs:
696908
exit 1
697909
else
698910
echo "All needed jobs passed."
699-
fi
911+
fi

0 commit comments

Comments
 (0)