Skip to content

Commit caad83a

Browse files
konardkonard
authored
Merge pull request #63 from link-foundation/issue-62-3002f19b72a9
fix: make CI toolchain tests fatal and add post-release smoke tests (issue #62)
2 parents db0d2a4 + 901653e commit caad83a

File tree

4 files changed

+53915
-26
lines changed

4 files changed

+53915
-26
lines changed

‎.changeset/fix-ci-tests-fatal-and-comprehensive.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
---
2+
bump: patch
3+
---
4+
5+
fix: make CI toolchain tests fatal and comprehensive (issue #62)
6+
7+
Previously, all toolchain test commands in `docker-build-test` used
8+
`|| echo "test failed"` patterns, making every failure non-fatal. An image
9+
where cargo, python, go, or any tool was missing would still pass CI and
10+
be released.
11+
12+
Changes:
13+
- Add `set -e` to all test steps so any failed command fails the CI job
14+
- Remove all `|| echo "..."` fallback patterns from test commands
15+
- Add comprehensive toolchain tests covering all installed runtimes:
16+
cargo, rustup (Rust), pip3, python3 (Python via pyenv), gem, ruby
17+
(Ruby via rbenv), kotlin (JVM via SDKMAN), swift, dotnet, Rscript (R)
18+
- Add post-release smoke tests in `docker-build-push` job that run
19+
against the actually-published image (not just the locally built one),
20+
so every main/dispatch release is validated before downstream jobs run
21+
- Add case study docs in `docs/case-studies/issue-62/`

‎.github/workflows/release.yml‎

Lines changed: 121 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -450,48 +450,85 @@ jobs:
450450
451451
- name: Test JS sandbox
452452
run: |
453+
set -e
453454
echo "=== Testing JS sandbox ==="
454-
docker run --rm sandbox-js bash -c '. $HOME/.nvm/nvm.sh && node --version' || echo "Node.js test failed"
455-
docker run --rm sandbox-js bash -c 'export PATH=$HOME/.bun/bin:$PATH && bun --version' || echo "Bun test failed"
456-
docker run --rm sandbox-js bash -c 'export PATH=$HOME/.deno/bin:$PATH && deno --version' || echo "Deno test failed"
457-
echo "=== JS sandbox tests completed ==="
455+
docker run --rm sandbox-js bash -c '. $HOME/.nvm/nvm.sh && node --version'
456+
docker run --rm sandbox-js bash -c 'export PATH=$HOME/.bun/bin:$PATH && bun --version'
457+
docker run --rm sandbox-js bash -c 'export PATH=$HOME/.deno/bin:$PATH && deno --version'
458+
echo "=== JS sandbox tests passed ==="
458459
459460
- name: Test essentials sandbox
460461
run: |
462+
set -e
461463
echo "=== Testing essentials sandbox ==="
462-
docker run --rm sandbox-essentials gh --version || echo "GitHub CLI test failed"
463-
docker run --rm sandbox-essentials glab --version || echo "GitLab CLI test failed"
464-
docker run --rm sandbox-essentials gh-setup-git-identity --version || echo "gh-setup-git-identity test failed"
465-
docker run --rm sandbox-essentials glab-setup-git-identity --version || echo "glab-setup-git-identity test failed"
466-
echo "=== Essentials sandbox tests completed ==="
464+
docker run --rm sandbox-essentials gh --version
465+
docker run --rm sandbox-essentials glab --version
466+
docker run --rm sandbox-essentials gh-setup-git-identity --version
467+
docker run --rm sandbox-essentials glab-setup-git-identity --version
468+
echo "=== Essentials sandbox tests passed ==="
467469
468470
- name: Test full sandbox
469471
run: |
472+
set -e
470473
echo "=== Testing full sandbox ==="
471474
echo "Note: Using entrypoint script which initializes all environments"
472475
473-
docker run --rm sandbox-test node --version || echo "Node.js test failed"
474-
docker run --rm sandbox-test python --version || echo "Python test failed"
475-
docker run --rm sandbox-test go version || echo "Go test failed"
476-
docker run --rm sandbox-test rustc --version || echo "Rust test failed"
477-
docker run --rm sandbox-test java -version || echo "Java test failed"
478-
docker run --rm sandbox-test bun --version || echo "Bun test failed"
479-
docker run --rm sandbox-test deno --version || echo "Deno test failed"
480-
docker run --rm sandbox-test gh --version || echo "GitHub CLI test failed"
481-
docker run --rm sandbox-test glab --version || echo "GitLab CLI test failed"
482-
docker run --rm sandbox-test gh-setup-git-identity --version || echo "gh-setup-git-identity test failed"
483-
docker run --rm sandbox-test glab-setup-git-identity --version || echo "glab-setup-git-identity test failed"
484-
docker run --rm sandbox-test lean --version || echo "Lean test failed"
485-
docker run --rm sandbox-test perl --version || echo "Perl test failed"
486-
docker run --rm sandbox-test php --version || echo "PHP test failed"
476+
# JavaScript/TypeScript runtimes
477+
docker run --rm sandbox-test node --version
478+
docker run --rm sandbox-test bun --version
479+
docker run --rm sandbox-test deno --version
480+
481+
# Python (pyenv)
482+
docker run --rm sandbox-test python3 --version
483+
docker run --rm sandbox-test pip3 --version
484+
485+
# Go
486+
docker run --rm sandbox-test go version
487+
488+
# Rust (rustup + cargo + rustc)
489+
docker run --rm sandbox-test rustc --version
490+
docker run --rm sandbox-test cargo --version
491+
docker run --rm sandbox-test rustup --version
492+
493+
# Java/JVM (SDKMAN)
494+
docker run --rm sandbox-test java -version
495+
docker run --rm sandbox-test kotlin -version
496+
497+
# Ruby (rbenv)
498+
docker run --rm sandbox-test ruby --version
499+
docker run --rm sandbox-test gem --version
500+
501+
# PHP
502+
docker run --rm sandbox-test php --version
503+
504+
# Perl (perlbrew)
505+
docker run --rm sandbox-test perl --version
506+
507+
# Swift
508+
docker run --rm sandbox-test swift --version
509+
510+
# Lean/Mathlib (elan)
511+
docker run --rm sandbox-test lean --version
512+
513+
# Dotnet
514+
docker run --rm sandbox-test dotnet --version
515+
516+
# R language
517+
docker run --rm sandbox-test Rscript --version
518+
519+
# CLI tools
520+
docker run --rm sandbox-test gh --version
521+
docker run --rm sandbox-test glab --version
522+
docker run --rm sandbox-test gh-setup-git-identity --version
523+
docker run --rm sandbox-test glab-setup-git-identity --version
487524
488525
echo ""
489526
echo "=== PHP install method check ==="
490-
docker run --rm sandbox-php cat /home/sandbox/.php-install-method || echo "PHP method marker not found"
491-
docker run --rm sandbox-test cat /home/sandbox/.php-install-method || echo "PHP method marker not found in full sandbox"
527+
docker run --rm sandbox-php cat /home/sandbox/.php-install-method
528+
docker run --rm sandbox-test cat /home/sandbox/.php-install-method
492529
493530
echo ""
494-
echo "=== All tests completed ==="
531+
echo "=== All full sandbox tests passed ==="
495532
496533
# === BUILD JS SANDBOX (amd64) ===
497534
# JS sandbox is the base layer - built first, other images depend on it
@@ -1449,6 +1486,64 @@ jobs:
14491486
cache-from: type=gha
14501487
cache-to: type=gha,mode=max
14511488

1489+
# Smoke test the published image to verify all toolchains work (issue #62)
1490+
# Tests each toolchain command to catch cases where tools are missing/broken
1491+
- name: Smoke test released full sandbox (amd64)
1492+
run: |
1493+
set -e
1494+
VERSION="${{ steps.version.outputs.version }}"
1495+
IMAGE="${{ env.DOCKERHUB_IMAGE_NAME }}:${VERSION}"
1496+
echo "=== Smoke testing released image: ${IMAGE} ==="
1497+
1498+
# JavaScript/TypeScript runtimes
1499+
docker run --rm "${IMAGE}" node --version
1500+
docker run --rm "${IMAGE}" bun --version
1501+
docker run --rm "${IMAGE}" deno --version
1502+
1503+
# Python (pyenv)
1504+
docker run --rm "${IMAGE}" python3 --version
1505+
docker run --rm "${IMAGE}" pip3 --version
1506+
1507+
# Go
1508+
docker run --rm "${IMAGE}" go version
1509+
1510+
# Rust (rustup + cargo + rustc)
1511+
docker run --rm "${IMAGE}" rustc --version
1512+
docker run --rm "${IMAGE}" cargo --version
1513+
docker run --rm "${IMAGE}" rustup --version
1514+
1515+
# Java/JVM (SDKMAN)
1516+
docker run --rm "${IMAGE}" java -version
1517+
docker run --rm "${IMAGE}" kotlin -version
1518+
1519+
# Ruby (rbenv)
1520+
docker run --rm "${IMAGE}" ruby --version
1521+
docker run --rm "${IMAGE}" gem --version
1522+
1523+
# PHP
1524+
docker run --rm "${IMAGE}" php --version
1525+
1526+
# Perl (perlbrew)
1527+
docker run --rm "${IMAGE}" perl --version
1528+
1529+
# Swift
1530+
docker run --rm "${IMAGE}" swift --version
1531+
1532+
# Lean/Mathlib (elan)
1533+
docker run --rm "${IMAGE}" lean --version
1534+
1535+
# Dotnet
1536+
docker run --rm "${IMAGE}" dotnet --version
1537+
1538+
# R language
1539+
docker run --rm "${IMAGE}" Rscript --version
1540+
1541+
# CLI tools
1542+
docker run --rm "${IMAGE}" gh --version
1543+
docker run --rm "${IMAGE}" glab --version
1544+
1545+
echo "=== All smoke tests passed for ${IMAGE} ==="
1546+
14521547
# === BUILD AND PUSH ARM64 IMAGE ===
14531548
# Using native ARM64 runner for optimal build performance
14541549
docker-build-push-arm64:

0 commit comments

Comments
 (0)