3737 build-outcome : ${{ steps.build.outcome }}
3838 archive-outcome : ${{ steps.archive.outcome }}
3939 counterexamples-outcome : ${{ steps.counterexamples.outcome }}
40+ cache-staging-has-files : ${{ steps.cache_staging_check.outputs.has_files }}
4041 get-cache-outcome : ${{ steps.get.outcome }}
4142 lint-outcome : ${{ steps.lint.outcome }}
4243 mk_all-outcome : ${{ steps.mk_all.outcome }}
@@ -367,31 +368,6 @@ jobs:
367368 cd pr-branch
368369 du .lake/build/lib/lean/Mathlib || echo "This code should be unreachable"
369370
370- # The cache secrets are available here, so we must not run any untrusted code.
371- - name : Upload cache to Azure
372- # We only upload the cache if the build started (whether succeeding, failing, or cancelled)
373- # but not if any earlier step failed or was cancelled.
374- # See discussion at https://leanprover.zulipchat.com/#narrow/stream/287929-mathlib4/topic/Some.20files.20not.20found.20in.20the.20cache/near/407183836
375- if : ${{ always() && steps.get.outcome == 'success' }}
376- shell : bash
377- run : |
378- cd pr-branch
379-
380- # Trim trailing whitespace from secrets to prevent issues with accidentally added spaces
381- export MATHLIB_CACHE_SAS="${MATHLIB_CACHE_SAS_RAW%"${MATHLIB_CACHE_SAS_RAW##*[![:space:]]}"}"
382-
383- # Use the trusted cache tool from tools-branch
384- # TODO: this is not doing anything currently, and needs to be integrated with put-unpacked
385- # ../tools-branch/.lake/build/bin/cache commit || true
386- # run this in CI if it gets an incorrect lake hash for existing cache files somehow
387- # ../tools-branch/.lake/build/bin/cache put!
388- # do not try to upload files just downloaded
389-
390- echo "Uploading cache to Azure..."
391- MATHLIB_CACHE_USE_CLOUDFLARE=0 ../tools-branch/.lake/build/bin/cache --repo=${{ github.event.pull_request.head.repo.full_name || github.repository }} put-unpacked
392- env :
393- MATHLIB_CACHE_SAS_RAW : ${{ secrets.MATHLIB_CACHE_SAS }}
394-
395371 # Note: we should not be including `Archive` and `Counterexamples` in the cache.
396372 # We do this for now for the sake of not rebuilding them in every CI run
397373 # even when they are not touched.
@@ -423,6 +399,47 @@ jobs:
423399 ../tools-branch/scripts/lake-build-with-retry.sh Counterexamples
424400 # results of build at pr-branch/.lake/build_summary_Counterexamples.json
425401
402+ - name : stage Mathlib cache files
403+ if : ${{ always() && (steps.build.outcome == 'success' || steps.build.outcome == 'failure' || steps.build.outcome == 'cancelled') }}
404+ shell : bash
405+ run : |
406+ rm -rf cache-staging
407+ mkdir -p cache-staging
408+ cd pr-branch
409+ lake env ../tools-branch/.lake/build/bin/cache --staging-dir="../cache-staging" stage
410+
411+ - name : stage Archive cache files
412+ if : ${{ steps.archive.outcome == 'success' }}
413+ shell : bash
414+ run : |
415+ cd pr-branch
416+ lake env ../tools-branch/.lake/build/bin/cache --staging-dir="../cache-staging" stage Archive.lean
417+
418+ - name : stage Counterexamples cache files
419+ if : ${{ steps.counterexamples.outcome == 'success' }}
420+ shell : bash
421+ run : |
422+ cd pr-branch
423+ lake env ../tools-branch/.lake/build/bin/cache --staging-dir="../cache-staging" stage Counterexamples.lean
424+
425+ - name : check cache staging contents
426+ id : cache_staging_check
427+ if : ${{ always() && (steps.build.outcome == 'success' || steps.build.outcome == 'failure' || steps.build.outcome == 'cancelled') }}
428+ shell : bash
429+ run : |
430+ if find cache-staging -type f -name '*.ltar' -print -quit | grep -q .; then
431+ echo "has_files=true" >> "$GITHUB_OUTPUT"
432+ else
433+ echo "has_files=false" >> "$GITHUB_OUTPUT"
434+ fi
435+
436+ - name : upload cache staging artifact
437+ if : ${{ always() && steps.cache_staging_check.outputs.has_files == 'true' }}
438+ uses : actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
439+ with :
440+ name : cache-staging
441+ path : cache-staging/
442+
426443 - name : Check if building Archive or Counterexamples failed
427444 if : steps.archive.outcome == 'failure' || steps.counterexamples.outcome == 'failure'
428445 run : |
@@ -434,21 +451,6 @@ jobs:
434451 fi
435452 exit 1
436453
437- # The cache secrets are available here, so we must not run any untrusted code.
438- - name : Upload Archive and Counterexamples cache to Azure
439- shell : bash
440- run : |
441- cd pr-branch
442-
443- # Trim trailing whitespace from secrets to prevent issues with accidentally added spaces
444- export MATHLIB_CACHE_SAS="${MATHLIB_CACHE_SAS_RAW%"${MATHLIB_CACHE_SAS_RAW##*[![:space:]]}"}"
445-
446- echo "Uploading Archive and Counterexamples cache to Azure..."
447- MATHLIB_CACHE_USE_CLOUDFLARE=0 ../tools-branch/.lake/build/bin/cache --repo=${{ github.event.pull_request.head.repo.full_name || github.repository }} put-unpacked Archive.lean
448- MATHLIB_CACHE_USE_CLOUDFLARE=0 ../tools-branch/.lake/build/bin/cache --repo=${{ github.event.pull_request.head.repo.full_name || github.repository }} put-unpacked Counterexamples.lean
449- env :
450- MATHLIB_CACHE_SAS_RAW : ${{ secrets.MATHLIB_CACHE_SAS }}
451-
452454 - name : Check {Mathlib, Tactic, Counterexamples, Archive}.lean
453455 if : ${{ always() && steps.mk_all.outcome != 'skipped' }}
454456 run : |
@@ -595,9 +597,60 @@ jobs:
595597 tools-branch/scripts/lean-pr-testing-comments.sh lean
596598 tools-branch/scripts/lean-pr-testing-comments.sh batteries
597599
600+ upload_cache :
601+ name : Upload to cache
602+ needs : [build]
603+ runs-on : ubuntu-latest # These steps run on a GitHub runner; no landrun sandboxing is needed.
604+ # We only upload the cache if the build started (whether succeeding, failing, or cancelled)
605+ # but not if any earlier step failed or was cancelled.
606+ # See discussion at https://leanprover.zulipchat.com/#narrow/stream/287929-mathlib4/topic/Some.20files.20not.20found.20in.20the.20cache/near/407183836
607+ if : ${{ always() && needs.build.outputs.cache-staging-has-files == 'true' }}
608+ steps :
609+ - name : Checkout tools branch
610+ uses : actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
611+ with :
612+ ref : ${{ inputs.tools_branch_ref != '' && inputs.tools_branch_ref || (github.repository == 'leanprover-community/mathlib4-nightly-testing' && 'nightly-testing-green' || 'master') }}
613+ fetch-depth : 1
614+
615+ - name : Configure Lean
616+ uses : leanprover/lean-action@c544e89643240c6b398f14a431bcdc6309e36b3e # v1.4.0
617+ with :
618+ auto-config : false # Don't run `lake build`, `lake test`, or `lake lint` automatically.
619+ use-github-cache : false
620+ use-mathlib-cache : false # This can be re-enabled once we are confident in the cache again.
621+ reinstall-transient-toolchain : true
622+
623+ - name : build cache executable
624+ run : |
625+ lake build cache
626+ CACHE_BIN="$(pwd)/.lake/build/bin/cache"
627+ if [ ! -x "$CACHE_BIN" ]; then
628+ echo "cache binary not found: $CACHE_BIN"
629+ exit 1
630+ fi
631+ echo "CACHE_BIN=$CACHE_BIN" >> "$GITHUB_ENV"
632+
633+ - name : Download cache staging artifact
634+ uses : actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
635+ with :
636+ name : cache-staging
637+ path : cache-staging
638+
639+ - name : Upload staged cache to Azure
640+ shell : bash
641+ run : |
642+ # Trim trailing whitespace from secrets to prevent issues with accidentally added spaces
643+ export MATHLIB_CACHE_SAS="${MATHLIB_CACHE_SAS_RAW%"${MATHLIB_CACHE_SAS_RAW##*[![:space:]]}"}"
644+
645+ echo "Uploading cache to Azure..."
646+ MATHLIB_CACHE_USE_CLOUDFLARE=0 lake env "$CACHE_BIN" put-staged --staging-dir="cache-staging" --repo=${{ github.event.pull_request.head.repo.full_name || github.repository }}
647+ env :
648+ MATHLIB_CACHE_SAS_RAW : ${{ secrets.MATHLIB_CACHE_SAS }}
649+
598650 post_steps :
599651 name : Post-Build Step
600- needs : [build]
652+ needs : [build, upload_cache]
653+ if : ${{ always() && needs.build.result == 'success' && (needs.upload_cache.result == 'success' || needs.upload_cache.result == 'skipped') }}
601654 runs-on : ubuntu-latest # Note these steps run on disposable GitHub runners, so no landrun sandboxing is needed.
602655 steps :
603656
@@ -674,6 +727,7 @@ jobs:
674727
675728 # We no longer run `lean4checker` in regular CI, as it is quite expensive for little benefit.
676729 # Instead we run it in a cron job on master: see `daily.yml`.
730+
677731 style_lint :
678732 name : Lint style
679733 runs-on : ubuntu-latest
@@ -686,7 +740,8 @@ jobs:
686740
687741 final :
688742 name : Post-CI job
689- if : ${{ inputs.run_post_ci }}
743+ # ensure that this runs iff direct dependencies succeeded even if transitive dependencies were skipped
744+ if : ${{ always() && inputs.run_post_ci && needs.style_lint.result == 'success' && needs.build.result == 'success' && needs.post_steps.result == 'success' }}
690745 needs : [style_lint, build, post_steps]
691746 runs-on : ubuntu-latest
692747 steps :
0 commit comments