zBuild-testing-chunks #28
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: zBuild-testing-chunks | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| chunks_make: | |
| runs-on: ubuntu-24.04 | |
| env: | |
| BASENAME: package_image_beforeBoot.tar.flx | |
| BASE: ./_local/package_image_beforeBoot.tar.flx | |
| RELEASE_TAG: build-${{ github.run_id }}-${{ github.run_attempt }} | |
| steps: | |
| - name: Maximize build space | |
| uses: easimon/maximize-build-space@master | |
| with: | |
| root-reserve-mb: 4125 | |
| temp-reserve-mb: 1950 | |
| swap-size-mb: 2 | |
| remove-dotnet: true | |
| remove-android: true | |
| remove-haskell: true | |
| remove-codeql: true | |
| remove-docker-images: true | |
| - name: Checkout (after mount) | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 1 | |
| submodules: recursive | |
| - name: Quick system info | |
| shell: bash | |
| run: | | |
| echo "Memory and swap:"; sudo free -h; echo | |
| sudo swapon --show || true; echo | |
| echo "Available storage:"; sudo df -h | |
| - name: Prep workspace | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| mkdir -p ./_local | |
| df -h . | |
| - name: Generate 30GB test file | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| BYTES=30000000000 | |
| head -c "${BYTES}" /dev/zero \ | |
| | openssl enc -aes-256-ctr -pass pass:16918050001 -nosalt \ | |
| > "${BASE}" | |
| ls -lh "${BASE}" | |
| sha256sum "${BASE}" | tee ./_local/original.sha256 | |
| - name: Split into ~1.86 GiB parts (tail+truncate style) | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| F="${BASE}" | |
| CHUNK=1997537280 | |
| for i in $(seq -w 0 63); do | |
| [[ -s "$F" ]] || break | |
| tail -c "$CHUNK" "$F" > "$F".part"$i" | |
| size=$(stat -c%s "$F") | |
| if (( size > CHUNK )); then truncate -s -$CHUNK "$F"; else truncate -s 0 "$F"; fi | |
| done | |
| echo "Generated parts:" | |
| ls -lh "${BASE}".part* || { echo "No parts produced"; exit 1; } | |
| parts_count=$(ls -1 "${BASE}".part* | wc -l) | |
| echo "parts_count=${parts_count}" | |
| [[ "${parts_count}" -ge 15 ]] | |
| - name: Sanity check (verify ubiquitous_bash.sh is present) | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| pwd | |
| df -h . | |
| ls -l ./ubiquitous_bash.sh || { echo "script missing"; exit 1; } | |
| head -n1 ./ubiquitous_bash.sh || true | |
| - name: Create (or confirm) GitHub Release | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| export GH_REPO="${GITHUB_REPOSITORY}" | |
| if ! gh release view "${RELEASE_TAG}" >/dev/null 2>&1; then | |
| gh release create "${RELEASE_TAG}" \ | |
| --title "${RELEASE_TAG}" \ | |
| --notes "created by zBuild-testing-chunks.yml" \ | |
| --latest=false | |
| fi | |
| gh release view "${RELEASE_TAG}" --json name,url | |
| # ################################################################## | |
| - name: Upload parts via ubiquitous_bash | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| bash ./ubiquitous_bash.sh _gh_release_upload_parts-multiple_sequence \ | |
| "${RELEASE_TAG}" \ | |
| ${BASE}.part* | |
| - name: Assert all .part assets are present on the release | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| mapfile -t parts < <(ls -1 ${BASE}.part*) | |
| expected=${#parts[@]} | |
| [[ $expected -gt 0 ]] || { echo "No local parts to assert"; exit 1; } | |
| mapfile -t remote_names < <(gh release view "${RELEASE_TAG}" --json assets --jq '.assets[].name') | |
| missing=0 | |
| for f in "${parts[@]}"; do | |
| bn=$(basename "$f") | |
| if ! printf '%s\n' "${remote_names[@]}" | grep -Fxq "$bn"; then | |
| echo "MISSING: $bn"; ((missing++)) | |
| fi | |
| done | |
| (( missing == 0 )) || { echo "ERROR: $missing assets missing"; exit 1; } | |
| echo "All assets present on release." | |
| # --- FREE SPACE #1: drop local parts now that they are uploaded --- | |
| - name: Free space - remove local .part files | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| rm -f ${BASE}.part* | |
| echo "After removing local parts:"; df -h . | |
| - name: Download & reassemble via ubiquitous_bash (join-stdout) | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| OUT="./_local/rejoined_stream.tar.flx" | |
| bash ./ubiquitous_bash.sh _wget_githubRelease_join-stdout \ | |
| "${GITHUB_REPOSITORY}" \ | |
| "${RELEASE_TAG}" \ | |
| "${BASENAME}" > "${OUT}" | |
| ls -lh "${OUT}" | |
| sha256sum "${OUT}" | tee ./_local/rejoined_stream.sha256 | |
| diff -u <(cut -d' ' -f1 ./_local/original.sha256) \ | |
| <(cut -d' ' -f1 ./_local/rejoined_stream.sha256) | |
| # --- FREE SPACE #2: drop the stream-joined image before the parts path --- | |
| - name: Free space - remove stream-joined image | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| rm -f ./_local/rejoined_stream.tar.flx | |
| echo "After removing stream-joined image:"; df -h . | |
| - name: Download individual .part files (GitHub CLI) | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p ./_local/downloaded_parts | |
| gh release download "${RELEASE_TAG}" \ | |
| --pattern "${BASENAME}.part*" \ | |
| --dir ./_local/downloaded_parts | |
| ls -lh ./_local/downloaded_parts | sed -n '1,200p' | |
| echo "After downloading parts:"; df -h . | |
| - name: Reassemble from downloaded parts (reverse order append) | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| OUT=./_local/rejoined_parts.tar.flx | |
| rm -f "$OUT" | |
| # reverse order because we split with tail+truncate | |
| for p in $(ls ./_local/downloaded_parts/${BASENAME}.part* | sort -r); do | |
| cat "$p" >> "$OUT" | |
| done | |
| ls -lh "$OUT" | |
| sha256sum "$OUT" | tee ./_local/rejoined_parts.sha256 | |
| diff -u <(cut -d' ' -f1 ./_local/original.sha256) \ | |
| <(cut -d' ' -f1 ./_local/rejoined_parts.sha256) | |
| # optional final cleanup to keep workspace tiny | |
| # - name: Free space - remove downloaded parts | |
| # if: always() | |
| # shell: bash | |
| # run: | | |
| # set -euo pipefail | |
| # rm -f ./_local/downloaded_parts/${BASENAME}.part* | |
| # echo "After removing downloaded parts:"; df -h . | |
| # ############################################################################ | |
| # - name: Cross-check both joined images | |
| # shell: bash | |
| # run: | | |
| # set -euo pipefail | |
| # a=$(sha256sum ./_local/rejoined_stream.tar.flx | awk '{print $1}') | |
| # b=$(sha256sum ./_local/rejoined_parts.tar.flx | awk '{print $1}') | |
| # echo "stream=${a}"; echo "parts=${b}" | |
| # [[ "$a" == "$b" ]] | |
| # check the stored digests (do not try to re-hash the files) | |
| - name: Cross-check both joined images | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| a=$(cut -d' ' -f1 ./_local/rejoined_stream.sha256) | |
| b=$(cut -d' ' -f1 ./_local/rejoined_parts.sha256) | |
| echo "stream=${a}" | |
| echo "parts=${b}" | |
| [[ "$a" == "$b" ]] || { echo "ERROR: digests differ"; exit 1; } | |