diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index ffe9723a5..9ba4e45a2 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -130,31 +130,90 @@ jobs: echo "OUTPUT_PATH=${OUTPUT_PATH}" >> $GITHUB_OUTPUT sudo rm -rf ${OCI_DIR} - - name: Rechunk Image - id: rechunk-image - shell: bash + # this tries to make delta updates smaller by pulling + # the last image we published under this tag to the registry + # by minimizing the shuffling of content into different layers + - name: Check Base Image Availability + id: check-baseline env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MATRIX_BASE_NAME: ${{ matrix.base_name }} - MATRIX_STREAM_NAME: ${{ matrix.stream_name }} - MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }} + BASELINE_IMAGE: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }} + run: | + # TODO: check if this pulls stable-daily when we run stable-daily and stable for weeklies + # checks if image is rechunked here: + # https://github.com/coreos/rpm-ostree/blob/653e6ee0ba5953665b2bc2606bd6ab89c3871454/rust/src/compose.rs#L511-L519 + if skopeo inspect docker://"${BASELINE_IMAGE}" | jq -e '.LayersData[1:] | all(.Annotations?["ostree.components"]?)'; then + echo "found ${BASELINE_IMAGE} in registry." + echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT" + echo "PULL_BASELINE_IMAGE=true" >> "$GITHUB_OUTPUT" + else + echo "no baseline image with tag $BASELINE_IMAGE found in registry" + BASELINE_IMAGE="localhost/${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }}" + echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT" + echo "PULL_BASELINE_IMAGE=false" >> "$GITHUB_OUTPUT" + fi + + - name: Pull Baseline Image For Rechunker + id: pull-baseline + if: steps.check-baseline.outputs.PULL_BASELINE_IMAGE == 'true' + env: + BASELINE_IMAGE: ${{ steps.check-baseline.outputs.BASELINE_IMAGE }} run: | - sudo -E $(command -v just) rechunk "${MATRIX_BASE_NAME}" \ - "${MATRIX_STREAM_NAME}" \ - "${MATRIX_IMAGE_FLAVOR}" \ - "1" + sudo podman pull ${BASELINE_IMAGE} + echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT" - - name: Load Image into Podman - id: load-rechunk - shell: bash + - name: debug + run: sudo podman images + + - name: Add debug symbols for rpm-ostree + run: | + cat <<'EOF' > debug.Containerfile + FROM quay.io/fedora/fedora-bootc:latest + RUN dnf install -y dnf5-plugins && \ + dnf5 config-manager setopt updates-debuginfo.enabled=1 && \ + dnf5 config-manager setopt fedora-debuginfo.enabled=1 && \ + dnf install -y rpm-ostree-debuginfo + EOF + + sudo podman build -t localhost/fedora-bootc-debug:latest -f debug.Containerfile . + + # This happens to rename localhost/aurora:stable to :stable-daily + # same behavior as before, we have to redo this logic when we are doing + # stable, testing, next anyway, leave the mess for now + - name: Rechunk Image with rpm-ostree + id: rechunker env: - MATRIX_BASE_NAME: ${{ matrix.base_name }} + IMAGE_NAME: ${{ env.IMAGE_NAME }} + BASELINE_IMAGE: ${{ steps.check-baseline.outputs.BASELINE_IMAGE }} + MATRIX_STREAM_NAME: ${{ matrix.stream_name }} + run: | + sudo podman run --rm \ + --privileged \ + --env=RUST_BACKTRACE=full \ + -v /var/lib/containers:/var/lib/containers \ + "localhost/fedora-bootc-debug:latest" \ + /usr/libexec/bootc-base-imagectl rechunk \ + localhost/${IMAGE_NAME}:${MATRIX_STREAM_NAME} \ + ${BASELINE_IMAGE} + + # we are prodcucing ghcr.io/ublue-os/aurora with the rechunker if we have a baseline image + # the rechunker needs to have the same target image tag for it to work + # ghcr.io/ublue-os/aurora:stable -> localhost/aurora + # later build steps expect localhost/aurora, this is essentially renaming the image again + - name: Rename Rechunker Baseline Image + id: retag-rechunk + if: steps.check-baseline.outputs.PULL_BASELINE_IMAGE == 'true' + env: + BASELINE_IMAGE: ${{ steps.pull-baseline.outputs.BASELINE_IMAGE }} + IMAGE_NAME: ${{ env.IMAGE_NAME }} DEFAULT_TAG: ${{ env.DEFAULT_TAG }} - MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }} run: | - sudo -E $(command -v just) load-rechunk "${MATRIX_BASE_NAME}" \ - "${DEFAULT_TAG}" \ - "${MATRIX_IMAGE_FLAVOR}" + sudo podman tag \ + ${BASELINE_IMAGE} \ + localhost/${IMAGE_NAME}:${DEFAULT_TAG} + sudo podman image rm -i ${BASELINE_IMAGE} + + - name: debug + run: sudo podman images - name: Secureboot Check id: secureboot @@ -216,6 +275,14 @@ jobs: with: string: ${{ env.IMAGE_REGISTRY }} + # TODO: remove me when we have a new podman in 26.04 runners + # needed because old podman doesn't push layer annotations for + # the rpm-ostree rechunker at all + - name: install podman from brew + if: github.event_name != 'pull_request' + run: | + /home/linuxbrew/.linuxbrew/bin/brew install podman + - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' run: | @@ -236,9 +303,15 @@ jobs: attempt_delay: 15000 command: | set -euox pipefail + # HACK: push a second time so layer annotations are pushed + # TODO: remove me when https://github.com/containers/podman/issues/27796 fixed for tag in ${ALIAS_TAGS}; do - sudo -E podman push ${IMAGE_NAME}:${tag} ${LOWERCASE}/${IMAGE_NAME}:${tag} + sudo -E /home/linuxbrew/.linuxbrew/bin/podman push ${IMAGE_NAME}:${tag} ${LOWERCASE}/${IMAGE_NAME}:${tag} + done + + for tag in ${ALIAS_TAGS}; do + sudo -E /home/linuxbrew/.linuxbrew/bin/podman push ${IMAGE_NAME}:${tag} ${LOWERCASE}/${IMAGE_NAME}:${tag} done digest=$(skopeo inspect docker://${LOWERCASE}/${IMAGE_NAME}:${DEFAULT_TAG} --format '{{.Digest}}') diff --git a/Justfile b/Justfile index da3553040..221e6d4f4 100644 --- a/Justfile +++ b/Justfile @@ -238,7 +238,7 @@ build $image="aurora" $tag="latest" $flavor="main" rechunk="0" ghcr="0" pipeline elif [[ "{{ rechunk }}" == "1" && "{{ ghcr }}" == "1" ]]; then ${SUDOIF} {{ just }} rechunk "${image}" "${tag}" "${flavor}" 1 elif [[ "{{ rechunk }}" == "1" ]]; then - ${SUDOIF} {{ just }} rechunk "${image}" "${tag}" "${flavor}" + {{ just }} rechunk "${image}" "${tag}" "${flavor}" fi # Build Image and Rechunk @@ -283,133 +283,27 @@ rechunk $image="aurora" $tag="latest" $flavor="main" ghcr="0" pipeline="0": {{ just }} build "${image}" "${tag}" "${flavor}" fi - # Load into Rootful Podman - ID=$(${SUDOIF} ${PODMAN} images --filter reference=localhost/"${image_name}":"${tag}" --format "'{{ '{{.ID}}' }}'") - if [[ -z "$ID" && ! ${PODMAN} =~ docker ]]; then - COPYTMP=$(mktemp -p "${PWD}" -d -t podman_scp.XXXXXXXXXX) - ${SUDOIF} TMPDIR=${COPYTMP} ${PODMAN} image scp ${UID}@localhost::localhost/"${image_name}":"${tag}" root@localhost::localhost/"${image_name}":"${tag}" - rm -rf "${COPYTMP}" + # Delete the rechunked image if present, rpm-ostree shits itself for whatever reason + # workaround for https://github.com/coreos/rpm-ostree/issues/5545 + if ${SUDOIF} ${PODMAN} image exists "localhost/${image_name}:${tag}-chunked"; then + ${SUDOIF} ${PODMAN} image rm -f "localhost/${image_name}:${tag}-chunked" fi - # Prep Container - CREF=$(${SUDOIF} ${PODMAN} create localhost/"${image_name}":"${tag}" bash) - OLD_IMAGE=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Image') - OUT_NAME="${image_name}_build" - MOUNT=$(${SUDOIF} ${PODMAN} mount "${CREF}") - - # Fedora Version - fedora_version=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["ostree.linux"]' | grep -oP 'fc\K[0-9]+') - - # Label Version - VERSION=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["org.opencontainers.image.version"]') - - # Git SHA - SHA="dedbeef" - if [[ -z "$(git status -s)" ]]; then - SHA=$(git rev-parse HEAD) - fi - - # Rest of Labels - LABELS=" - io.artifacthub.package.deprecated=false - io.artifacthub.package.keywords=bootc,fedora,aurora,ublue,universal-blue - io.artifacthub.package.logo-url=https://avatars.githubusercontent.com/u/120078124?s=200&v=4 - io.artifacthub.package.maintainers=[{\"name\": \"NiHaiden\", \"email\": \"me@nhaiden.io\"}] - io.artifacthub.package.readme-url=https://raw.githubusercontent.com/ublue-os/aurora/refs/heads/main/README.md - org.opencontainers.image.created=$(date -u +%Y\-%m\-%d\T%H\:%M\:%S\Z) - org.opencontainers.image.license=Apache-2.0 - org.opencontainers.image.source=https://raw.githubusercontent.com/ublue-os/aurora/refs/heads/main/Containerfile - org.opencontainers.image.title=${image_name} - org.opencontainers.image.url=https://getaurora.dev - org.opencontainers.image.vendor={{ repo_organization }} - ostree.linux=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["ostree.linux"]') - containers.bootc=1 - " - - # Cleanup Space during Github Action - if [[ "{{ ghcr }}" == "1" ]]; then - base_image_name=kinoite - if [[ "${tag}" =~ stable ]]; then - tag="stable-daily" - fi - ID=$(${SUDOIF} ${PODMAN} images --filter reference=ghcr.io/{{ repo_organization }}/"${base_image_name}":${fedora_version} --format "{{ '{{.ID}}' }}") - if [[ -n "$ID" ]]; then - ${PODMAN} rmi "$ID" - fi + # Load into Rootful Podman + ID=$(${PODMAN} inspect --format={{ '{{.Digest}}' }} localhost/"${image_name}":"${tag}") + ID_ROOT=$(sudo ${PODMAN} inspect --format={{ '{{.Digest}}' }} localhost/"${image_name}":"${tag}") + if [[ ! "${PODMAN}" =~ "docker" ]] && [[ -n "$ID" ]] && [[ "$ID" != "$ID_ROOT" ]]; then + ${PODMAN} image scp $(whoami)@localhost::localhost/"${image_name}":"${tag}" fi - # Rechunk Container - rechunker="{{ rechunker_image }}" - - echo "::endgroup::" - echo "::group:: Prune" - - # Run Rechunker's Prune ${SUDOIF} ${PODMAN} run --rm \ --pull=${PULL_POLICY} \ - --security-opt label=disable \ - --volume "$MOUNT":/var/tree \ - --env TREE=/var/tree \ - --user 0:0 \ - "${rechunker}" \ - /sources/rechunk/1_prune.sh - - echo "::endgroup::" - echo "::group:: Create ostree tree" - - # Run Rechunker's Create - ${SUDOIF} ${PODMAN} run --rm \ - --security-opt label=disable \ - --volume "$MOUNT":/var/tree \ - --volume "cache_ostree:/var/ostree" \ - --env TREE=/var/tree \ - --env REPO=/var/ostree/repo \ - --env RESET_TIMESTAMP=1 \ - --user 0:0 \ - "${rechunker}" \ - /sources/rechunk/2_create.sh - - # Cleanup Temp Container Reference - ${SUDOIF} ${PODMAN} unmount "$CREF" - ${SUDOIF} ${PODMAN} rm "$CREF" - ${SUDOIF} ${PODMAN} rmi "$OLD_IMAGE" - - echo "::endgroup::" - echo "::group:: Rechunker" - - # Run Rechunker - ${SUDOIF} ${PODMAN} run --rm \ - --pull=${PULL_POLICY} \ - --security-opt label=disable \ - --volume "$PWD:/workspace" \ - --volume "$PWD:/var/git" \ - --volume cache_ostree:/var/ostree \ - --env REPO=/var/ostree/repo \ - --env PREV_REF=ghcr.io/ublue-os/"${image_name}":"${tag}" \ - --env OUT_NAME="$OUT_NAME" \ - --env LABELS="${LABELS}" \ - --env "DESCRIPTION='The ultimate productivity workstation'" \ - --env "VERSION=${VERSION}" \ - --env VERSION_FN=/workspace/version.txt \ - --env OUT_REF="oci:$OUT_NAME" \ - --env GIT_DIR="/var/git" \ - --env REVISION="$SHA" \ - --user 0:0 \ - "${rechunker}" \ - /sources/rechunk/3_chunk.sh - - # Fix Permissions of OCI - ${SUDOIF} find ${OUT_NAME} -type d -exec chmod 0755 {} \; || true - ${SUDOIF} find ${OUT_NAME}* -type f -exec chmod 0644 {} \; || true - - if [[ "${UID}" -gt "0" ]]; then - ${SUDOIF} chown "${UID}:${GROUPS}" -R "${PWD}" - elif [[ -n "${SUDO_UID:-}" ]]; then - chown "${SUDO_UID}":"${SUDO_GID}" -R "${PWD}" - fi - - # Remove cache_ostree - ${SUDOIF} ${PODMAN} volume rm cache_ostree + --privileged \ + -v "/var/lib/containers:/var/lib/containers" \ + "quay.io/fedora/fedora-bootc:latest" \ + /usr/libexec/bootc-base-imagectl rechunk \ + "localhost/${image_name}":"${tag}" \ + "localhost/${image_name}":"${tag}-chunked" echo "::endgroup::" @@ -419,27 +313,6 @@ rechunk $image="aurora" $tag="latest" $flavor="main" ghcr="0" pipeline="0": sudo -u "${SUDO_USER}" {{ just }} secureboot "${image}" "${tag}" "${flavor}" fi -# Load OCI into Podman Store -[group('Image')] -load-rechunk image="aurora" tag="latest" flavor="main": - #!/usr/bin/bash - set -eou pipefail - - # Validate - {{ just }} validate {{ image }} {{ tag }} {{ flavor }} - - # Image Name - image_name=$({{ just }} image_name {{ image }} {{ tag }} {{ flavor }}) - - # Load Image - OUT_NAME="${image_name}_build" - IMAGE=$(${PODMAN} pull oci:"${PWD}"/"${OUT_NAME}") - ${PODMAN} tag ${IMAGE} localhost/"${image_name}":{{ tag }} - - # Cleanup - rm -rf "${OUT_NAME}*" - rm -f previous.manifest.json - # Run Container [group('Image')] run $image="aurora" $tag="latest" $flavor="main": @@ -680,8 +553,6 @@ tag-images image_name="" default_tag="" tags="": ${PODMAN} tag $IMAGE {{ image_name }}:${tag} done - - # Show Images ${PODMAN} images diff --git a/build_files/base/20-layer-definitions.sh b/build_files/base/20-layer-definitions.sh new file mode 100755 index 000000000..abfd9d53b --- /dev/null +++ b/build_files/base/20-layer-definitions.sh @@ -0,0 +1,95 @@ +#!/usr/bin/bash + +# to reduce the indiviual layer size we put big files that are not shipped with RPMs in their own layers +# See: https://coreos.github.io/rpm-ostree/build-chunked-oci/ + +set -eoux pipefail + +# Enable all this when fixed: https://github.com/coreos/rpm-ostree/issues/5545 +# For now just hardcode the big files manually, keep the non-rpm layer amount static +# use systemd-escape -p +#setfattr -n user.component -v usr-share-doc-aurora-aurora.pdf /usr/share/doc/aurora/aurora.pdf +#setfattr -n user.component -v usr-share-homebrew.tar.zst /usr/share/homebrew.tar.zst +#setfattr -n user.component -v usr-bin-starhsip /usr/bin/starship +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d3-contents-images-3840x2160.png /usr/share/backgrounds/aurora/aurora-wallpaper-3/contents/images/3840x2160.png +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d4-contents-images-3840x2160.png /usr/share/backgrounds/aurora/aurora-wallpaper-4/contents/images/3840x2160.png +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d2-contents-images-3840x2160.png /usr/share/backgrounds/aurora/aurora-wallpaper-2/contents/images/3840x2160.png +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d7-contents-images-3840x2160.jxl /usr/share/backgrounds/aurora/aurora-wallpaper-7/contents/images/3840x2160.jxl +#setfattr -n user.component -v usr-lib-firmware-intel\x2ducode-06\x2dad\x2d01 /usr/lib/firmware/intel-ucode/06-ad-01 +#setfattr -n user.component -v usr-share-fonts-nerd\x2dfonts-NerdFontsSymbolsOnly-SymbolsNerdFontMono\x2dRegular.ttf /usr/share/fonts/nerd-fonts/NerdFontsSymbolsOnly/SymbolsNerdFontMono-Regular.ttf +#setfattr -n user.component -v usr-share-fonts-nerd\x2dfonts-NerdFontsSymbolsOnly-SymbolsNerdFont\x2dRegular.ttf /usr/share/fonts/nerd-fonts/NerdFontsSymbolsOnly/SymbolsNerdFont-Regular.ttf +#setfattr -n user.component -v usr-share-backgrounds-aurora-greg\x2drakozy\x2daurora-contents-images-5616x3744.jxl /usr/share/backgrounds/aurora/greg-rakozy-aurora/contents/images/5616x3744.jxl +#setfattr -n user.component -v usr-lib-firmware-asihpi-dsp8900.bin /usr/lib/firmware/asihpi/dsp8900.bin +#setfattr -n user.component -v usr-lib-firmware-intel\x2ducode-06\x2daf\x2d03 /usr/lib/firmware/intel-ucode/06-af-03 +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d8-contents-images-3840x2160.jxl /usr/share/backgrounds/aurora/aurora-wallpaper-8/contents/images/3840x2160.jxl +#setfattr -n user.component -v usr-share-backgrounds-aurora-jonatan\x2dpie\x2daurora-contents-images-3944x2770.jxl /usr/share/backgrounds/aurora/jonatan-pie-aurora/contents/images/3944x2770.jxl +#setfattr -n user.component -v usr-lib-firmware-intel\x2ducode-06\x2d8f\x2d08 /usr/lib/firmware/intel-ucode/06-8f-08 +#setfattr -n user.component -v usr-share-sddm-themes-01\x2dbreeze\x2daurora-preview.png /usr/share/sddm/themes/01-breeze-aurora/preview.png +#setfattr -n user.component -v usr-share-color-icc-colord-framework16.icc /usr/share/color/icc/colord/framework16.icc +#setfattr -n user.component -v usr-share-color-icc-colord-framework13.icc /usr/share/color/icc/colord/framework13.icc +#setfattr -n user.component -v usr-share-plasma-avatars-lumina.png /usr/share/plasma/avatars/lumina.png +#setfattr -n user.component -v usr-share-backgrounds-aurora-aurora\x2dwallpaper\x2d6-contents-images-3840x2160.jxl /usr/share/backgrounds/aurora/aurora-wallpaper-6/contents/images/3840x2160.jxl +#setfattr -n user.component -v usr-lib-firmware-mixart-miXart8.elf /usr/lib/firmware/mixart/miXart8.elf +#setfattr -n user.component -v usr-share-plasma-avatars-vincent.png /usr/share/plasma/avatars/vincent.png +#setfattr -n user.component -v usr-lib-firmware-asihpi-dsp6200.bin /usr/lib/firmware/asihpi/dsp6200.bin +#setfattr -n user.component -v usr-share-backgrounds-aurora-xe_space_needle-contents-images-6000x4000.jxl /usr/share/backgrounds/aurora/xe_space_needle/contents/images/6000x4000.jxl +#setfattr -n user.component -v usr-share-plasma-avatars-echo.png /usr/share/plasma/avatars/echo.png +#setfattr -n user.component -v usr-share-plasma-avatars-phlip.png /usr/share/plasma/avatars/phlip.png +#setfattr -n user.component -v usr-share-plasma-avatars-scope.png /usr/share/plasma/avatars/scope.png +#setfattr -n user.component -v usr-share-plasma-avatars-tina.png /usr/share/plasma/avatars/tina.png +#setfattr -n user.component -v usr-lib-firmware-ctefx\x2ddesktop.bin /usr/lib/firmware/ctefx-desktop.bin +#setfattr -n user.component -v usr-lib-firmware-ctefx\x2dr3di.bin /usr/lib/firmware/ctefx-r3di.bin + +#COUNT=30 +#OUTPUT="/tmp/nonrpm.json" +## 600KiB +#MIN_SIZE=614400 +#IGNORE=( +# "^/usr/lib/sysimage/rpm-ostree-base-db/rpmdb.sqlite" +# "^/usr/share/rpm/rpmdb.sqlite" +# "^/usr/lib/modules" # spam +# "^/usr/bin/tailscaled" # packaging is bumd, hardlinks +# "^/usr/bin/zpool" +# "^/usr/lib/fontconfig/cache" +# "^/usr/lib64/.*\.so" # mostly useless files +# "^/usr/lib32/.*\.so" # can't be assed to make this one thing +# "^/usr/lib/.*\.so" # and do it properly +# "^/usr/lib/bootupd/" # not worth +#) +# +#IGNORE_REGEX=$(printf "|%s" "${IGNORE[@]}") +#IGNORE_REGEX="${IGNORE_REGEX:1}" +# +## which files are not tracked by the rpmdb +#RPM_FILES=$(mktemp) +#rpm -qa --qf '[%{FILENAMES}\n]' | grep -E "^/usr|^/opt" | sort > "$RPM_FILES" +# +#NON_RPM_FILES=$(comm -23 \ +# <(find /usr -type f 2>/dev/null | grep -Ev "$IGNORE_REGEX" | sort) \ +# "$RPM_FILES" +#) +# +#echo "$NON_RPM_FILES" | xargs -d '\n' du -b 2>/dev/null | jq -Rn --argjson min "$MIN_SIZE" ' +# [ +# inputs | split("\t") | { +# "path": .[1], +# "size": (.[0] | tonumber), +# "component": (.[1] | ltrimstr("/") | gsub("/"; "-")) +# } | select(.size >= $min) +# ] +#' > "$OUTPUT" +# +#echo "found $(jq 'length' "$OUTPUT") non-rpm tracked files" +# +## generated example command +## setfattr -n user.component -v usr-share-doc-aurora-aurora.pdf /usr/share/doc/aurora/aurora.pdf +#jq -c '.[]' "$OUTPUT" | while read -r item; do +# path=$(echo "$item" | jq -r '.path') +# comp=$(echo "$item" | jq -r '.component') +# setfattr -n user.component -v "$comp" "$path" +#done +# +#echo "listing $COUNT biggest ones" +#jq -r --arg n "$COUNT" ' sort_by(.size) | reverse | .[0:($n | tonumber)] | .[] | "\(.size)\t\(.path)" ' "$OUTPUT" | numfmt --to=iec --field=1 + +echo "::endgroup::" diff --git a/build_files/base/20-tests.sh b/build_files/base/21-tests.sh similarity index 100% rename from build_files/base/20-tests.sh rename to build_files/base/21-tests.sh diff --git a/build_files/shared/build.sh b/build_files/shared/build.sh index 06549b324..542f5d931 100755 --- a/build_files/shared/build.sh +++ b/build_files/shared/build.sh @@ -56,8 +56,11 @@ fi # Validate all repos are disabled before committing /ctx/build_files/shared/validate-repos.sh +# Set filesystem properties for rechunker +/ctx/build_files/base/20-layer-definitions.sh + # Clean Up /ctx/build_files/shared/clean-stage.sh # Simple Tests -/ctx/build_files/base/20-tests.sh +/ctx/build_files/base/21-tests.sh diff --git a/system_files/shared/usr/bin/rechunker-group-fix b/system_files/shared/usr/bin/rechunker-group-fix new file mode 100755 index 000000000..ce625e05f --- /dev/null +++ b/system_files/shared/usr/bin/rechunker-group-fix @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# See /usr/lib/systemd/system/rechunker-group-fix.service for more details +# https://github.com/zirconium-dev/zirconium/commit/85cc566697b3337d37e2b6c6305c51b7f1a9e9b1 +# thanks @tullilirockz + +IGNORE_MANUAL="wheel|root|sudo|nobody" +for manually_created_user in $(grep -E -e ".*:[1-3][[:digit:]]{3}:.*" "$1") ; do + # `grep` matched on a group with GID [1-3]000 or so + if [ ! -z "$(cut -f4 -d: <<< "${manually_created_user}")" ] ; then + continue + fi + IGNORE_MANUAL="$(cut -f1,3 -d: --output-delimiter="|" <<< "${manually_created_user}")|${IGNORE_MANUAL:-}" + for related_group in $(grep "$(cut -f1 -d: <<< ${manually_created_user})" "$1"); do + # Deduplicates matches for the same group/user + if [ "$(cut -f1 <<< "${related_group}")" == "$(cut -f1 <<< "${manually_created_user}")" ] ; then + continue + fi + IGNORE_MANUAL="$(cut -f1 -d: <<< "${related_group}")|${IGNORE_MANUAL:-}" + done +done + +grep --no-filename -e "^g" /usr/lib/sysusers.d/*.conf | grep -v -E -e "${IGNORE_MANUAL}" | tr -s " " | cut -d" " -f2 | uniq | xargs -I{} sed -i "/{}/d" "${1}" diff --git a/system_files/shared/usr/lib/systemd/system/rechunker-group-fix.service b/system_files/shared/usr/lib/systemd/system/rechunker-group-fix.service new file mode 100644 index 000000000..63a0cb86d --- /dev/null +++ b/system_files/shared/usr/lib/systemd/system/rechunker-group-fix.service @@ -0,0 +1,29 @@ +# We have this script so that people using images with `nss-altfiles` (`/usr/lib/g{roup,shadow}`) +# do not break their systems when rebasing to an image without that +# This usually happens when using https://github.com/hhd-dev/rechunk then rebasing to an image without it. +# Please DO NOT remove this unless this is fully, completely obsolete. +# This is exactly what is making it break: https://github.com/ublue-os/legacy-rechunk/blob/1d2b0c2e99afbdc2eb06788ae28e157a88b03d70/1_prune.sh#L41-L47 +# Users WILL experience black screens and systems will NOT boot if this script malfunctions. Please test this properly and always make sure this works +# Relevant issues: +# - https://github.com/bootc-dev/bootc/issues/1179#issuecomment-2708305926 +# - https://github.com/ublue-os/main/issues/759 +# - https://github.com/ublue-os/bluefin-lts/issues/918 +# - https://github.com/ublue-os/image-template/issues/177 +# - https://github.com/ublue-os/aurora/issues/1468 +# - https://github.com/ublue-os/bluefin/issues/3852 +# This got created on Tue, 16 Dec 2025 00:44:58 -0300 +[Unit] +Description=Fix groups for Legacy rechunker +Wants=local-fs.target +After=local-fs.target +# Before=systemd-sysusers.service + +[Service] +Type=oneshot +ExecStart=rechunker-group-fix /etc/group +ExecStart=rechunker-group-fix /etc/gshadow +ExecStart=systemd-sysusers +ExecStart=systemd-tmpfiles --create --remove --boot --exclude-prefix=/dev + +[Install] +WantedBy=default.target multi-user.target