chore: update from rechunk branch

.github/workflows/reusable-build.yml

@@ -130,31 +130,90 @@ jobs:
           echo "OUTPUT_PATH=${OUTPUT_PATH}" >> $GITHUB_OUTPUT
           sudo rm -rf ${OCI_DIR}
 
-      - name: Rechunk Image
-        id: rechunk-image
-        shell: bash
+      # this tries to make delta updates smaller by pulling
+      # the last image we published under this tag to the registry
+      # by minimizing the shuffling of content into different layers
+      - name: Check Base Image Availability
+        id: check-baseline
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          MATRIX_BASE_NAME: ${{ matrix.base_name }}
-          MATRIX_STREAM_NAME: ${{ matrix.stream_name }}
-          MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }}
+          BASELINE_IMAGE: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }}
+        run: |
+          # TODO: check if this pulls stable-daily when we run stable-daily and stable for weeklies
+          # checks if image is rechunked here:
+          # https://github.com/coreos/rpm-ostree/blob/653e6ee0ba5953665b2bc2606bd6ab89c3871454/rust/src/compose.rs#L511-L519
+          if skopeo inspect docker://"${BASELINE_IMAGE}" | jq -e '.LayersData[1:] | all(.Annotations?["ostree.components"]?)'; then
+            echo "found ${BASELINE_IMAGE} in registry."
+            echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT"
+            echo "PULL_BASELINE_IMAGE=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "no baseline image with tag $BASELINE_IMAGE found in registry"
+            BASELINE_IMAGE="localhost/${{ env.IMAGE_NAME }}:${{ env.DEFAULT_TAG }}"
+            echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT"
+            echo "PULL_BASELINE_IMAGE=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Pull Baseline Image For Rechunker
+        id: pull-baseline
+        if: steps.check-baseline.outputs.PULL_BASELINE_IMAGE == 'true'
+        env:
+          BASELINE_IMAGE: ${{ steps.check-baseline.outputs.BASELINE_IMAGE }}
         run: |
-          sudo -E $(command -v just) rechunk "${MATRIX_BASE_NAME}" \
-                            "${MATRIX_STREAM_NAME}" \
-                            "${MATRIX_IMAGE_FLAVOR}" \
-                            "1"
+          sudo podman pull ${BASELINE_IMAGE}
+          echo "BASELINE_IMAGE=${BASELINE_IMAGE}" >> "$GITHUB_OUTPUT"
 
-      - name: Load Image into Podman
-        id: load-rechunk
-        shell: bash
+      - name: debug
+        run: sudo podman images
+
+      - name: Add debug symbols for rpm-ostree
+        run: |
+          cat <<'EOF' > debug.Containerfile
+          FROM quay.io/fedora/fedora-bootc:latest
+          RUN dnf install -y dnf5-plugins && \
+            dnf5 config-manager setopt updates-debuginfo.enabled=1 && \
+            dnf5 config-manager setopt fedora-debuginfo.enabled=1 && \
+            dnf install -y rpm-ostree-debuginfo
+          EOF
+
+          sudo podman build -t localhost/fedora-bootc-debug:latest -f debug.Containerfile .
+
+      # This happens to rename localhost/aurora:stable to :stable-daily
+      # same behavior as before, we have to redo this logic when we are doing
+      # stable, testing, next anyway, leave the mess for now
+      - name: Rechunk Image with rpm-ostree
+        id: rechunker
         env:
-          MATRIX_BASE_NAME: ${{ matrix.base_name }}
+          IMAGE_NAME: ${{ env.IMAGE_NAME }}
+          BASELINE_IMAGE: ${{ steps.check-baseline.outputs.BASELINE_IMAGE }}
+          MATRIX_STREAM_NAME: ${{ matrix.stream_name }}
+        run: |
+          sudo podman run --rm \
+            --privileged \
+            --env=RUST_BACKTRACE=full \
+            -v /var/lib/containers:/var/lib/containers \
+            "localhost/fedora-bootc-debug:latest" \
+            /usr/libexec/bootc-base-imagectl rechunk \
+                localhost/${IMAGE_NAME}:${MATRIX_STREAM_NAME} \
+                ${BASELINE_IMAGE}
+
+      # we are prodcucing ghcr.io/ublue-os/aurora with the rechunker if we have a baseline image
+      # the rechunker needs to have the same target image tag for it to work
+      # ghcr.io/ublue-os/aurora:stable -> localhost/aurora
+      # later build steps expect localhost/aurora, this is essentially renaming the image again
+      - name: Rename Rechunker Baseline Image
+        id: retag-rechunk
+        if: steps.check-baseline.outputs.PULL_BASELINE_IMAGE == 'true'
+        env:
+          BASELINE_IMAGE: ${{ steps.pull-baseline.outputs.BASELINE_IMAGE }}
+          IMAGE_NAME: ${{ env.IMAGE_NAME }}
           DEFAULT_TAG: ${{ env.DEFAULT_TAG }}
-          MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }}
         run: |
-          sudo -E $(command -v just) load-rechunk "${MATRIX_BASE_NAME}" \
-                            "${DEFAULT_TAG}" \
-                            "${MATRIX_IMAGE_FLAVOR}"
+          sudo podman tag \
+            ${BASELINE_IMAGE} \
+            localhost/${IMAGE_NAME}:${DEFAULT_TAG}
+          sudo podman image rm -i ${BASELINE_IMAGE}
+
+      - name: debug
+        run: sudo podman images
 
       - name: Secureboot Check
         id: secureboot
@@ -216,6 +275,14 @@ jobs:
         with:
           string: ${{ env.IMAGE_REGISTRY }}
 
+      # TODO: remove me when we have a new podman in 26.04 runners
+      # needed because old podman doesn't push layer annotations for
+      # the rpm-ostree rechunker at all
+      - name: install podman from brew
+        if: github.event_name != 'pull_request'
+        run: |
+          /home/linuxbrew/.linuxbrew/bin/brew install podman
+
       - name: Login to GitHub Container Registry
         if: github.event_name != 'pull_request'
         run: |
@@ -236,9 +303,15 @@ jobs:
           attempt_delay: 15000
           command: |
             set -euox pipefail
+            # HACK: push a second time so layer annotations are pushed
+            # TODO: remove me when https://github.com/containers/podman/issues/27796 fixed
 
             for tag in ${ALIAS_TAGS}; do
-              sudo -E podman push ${IMAGE_NAME}:${tag} ${LOWERCASE}/${IMAGE_NAME}:${tag}
+              sudo -E /home/linuxbrew/.linuxbrew/bin/podman push ${IMAGE_NAME}:${tag} ${LOWERCASE}/${IMAGE_NAME}:${tag}
+            done
+
+            for tag in ${ALIAS_TAGS}; do
+              sudo -E /home/linuxbrew/.linuxbrew/bin/podman push ${IMAGE_NAME}:${tag} ${LOWERCASE}/${IMAGE_NAME}:${tag}
             done
 
             digest=$(skopeo inspect docker://${LOWERCASE}/${IMAGE_NAME}:${DEFAULT_TAG} --format '{{.Digest}}')

Justfile

@@ -238,7 +238,7 @@ build $image="aurora" $tag="latest" $flavor="main" rechunk="0" ghcr="0" pipeline
     elif [[ "{{ rechunk }}" == "1" && "{{ ghcr }}" == "1" ]]; then
         ${SUDOIF} {{ just }} rechunk "${image}" "${tag}" "${flavor}" 1
     elif [[ "{{ rechunk }}" == "1" ]]; then
-        ${SUDOIF} {{ just }} rechunk "${image}" "${tag}" "${flavor}"
+        {{ just }} rechunk "${image}" "${tag}" "${flavor}"
     fi
 
 # Build Image and Rechunk
@@ -283,133 +283,27 @@ rechunk $image="aurora" $tag="latest" $flavor="main" ghcr="0" pipeline="0":
         {{ just }} build "${image}" "${tag}" "${flavor}"
     fi
 
-    # Load into Rootful Podman
-    ID=$(${SUDOIF} ${PODMAN} images --filter reference=localhost/"${image_name}":"${tag}" --format "'{{ '{{.ID}}' }}'")
-    if [[ -z "$ID" && ! ${PODMAN} =~ docker ]]; then
-        COPYTMP=$(mktemp -p "${PWD}" -d -t podman_scp.XXXXXXXXXX)
-        ${SUDOIF} TMPDIR=${COPYTMP} ${PODMAN} image scp ${UID}@localhost::localhost/"${image_name}":"${tag}" root@localhost::localhost/"${image_name}":"${tag}"
-        rm -rf "${COPYTMP}"
+    # Delete the rechunked image if present, rpm-ostree shits itself for whatever reason
+    # workaround for https://github.com/coreos/rpm-ostree/issues/5545
+    if ${SUDOIF} ${PODMAN} image exists "localhost/${image_name}:${tag}-chunked"; then
+      ${SUDOIF} ${PODMAN} image rm -f "localhost/${image_name}:${tag}-chunked"
     fi
 
-    # Prep Container
-    CREF=$(${SUDOIF} ${PODMAN} create localhost/"${image_name}":"${tag}" bash)
-    OLD_IMAGE=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Image')
-    OUT_NAME="${image_name}_build"
-    MOUNT=$(${SUDOIF} ${PODMAN} mount "${CREF}")
-
-    # Fedora Version
-    fedora_version=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["ostree.linux"]' | grep -oP 'fc\K[0-9]+')
-
-    # Label Version
-    VERSION=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["org.opencontainers.image.version"]')
-
-    # Git SHA
-    SHA="dedbeef"
-    if [[ -z "$(git status -s)" ]]; then
-        SHA=$(git rev-parse HEAD)
-    fi
-
-    # Rest of Labels
-    LABELS="
-        io.artifacthub.package.deprecated=false
-        io.artifacthub.package.keywords=bootc,fedora,aurora,ublue,universal-blue
-        io.artifacthub.package.logo-url=https://avatars.githubusercontent.com/u/120078124?s=200&v=4
-        io.artifacthub.package.maintainers=[{\"name\": \"NiHaiden\", \"email\": \"[email protected]\"}]
-        io.artifacthub.package.readme-url=https://raw.githubusercontent.com/ublue-os/aurora/refs/heads/main/README.md
-        org.opencontainers.image.created=$(date -u +%Y\-%m\-%d\T%H\:%M\:%S\Z)
-        org.opencontainers.image.license=Apache-2.0
-        org.opencontainers.image.source=https://raw.githubusercontent.com/ublue-os/aurora/refs/heads/main/Containerfile
-        org.opencontainers.image.title=${image_name}
-        org.opencontainers.image.url=https://getaurora.dev
-        org.opencontainers.image.vendor={{ repo_organization }}
-        ostree.linux=$(${SUDOIF} ${PODMAN} inspect $CREF | jq -r '.[].Config.Labels["ostree.linux"]')
-        containers.bootc=1
-    "
-
-    # Cleanup Space during Github Action
-    if [[ "{{ ghcr }}" == "1" ]]; then
-        base_image_name=kinoite
-        if [[ "${tag}" =~ stable ]]; then
-            tag="stable-daily"
-        fi
-        ID=$(${SUDOIF} ${PODMAN} images --filter reference=ghcr.io/{{ repo_organization }}/"${base_image_name}":${fedora_version} --format "{{ '{{.ID}}' }}")
-        if [[ -n "$ID" ]]; then
-            ${PODMAN} rmi "$ID"
-        fi
+    # Load into Rootful Podman
+    ID=$(${PODMAN} inspect --format={{ '{{.Digest}}' }} localhost/"${image_name}":"${tag}")
+    ID_ROOT=$(sudo ${PODMAN} inspect --format={{ '{{.Digest}}' }} localhost/"${image_name}":"${tag}")
+    if [[ ! "${PODMAN}" =~ "docker" ]] && [[ -n "$ID" ]] && [[ "$ID" != "$ID_ROOT" ]]; then
+        ${PODMAN} image scp $(whoami)@localhost::localhost/"${image_name}":"${tag}"
     fi
 
-    # Rechunk Container
-    rechunker="{{ rechunker_image }}"
-
-    echo "::endgroup::"
-    echo "::group:: Prune"
-
-    # Run Rechunker's Prune
     ${SUDOIF} ${PODMAN} run --rm \
         --pull=${PULL_POLICY} \
-        --security-opt label=disable \
-        --volume "$MOUNT":/var/tree \
-        --env TREE=/var/tree \
-        --user 0:0 \
-        "${rechunker}" \
-        /sources/rechunk/1_prune.sh
-
-    echo "::endgroup::"
-    echo "::group:: Create ostree tree"
-
-    # Run Rechunker's Create
-    ${SUDOIF} ${PODMAN} run --rm \
-        --security-opt label=disable \
-        --volume "$MOUNT":/var/tree \
-        --volume "cache_ostree:/var/ostree" \
-        --env TREE=/var/tree \
-        --env REPO=/var/ostree/repo \
-        --env RESET_TIMESTAMP=1 \
-        --user 0:0 \
-        "${rechunker}" \
-        /sources/rechunk/2_create.sh
-
-    # Cleanup Temp Container Reference
-    ${SUDOIF} ${PODMAN} unmount "$CREF"
-    ${SUDOIF} ${PODMAN} rm "$CREF"
-    ${SUDOIF} ${PODMAN} rmi "$OLD_IMAGE"
-
-    echo "::endgroup::"
-    echo "::group:: Rechunker"
-
-    # Run Rechunker
-    ${SUDOIF} ${PODMAN} run --rm \
-        --pull=${PULL_POLICY} \
-        --security-opt label=disable \
-        --volume "$PWD:/workspace" \
-        --volume "$PWD:/var/git" \
-        --volume cache_ostree:/var/ostree \
-        --env REPO=/var/ostree/repo \
-        --env PREV_REF=ghcr.io/ublue-os/"${image_name}":"${tag}" \
-        --env OUT_NAME="$OUT_NAME" \
-        --env LABELS="${LABELS}" \
-        --env "DESCRIPTION='The ultimate productivity workstation'" \
-        --env "VERSION=${VERSION}" \
-        --env VERSION_FN=/workspace/version.txt \
-        --env OUT_REF="oci:$OUT_NAME" \
-        --env GIT_DIR="/var/git" \
-        --env REVISION="$SHA" \
-        --user 0:0 \
-        "${rechunker}" \
-        /sources/rechunk/3_chunk.sh
-
-    # Fix Permissions of OCI
-    ${SUDOIF} find ${OUT_NAME} -type d -exec chmod 0755 {} \; || true
-    ${SUDOIF} find ${OUT_NAME}* -type f -exec chmod 0644 {} \; || true
-
-    if [[ "${UID}" -gt "0" ]]; then
-        ${SUDOIF} chown "${UID}:${GROUPS}" -R "${PWD}"
-    elif [[ -n "${SUDO_UID:-}" ]]; then
-        chown "${SUDO_UID}":"${SUDO_GID}" -R "${PWD}"
-    fi
-
-    # Remove cache_ostree
-    ${SUDOIF} ${PODMAN} volume rm cache_ostree
+        --privileged \
+        -v "/var/lib/containers:/var/lib/containers" \
+        "quay.io/fedora/fedora-bootc:latest" \
+        /usr/libexec/bootc-base-imagectl rechunk \
+        "localhost/${image_name}":"${tag}" \
+        "localhost/${image_name}":"${tag}-chunked"
 
     echo "::endgroup::"
 
@@ -419,27 +313,6 @@ rechunk $image="aurora" $tag="latest" $flavor="main" ghcr="0" pipeline="0":
         sudo -u "${SUDO_USER}" {{ just }} secureboot "${image}" "${tag}" "${flavor}"
     fi
 
-# Load OCI into Podman Store
-[group('Image')]
-load-rechunk image="aurora" tag="latest" flavor="main":
-    #!/usr/bin/bash
-    set -eou pipefail
-
-    # Validate
-    {{ just }} validate {{ image }} {{ tag }} {{ flavor }}
-
-    # Image Name
-    image_name=$({{ just }} image_name {{ image }} {{ tag }} {{ flavor }})
-
-    # Load Image
-    OUT_NAME="${image_name}_build"
-    IMAGE=$(${PODMAN} pull oci:"${PWD}"/"${OUT_NAME}")
-    ${PODMAN} tag ${IMAGE} localhost/"${image_name}":{{ tag }}
-
-    # Cleanup
-    rm -rf "${OUT_NAME}*"
-    rm -f previous.manifest.json
-
 # Run Container
 [group('Image')]
 run $image="aurora" $tag="latest" $flavor="main":
@@ -680,8 +553,6 @@ tag-images image_name="" default_tag="" tags="":
         ${PODMAN} tag $IMAGE {{ image_name }}:${tag}
     done
 
-
-
     # Show Images
     ${PODMAN} images