ci: Split RPM building into separate job

.dockerignore

@@ -21,5 +21,7 @@
 # Workaround for podman bug with secrets + remote
 # https://github.com/containers/podman/issues/25314
 !podman-build-secret*
+# Pre-built packages for builds that use them
+!target/packages/
 # And finally of course all the Rust sources
 !crates/

.github/workflows/ci.yml

@@ -71,12 +71,14 @@ jobs:
           sudo just build-install-test-image
           sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity
 
+          # Grant permission
+          sudo chown -R "$(id -u):$(id -g)" /home/runner/work/bootc/bootc
           # TODO move into a container, and then have this tool run other containers
           cargo build --release -p tests-integration
 
           df -h /
           sudo install -m 0755 target/release/tests-integration /usr/bin/bootc-integration-tests
-          rm target -rf
+          sudo rm target -rf
           df -h /
           # The ostree-container tests
           sudo podman run --privileged --pid=host -v /:/run/host -v $(pwd):/src:ro -v /var/tmp:/var/tmp \
@@ -113,10 +115,40 @@ jobs:
         uses: ./.github/actions/bootc-ubuntu-setup
       - name: Build mdbook
         run: just build-mdbook
+  # Build packages for each test OS
+  package:
+    strategy:
+      fail-fast: false
+      matrix:
+        test_os: [fedora-42, fedora-43, fedora-44, centos-9, centos-10]
+
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Bootc Ubuntu Setup
+        uses: ./.github/actions/bootc-ubuntu-setup
+
+      - name: Setup env
+        run: |
+          BASE=$(just pullspec-for-os base ${{ matrix.test_os }})
+          echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
+
+      - name: Build packages
+        run: just package
+
+      - name: Upload package artifacts
+        uses: actions/upload-artifact@v5
+        with:
+          name: packages-${{ matrix.test_os }}
+          path: target/packages/*.rpm
+          retention-days: 1
+
   # Build bootc from source into a container image FROM each specified base `test_os`
   # running unit and integration tests (using TMT, leveraging the support for nested virtualization
   # in the GHA runners)
   test-integration:
+    needs: package
     strategy:
       fail-fast: false
       matrix:
@@ -149,9 +181,15 @@ jobs:
             echo "BOOTC_buildroot_base=${BUILDROOTBASE}" >> $GITHUB_ENV
           fi
 
+      - name: Download package artifacts
+        uses: actions/download-artifact@v5
+        with:
+          name: packages-${{ matrix.test_os }}
+          path: target/packages/
+
       - name: Build container
         run: |
-          just build-integration-test-image
+          just build-integration-test-image-from-package target/packages
           # Extra cross-check (duplicating the integration test) that we're using the right base
           used_vid=$(podman run --rm localhost/bootc-integration bash -c '. /usr/lib/os-release && echo ${ID}-${VERSION_ID}')
           test ${{ matrix.test_os }} = "${used_vid}"
@@ -177,11 +215,12 @@ jobs:
   # Sentinel job for required checks - configure this job name in repository settings
   required-checks:
     if: always()
-    needs: [cargo-deny, validate, test-integration]
+    needs: [cargo-deny, validate, package, test-integration]
     runs-on: ubuntu-latest
     steps:
       - run: exit 1
         if: >-
           needs.cargo-deny.result != 'success' ||
           needs.validate.result != 'success' ||
+          needs.package.result != 'success' ||
           needs.test-integration.result != 'success'

Dockerfile

@@ -15,6 +15,11 @@ COPY . /src
 FROM scratch as packaging
 COPY contrib/packaging /
 
+# This image captures pre-built packages from the context.
+# By COPYing into a stage, we avoid SELinux issues with context bind mounts.
+FROM scratch as packages
+COPY target/packages/*.rpm /
+
 FROM $base as base
 # Mark this as a test image (moved from --label build flag to fix layer caching)
 LABEL bootc.testimage="1"
@@ -25,8 +30,6 @@ LABEL bootc.testimage="1"
 FROM base as buildroot
 # Flip this off to disable initramfs code
 ARG initramfs=1
-# Version for RPM build (optional, computed from git in Justfile)
-ARG pkgversion=
 # This installs our buildroot, and we want to cache it independently of the rest.
 # Basically we don't want changing a .rs file to blow out the cache of packages.
 RUN --mount=type=bind,from=packaging,target=/run/packaging /run/packaging/install-buildroot
@@ -39,8 +42,10 @@ WORKDIR /src
 RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome cargo fetch
 
 FROM buildroot as build
+# Version for RPM build (optional, computed from git in Justfile)
+ARG pkgversion
 # Build RPM directly from source, using cached target directory
-RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome --network=none RPM_VERSION=${pkgversion} /src/contrib/packaging/build-rpm
+RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome --network=none RPM_VERSION="${pkgversion}" /src/contrib/packaging/build-rpm
 
 # This "build" includes our unit tests
 FROM build as units
@@ -59,14 +64,15 @@ FROM base
 ARG variant
 RUN --mount=type=bind,from=packaging,target=/run/packaging /run/packaging/configure-variant "${variant}"
 # Support overriding the rootfs at build time conveniently
-ARG rootfs=
+ARG rootfs
 RUN --mount=type=bind,from=packaging,target=/run/packaging /run/packaging/configure-rootfs "${variant}" "${rootfs}"
 # Inject additional content
 COPY --from=packaging /usr-extras/ /usr/
-# Install the RPM built in the build stage
-# This replaces the manual file deletion hack and COPY, ensuring proper package management
-# Use rpm -Uvh with --oldpackage to allow replacing with dev version
-COPY --from=build /out/*.rpm /tmp/
-RUN --mount=type=bind,from=packaging,target=/run/packaging --network=none /run/packaging/install-rpm-and-setup /tmp
+# Install packages from the packages stage
+# Using bind from a stage avoids SELinux issues with context bind mounts
+RUN --mount=type=bind,from=packaging,target=/run/packaging \
+    --mount=type=bind,from=packages,target=/build-packages \
+    --network=none \
+    /run/packaging/install-rpm-and-setup /build-packages
 # Finally, testour own linting
 RUN bootc container lint --fatal-warnings

Justfile

@@ -40,7 +40,13 @@ buildargs := "--build-arg=base=" + base + " --build-arg=variant=" + variant
 # Build the container image from current sources.
 # Note commonly you might want to override the base image via e.g.
 # `just build --build-arg=base=quay.io/fedora/fedora-bootc:42`
-build:
+build: package
+    podman build {{base_buildargs}} -t {{base_img}}-bin {{buildargs}} .
+    ./tests/build-sealed {{variant}} {{base_img}}-bin {{base_img}} {{buildroot_base}}
+
+# Build the container image using pre-existing packages from PATH
+build-from-package PATH:
+    # @just copy-packages-from {{PATH}}
     podman build {{base_buildargs}} -t {{base_img}}-bin {{buildargs}} .
     ./tests/build-sealed {{variant}} {{base_img}}-bin {{base_img}} {{buildroot_base}}
 
@@ -65,12 +71,30 @@ _packagecontainer:
     echo "Building RPM with version: ${VERSION}"
     podman build {{base_buildargs}} {{buildargs}} --build-arg=pkgversion=${VERSION} -t localhost/bootc-pkg --target=build .
 
-# Build a packages (e.g. RPM) into target/
+# Build packages (e.g. RPM) into target/packages/
 # Any old packages will be removed.
 package: _packagecontainer
-    mkdir -p target
-    rm -vf target/*.rpm
-    podman run --rm localhost/bootc-pkg tar -C /out/ -cf - . | tar -C target/ -xvf -
+    mkdir -p target/packages
+    rm -vf target/packages/*.rpm
+    podman run --rm localhost/bootc-pkg tar -C /out/ -cf - . | tar -C target/packages/ -xvf -
+    chmod a+rx target target/packages
+    chmod a+r target/packages/*.rpm
+    podman rmi localhost/bootc-pkg
+
+# Copy pre-existing packages from PATH into target/packages/
+# Used to prepare for building with pre-built packages
+copy-packages-from PATH:
+    #!/bin/bash
+    set -xeuo pipefail
+    if ! compgen -G "{{PATH}}/*.rpm" > /dev/null; then
+        echo "Error: No packages found in {{PATH}}" >&2
+        exit 1
+    fi
+    mkdir -p target/packages
+    rm -vf target/packages/*.rpm
+    cp -v {{PATH}}/*.rpm target/packages/
+    chmod a+rx target target/packages
+    chmod a+r target/packages/*.rpm
 
 # This container image has additional testing content and utilities
 build-integration-test-image: build
@@ -79,19 +103,22 @@ build-integration-test-image: build
     # Keep these in sync with what's used in hack/lbi
     podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest
 
+# Build integration test image using pre-existing packages from PATH
+build-integration-test-image-from-package PATH:
+    @just build-from-package {{PATH}}
+    cd hack && podman build {{base_buildargs}} -t {{integration_img}}-bin -f Containerfile .
+    ./tests/build-sealed {{variant}} {{integration_img}}-bin {{integration_img}} {{buildroot_base}}
+    # Keep these in sync with what's used in hack/lbi
+    podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest
+
 # Build+test using the `composefs-sealeduki-sdboot` variant.
 test-composefs:
     # These first two are currently a distinct test suite from tmt that directly
     # runs an integration test binary in the base image via bcvk
-    just _composefs-build-image
     cargo run --release -p tests-integration -- composefs-bcvk {{integration_img}}
     # We're trying to move more testing to tmt
     just variant=composefs-sealeduki-sdboot test-tmt readonly local-upgrade-reboot
 
-# Internal helper to build integration test image with composefs variant
-_composefs-build-image:
-    just variant=composefs-sealeduki-sdboot build-integration-test-image
-
 # Only used by ci.yml right now
 build-install-test-image: build-integration-test-image
     cd hack && podman build {{base_buildargs}} -t {{integration_img}}-install -f Containerfile.drop-lbis

contrib/packaging/install-rpm-and-setup

@@ -7,7 +7,7 @@ RPM_DIR="${1:-/tmp}"
 # Install the RPM package
 # Use rpm -Uvh with --oldpackage to allow replacing with dev version
 rpm -Uvh --oldpackage "${RPM_DIR}"/*.rpm
-rm -f "${RPM_DIR}"/*.rpm
+# Note: we don't need to clean up the source directory since it's a bind mount
 
 # Regenerate initramfs if we have initramfs-setup
 kver=$(cd /usr/lib/modules && echo *)

tmt/tests/booted/test-image-pushpull-upgrade.nu

@@ -86,6 +86,11 @@ RUN echo test content > /usr/share/blah.txt
     # Test for https://github.com/ostreedev/ostree/issues/3544
     # Add a quoted karg using rpm-ostree if available
     if not $is_composefs {
+        # Check rpm-ostree and rpm-ostreed service status before run rpm-ostree
+        # And collect info for flaky error "error: System transaction in progress"
+        rpm-ostree status
+        journalctl -u rpm-ostreed
+
         print "Adding quoted karg via rpm-ostree to test ostree issue #3544"
         rpm-ostree kargs --append=($quoted_karg)
     }