Rework GHA testing: Use bcvk, cover composefs with tmt

Part 1: Use bcvk

For local tests, right now testcloud+tmt doesn't support UEFI, see
teemtee/tmt#4203

This is a blocker for us doing more testing with UKIs.

In this patch we switch to provisioning VMs with bcvk, which
fixes this - but beyond that a really compelling thing about
this is that bcvk is *also* designed to be ergonomic and efficient
beyond just being a test runner, with things like virtiofs
mounting of host container storage, etc.

In other words, bcvk is the preferred way to run local virt
with bootc, and this makes our TMT tests use it.

Now a major downside of this though is we're effectively
implementing a new "provisioner" for tmt (bypassing the
existing `virtual`). In the more medium term I think we
want to add `bcvk` as a provisioner option to tmt.

Anyways for now, this works by discovers test plans via `tmt plan ls`,
spawning a separate VM per test, and then using uses tmt's connect
provisioner to run tests targeting these externally provisioned
systems.

Part 2: Rework the Justfile and Dockerfile

This adds `base` and `variant` arguments which are propagated through
the system, and we have a new `variant` for sealed composefs.

The readonly tests now pass with composefs.

Assisted-by: Claude Code (Sonnet 4.5)
Signed-off-by: Colin Walters <[email protected]>

Author: cgwalters

.github/workflows/ci.yml

@@ -141,47 +141,23 @@ jobs:
       - name: Install tmt
         run: pip install --user "tmt[provision-virtual]"
 
-      - name: Build container and disk image
+      - name: Setup env
         run: |
-          set -xeuo pipefail
-          build_args=()
-          # Map from an ID-VERSIONID pair to a container ref
-          target=${{ matrix.test_os }}
-          OS_ID=$(echo "$target" | cut -d '-' -f 1)
-          OS_VERSION_ID=$(echo "$target" | cut -d '-' -f 2)
-          # Base image
-          case "$OS_ID" in
-              "centos")
-                  BASE="quay.io/centos-bootc/centos-bootc:stream${OS_VERSION_ID}"
-              ;;
-              "fedora")
-                  BASE="quay.io/fedora/fedora-bootc:${OS_VERSION_ID}"
-              ;;
-              *) echo "Unknown OS: ${OS_ID}" 1>&2; exit 1
-              ;;
-          esac
-          build_args+=("--build-arg=base=$BASE")
-          just build ${build_args[@]}
-          just build-integration-test-image
-          # Cross check we're using the right base
-          used_vid=$(podman run --rm localhost/bootc-integration bash -c '. /usr/lib/os-release && echo $VERSION_ID')
-          test "$OS_VERSION_ID" = "${used_vid}"
-
-      - name: Run container tests
-        run: |
-          just test-container
+          BASE=$(just pullspec-for-os ${{ matrix.test_os }})
+          echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
 
-      - name: Generate disk image
+      - name: Build container
         run: |
-          mkdir -p target
-          just build-disk-image localhost/bootc-integration target/bootc-integration-test.qcow2
+          just build-integration-test-image
+          # Extra cross-check (duplicating the integration test) that we're using the right base
+          used_vid=$(podman run --rm localhost/bootc-integration bash -c '. /usr/lib/os-release && echo ${ID}-${VERSION_ID}')
+          test ${{ matrix.test_os }} = "${used_vid}"
 
-      - name: Workaround https://github.com/teemtee/testcloud/issues/18
-        run: sudo rm -f /usr/bin/chcon && sudo ln -sr /usr/bin/true /usr/bin/chcon
+      - name: Unit and container integration tests
+        run: just test-container
 
       - name: Run all TMT tests
-        run: |
-          just test-tmt-nobuild
+        run: just test-tmt
 
       - name: Archive TMT logs
         if: always()
@@ -194,7 +170,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # TODO expand this matrix, we need to make it better to override the target
+        # OS via Justfile variables too
         test_os: [centos-10]
+        variant: [composefs-sealeduki-sdboot]
 
     runs-on: ubuntu-24.04
 
@@ -204,9 +183,29 @@ jobs:
         uses: ./.github/actions/bootc-ubuntu-setup
         with:
           libvirt: true
+      - name: Install tmt
+        run: pip install --user "tmt[provision-virtual]"
+
+      - name: Setup env
+        run: |
+          BASE=$(just pullspec-for-os ${{ matrix.test_os }})
+          echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
+          echo "BOOTC_variant="${{ matrix.variant }} >> $GITHUB_ENV
 
       - name: Build container
-        run: just build-sealed
+        run: |
+          just build-integration-test-image
+
+      - name: Unit and container integration tests
+        run: just test-container
 
-      - name: Test
-        run: just test-composefs
+      - name: Run readonly TMT tests
+        # TODO: expand to more tests
+        run: just test-tmt readonly
+
+      - name: Archive TMT logs
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-${{ matrix.tmt_plan }}
+          path: /var/tmp/tmt

Cargo.lock

@@ -1,10 +1,8 @@
 # Build this project from source and write the updated content
 # (i.e. /usr/bin/bootc and systemd units) to a new derived container
 # image. See the `Justfile` for an example
-#
-# Use e.g. --build-arg=base=quay.io/fedora/fedora-bootc:42 to target
-# Fedora instead.
 
+# Note this is usually overridden via Justfile
 ARG base=quay.io/centos-bootc/centos-bootc:stream10
 
 # This first image captures a snapshot of the source code,
@@ -94,20 +92,60 @@ RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothom
 
 # The final image that derives from the original base and adds the release binaries
 FROM base
-# Set this to 1 to default to systemd-boot
-ARG sdboot=0
+# See the Justfile for possible variants
+ARG variant
 RUN <<EORUN
 set -xeuo pipefail
 # Ensure we've flushed out prior state (i.e. files no longer shipped from the old version);
 # and yes, we may need to go to building an RPM in this Dockerfile by default.
 rm -vf /usr/lib/systemd/system/multi-user.target.wants/bootc-*
-if test "$sdboot" = 1; then
-  dnf -y install systemd-boot-unsigned
-  # And uninstall bootupd
-  rpm -e bootupd
-  rm /usr/lib/bootupd/updates -rf
-  dnf clean all
-  rm -rf /var/cache /var/lib/{dnf,rhsm} /var/log/*
+case "${variant}" in
+  *-sdboot)
+    dnf -y install systemd-boot-unsigned
+    # And uninstall bootupd
+    rpm -e bootupd
+    rm /usr/lib/bootupd/updates -rf
+    dnf clean all
+    rm -rf /var/cache /var/lib/{dnf,rhsm} /var/log/*
+  ;;
+esac
+EORUN
+# Support overriding the rootfs at build time conveniently
+ARG rootfs=
+RUN <<EORUN
+set -xeuo pipefail
+# Do we have an explicit build-time override? Then write it.
+if test -n "$rootfs"; then
+  cat > /usr/lib/bootc/install/80-rootfs-override.toml <<EOF
+[install.filesystem.root]
+type = "$rootfs"
+EOF
+else
+  # Query the default rootfs
+  base_rootfs=$(bootc install print-configuration | jq -r '.filesystem.root.type // ""')
+  # No filesystem override set. If we're doing composefs, we need a FS that
+  # supports fsverity. If btrfs is available we'll pick that, otherwise ext4.
+  fs=
+  case "${variant}" in
+    composefs*)
+      btrfs=$(grep -qEe '^CONFIG_BTRFS_FS' /usr/lib/modules/*/config && echo btrfs || true)
+      fs=${btrfs:-ext4}
+      ;;
+    *)
+      # No explicit filesystem set and we're not using composefs. Default to xfs
+      # with the rationale that we're trying to get filesystem coverage across
+      # all the cases in general.
+      if test -z "${base_rootfs}"; then
+        fs=xfs
+      fi 
+      ;;
+  esac
+  if test -n "$fs"; then
+    cat > /usr/lib/bootc/install/80-ext4-composefs.toml <<EOF
+[install.filesystem.root]
+type = "${fs}"
+EOF
+  fi
 fi
 EORUN
 # Create a layer that is our new binaries

Dockerfile

@@ -11,26 +11,35 @@
 
 # --------------------------------------------------------------------
 
+# ostree: The default
+# composefs-sealeduki-sdboot: A system with a sealed composefs using systemd-boot
+variant := env("BOOTC_variant", "ostree")
+base := env("BOOTC_base", "quay.io/centos-bootc/centos-bootc:stream10")
+
+buildargs := "--build-arg=base=" + base + " --build-arg=variant=" + variant
+
 # Build the container image from current sources.
 # Note commonly you might want to override the base image via e.g.
 # `just build --build-arg=base=quay.io/fedora/fedora-bootc:42`
-build *ARGS:
-    podman build --jobs=4 -t localhost/bootc {{ARGS}} .
-
-# Build a sealed image from current sources. This will default to
-# generating Secure Boot keys in target/test-secureboot.
-build-sealed *ARGS:
-    podman build --build-arg=sdboot=1 --jobs=4 -t localhost/bootc-unsealed {{ARGS}} .
-    ./tests/build-sealed localhost/bootc-unsealed localhost/bootc
+build:
+    podman build --jobs=4 -t localhost/bootc-bin {{buildargs}} .
+    ./tests/build-sealed {{variant}} localhost/bootc-bin localhost/bootc
 
 # This container image has additional testing content and utilities
-build-integration-test-image *ARGS:
-    cd hack && podman build --jobs=4 -t localhost/bootc-integration -f Containerfile {{ARGS}} .
+build-integration-test-image: build
+    cd hack && podman build --jobs=4 -t localhost/bootc-integration-bin {{buildargs}} -f Containerfile .
+    ./tests/build-sealed {{variant}} localhost/bootc-integration-bin localhost/bootc-integration
     # Keep these in sync with what's used in hack/lbi
     podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest
 
-test-composefs: build-sealed
+# Build+test composefs; compat alias
+test-composefs:
+    # These first two are currently a distinct test suite from tmt that directly
+    # runs an integration test binary in the base image via bcvk
+    just variant=composefs-sealeduki-sdboot build
     cargo run --release -p tests-integration -- composefs-bcvk localhost/bootc
+    # We're trying to move more testing to tmt, so 
+    just variant=composefs-sealeduki-sdboot test-tmt readonly
 
 # Only used by ci.yml right now
 build-install-test-image: build-integration-test-image
@@ -59,27 +68,27 @@ validate-local:
 build-disk *ARGS:
     ./tests/build.sh {{ARGS}}
 
-# The tests which run a fully booted bootc system (i.e. where in place
-# updates are supported) as if it were a production environment use
-# https://github.com/teemtee/tmt.
+# Run tmt-based test suites using local virtual machines with
+# bcvk.
 #
-# This task runs *all* of the tmt-based tests targeting the disk image generated
-# in the previous step.
-test-tmt *ARGS: build-disk
-    ./tests/run-tmt.sh {{ARGS}}
+# To run an individual test, pass it as an argument like:
+# `just test-tmt readonly`
+test-tmt *ARGS: build-integration-test-image
+    cargo xtask run-tmt --env=BOOTC_variant={{variant}} localhost/bootc-integration {{ARGS}}
 
-# Like test-tmt but assumes that a disk image is already built
-test-tmt-nobuild *ARGS:
-    ./tests/run-tmt.sh {{ARGS}}
-
-# Run just one tmt test: `just test-tmt-one test-20-local-upgrade`
-test-tmt-one PLAN: build-disk
-    ./tests/run-tmt.sh plan --name {{PLAN}}
+# Cleanup all test VMs created by tmt tests
+tmt-vm-cleanup:
+    bcvk libvirt rm --stop --force --label bootc.test=1
 
 # Run tests (unit and integration) that are containerized
 test-container: build-units build-integration-test-image
     podman run --rm --read-only localhost/bootc-units /usr/bin/bootc-units
-    podman run --rm localhost/bootc-integration bootc-integration-tests container
+    # Pass these through for cross-checking
+    podman run --rm --env=BOOTC_variant={{variant}} --env=BOOTC_base={{base}} localhost/bootc-integration bootc-integration-tests container
+
+# Print the container image reference for a given short $ID-VERSION_ID
+pullspec-for-os NAME:
+    @jq -r --arg v "{{NAME}}" '.[$v]' < hack/os-image-map.json
 
 build-mdbook:
     cd docs && podman build -t localhost/bootc-mdbook -f Dockerfile.mdbook

Justfile

@@ -1,6 +1,7 @@
 use std::process::Command;
 
 use anyhow::{Context, Result};
+use camino::Utf8Path;
 use fn_error_context::context;
 use libtest_mimic::Trial;
 use xshell::{cmd, Shell};
@@ -49,10 +50,42 @@ fn test_system_reinstall_help() -> Result<()> {
     Ok(())
 }
 
+/// Verify that the values of `variant` and `base` from Justfile actually applied
+/// to this container image.
+fn test_variant_base_crosscheck() -> Result<()> {
+    if let Some(variant) = std::env::var("BOOTC_variant").ok() {
+        // TODO add this to `bootc status` or so?
+        let boot_efi = Utf8Path::new("/boot/EFI");
+        match variant.as_str() {
+            "ostree" => {
+                assert!(!boot_efi.try_exists()?);
+            }
+            "composefs-sealeduki-sdboot" => {
+                assert!(boot_efi.try_exists()?);
+            }
+            o => panic!("Unhandled variant: {o}"),
+        }
+    }
+    if let Some(base) = std::env::var("BOOTC_base").ok() {
+        // Hackily reverse back from container pull spec to ID-VERSION_ID
+        // TODO: move the OsReleaseInfo into an internal crate we use
+        let osrelease = std::fs::read_to_string("/usr/lib/os-release")?;
+        if base.contains("centos-bootc") {
+            assert!(osrelease.contains(r#"ID="centos""#))
+        } else if base.contains("fedora-bootc") {
+            assert!(osrelease.contains(r#"ID=fedora"#));
+        } else {
+            eprintln!("notice: Unhandled base {base}")
+        }
+    }
+    Ok(())
+}
+
 /// Tests that should be run in a default container image.
 #[context("Container tests")]
 pub(crate) fn run(testargs: libtest_mimic::Arguments) -> Result<()> {
     let tests = [
+        new_test("variant-base-crosscheck", test_variant_base_crosscheck),
         new_test("bootc upgrade", test_bootc_upgrade),
         new_test("install config", test_bootc_install_config),
         new_test("status", test_bootc_status),

crates/tests-integration/src/container.rs

@@ -17,6 +17,7 @@ anyhow = { workspace = true }
 anstream = { workspace = true }
 camino = { workspace = true }
 chrono = { workspace = true, features = ["std"] }
+clap = { workspace = true, features = ["derive"] }
 fn-error-context = { workspace = true }
 owo-colors = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
@@ -27,6 +28,7 @@ xshell = { workspace = true }
 
 # Crate-specific dependencies
 mandown = "1.1.0"
+rand = "0.8"
 tar = "0.4"
 
 [lints]