wip: refactor build+test to support test composefs with tmt

Signed-off-by: Colin Walters <[email protected]>

Author: cgwalters

.github/actions/bootc-ubuntu-setup/action.yml

@@ -77,7 +77,7 @@ runs:
       shell: bash
       run: |
         set -xeuo pipefail
-        export BCVK_VERSION=0.5.3
+        export BCVK_VERSION=0.6.0
         /bin/time -f '%E %C' sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-utils qemu-kvm virtiofsd libvirt-daemon-system
         # Something in the stack is overriding this, but we want session right now for bcvk
         echo LIBVIRT_DEFAULT_URI=qemu:///session >> $GITHUB_ENV

.github/workflows/ci.yml

@@ -144,23 +144,8 @@ jobs:
       - name: Build container and disk image
         run: |
           set -xeuo pipefail
-          build_args=()
-          # Map from an ID-VERSIONID pair to a container ref
-          target=${{ matrix.test_os }}
-          OS_ID=$(echo "$target" | cut -d '-' -f 1)
-          OS_VERSION_ID=$(echo "$target" | cut -d '-' -f 2)
-          # Base image
-          case "$OS_ID" in
-              "centos")
-                  BASE="quay.io/centos-bootc/centos-bootc:stream${OS_VERSION_ID}"
-              ;;
-              "fedora")
-                  BASE="quay.io/fedora/fedora-bootc:${OS_VERSION_ID}"
-              ;;
-              *) echo "Unknown OS: ${OS_ID}" 1>&2; exit 1
-              ;;
-          esac
-          build_args+=("--build-arg=base=$BASE")
+          target=$(just pullspec-for-os ${{ matrix.test_os }})
+          build_args=("--build-arg=base=$BASE")
           just build ${build_args[@]}
           just build-integration-test-image
           # Cross check we're using the right base
@@ -194,7 +179,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # TODO expand this matrix, we need to make it better to override the target
+        # OS via Justfile variables too
         test_os: [centos-10]
+        variant: [composefs-sealeduki-sdboot]
 
     runs-on: ubuntu-24.04
 
@@ -204,9 +192,11 @@ jobs:
         uses: ./.github/actions/bootc-ubuntu-setup
         with:
           libvirt: true
+      - name: Install tmt
+        run: pip install --user "tmt[provision-virtual]"
 
       - name: Build container
-        run: just build-sealed
+        run: just variant=composefs-sealeduki-sdboot build-integration-test-image
 
-      - name: Test
-        run: just test-composefs
+      - name: Test (readonly)
+        run: just variant=composefs-sealeduki-sdboot test-tmt readonly

Dockerfile

@@ -94,21 +94,31 @@ RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothom
 
 # The final image that derives from the original base and adds the release binaries
 FROM base
-# Set this to 1 to default to systemd-boot
-ARG sdboot=0
+# See the Justfile for more on this
+ARG variant
 RUN <<EORUN
 set -xeuo pipefail
 # Ensure we've flushed out prior state (i.e. files no longer shipped from the old version);
 # and yes, we may need to go to building an RPM in this Dockerfile by default.
 rm -vf /usr/lib/systemd/system/multi-user.target.wants/bootc-*
-if test "$sdboot" = 1; then
-  dnf -y install systemd-boot-unsigned
-  # And uninstall bootupd
-  rpm -e bootupd
-  rm /usr/lib/bootupd/updates -rf
-  dnf clean all
-  rm -rf /var/cache /var/lib/{dnf,rhsm} /var/log/*
-fi
+case "${variant}" in
+  *-sdboot)
+    dnf -y install systemd-boot-unsigned
+    # And uninstall bootupd
+    rpm -e bootupd
+    rm /usr/lib/bootupd/updates -rf
+    dnf clean all
+    rm -rf /var/cache /var/lib/{dnf,rhsm} /var/log/*
+  ;;
+esac
+case "${variant}" in
+  composefs*)
+    cat > /usr/lib/bootc/install/80-ext4-composefs.toml <<EOF
+[install.filesystem.root]
+type = "ext4"
+EOF
+  ;;
+esac
 EORUN
 # Create a layer that is our new binaries
 COPY --from=build /out/ /

Justfile

@@ -11,25 +11,27 @@
 
 # --------------------------------------------------------------------
 
+# ostree: The default
+# composefs-sealeduki-sdboot: A system with a sealed composefs using systemd-boot
+variant := "ostree"
+
 # Build the container image from current sources.
 # Note commonly you might want to override the base image via e.g.
 # `just build --build-arg=base=quay.io/fedora/fedora-bootc:42`
 build *ARGS:
-    podman build --jobs=4 -t localhost/bootc {{ARGS}} .
-
-# Build a sealed image from current sources. This will default to
-# generating Secure Boot keys in target/test-secureboot.
-build-sealed *ARGS:
-    podman build --build-arg=sdboot=1 --jobs=4 -t localhost/bootc-unsealed {{ARGS}} .
-    ./tests/build-sealed localhost/bootc-unsealed localhost/bootc
+    podman build --jobs=4 -t localhost/bootc-bin --build-arg=variant={{variant}} {{ARGS}} .
+    ./tests/build-sealed {{variant}} localhost/bootc-bin localhost/bootc
 
 # This container image has additional testing content and utilities
-build-integration-test-image *ARGS:
-    cd hack && podman build --jobs=4 -t localhost/bootc-integration -f Containerfile {{ARGS}} .
+build-integration-test-image *ARGS: build
+    cd hack && podman build --jobs=4 -t localhost/bootc-integration-bin -f Containerfile {{ARGS}} .
+    ./tests/build-sealed {{variant}} localhost/bootc-integration-bin localhost/bootc-integration
     # Keep these in sync with what's used in hack/lbi
     podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest
 
-test-composefs: build-sealed
+# Build+test composefs; compat alias
+test-composefs:
+    just build --variant=composefs-sealeduki-sdboot
     cargo run --release -p tests-integration -- composefs-bcvk localhost/bootc
 
 # Only used by ci.yml right now
@@ -80,6 +82,23 @@ test-container: build-units build-integration-test-image
     podman run --rm --read-only localhost/bootc-units /usr/bin/bootc-units
     podman run --rm localhost/bootc-integration bootc-integration-tests container
 
+# Print the container image reference for a given short $ID-VERSION_ID
+pullspec-for-os NAME:
+    #!/bin/bash
+    OS_ID=$(echo "{{NAME}}" | cut -d '-' -f 1)
+    OS_VERSION_ID=$(echo "{{NAME}}" | cut -d '-' -f 2)
+    # Base image
+    case "$OS_ID" in
+        "centos")
+            echo "quay.io/centos-bootc/centos-bootc:stream${OS_VERSION_ID}"
+        ;;
+        "fedora")
+            echo "quay.io/fedora/fedora-bootc:${OS_VERSION_ID}"
+        ;;
+        *) echo "Unknown OS: ${OS_ID}" 1>&2; exit 1
+        ;;
+    esac
+
 build-mdbook:
     cd docs && podman build -t localhost/bootc-mdbook -f Dockerfile.mdbook
 

crates/xtask/src/xtask.rs

@@ -468,6 +468,15 @@ fn check_dependencies(sh: &Shell) -> Result<()> {
     Ok(())
 }
 
+const COMMON_INST_ARGS: &[&str] = &[
+    // We don't use cloud-init with bcvk right now, but it needs to be there for
+    // testing-farm+tmt
+    "--karg=ds=iid-datasource-none",
+    // TODO: Pass down the Secure Boot keys for tests if present
+    "--firmware=uefi-insecure",
+    "--label=bootc.test=1"
+];
+
 /// Run TMT tests using bcvk for VM management
 /// This spawns a separate VM per test plan to avoid state leakage between tests.
 ///
@@ -561,10 +570,13 @@ fn run_tmt(sh: &Shell, args: &[String]) -> Result<()> {
         println!("========================================\n");
 
         // Launch VM with bcvk
-        // Use ds=iid-datasource-none to disable cloud-init for faster boot
-        let launch_result = cmd!(sh, "bcvk libvirt run --name {vm_name} --detach --filesystem ext4 --karg=ds=iid-datasource-none {image}")
-            .run()
-            .context("Launching VM with bcvk");
+
+        let launch_result = cmd!(
+            sh,
+            "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {image}"
+        )
+        .run()
+        .context("Launching VM with bcvk");
 
         if let Err(e) = launch_result {
             eprintln!("Failed to launch VM for plan {}: {:#}", plan, e);
@@ -743,9 +755,12 @@ fn tmt_provision(sh: &Shell, args: &[String]) -> Result<()> {
 
     // Launch VM with bcvk
     // Use ds=iid-datasource-none to disable cloud-init for faster boot
-    cmd!(sh, "bcvk libvirt run --name {vm_name} --detach --filesystem ext4 --karg=ds=iid-datasource-none {image}")
-        .run()
-        .context("Launching VM with bcvk")?;
+    cmd!(
+        sh,
+        "bcvk libvirt run --name {vm_name} --detach {COMMON_INST_ARGS...} {image}"
+    )
+    .run()
+    .context("Launching VM with bcvk")?;
 
     println!("VM launched, waiting for SSH...");
 
@@ -758,8 +773,7 @@ fn tmt_provision(sh: &Shell, args: &[String]) -> Result<()> {
         .context("Creating target directory")?;
     let key_path = key_dir.join(format!("{}.ssh-key", vm_name));
 
-    std::fs::write(&key_path, ssh_key)
-        .context("Writing SSH key file")?;
+    std::fs::write(&key_path, ssh_key).context("Writing SSH key file")?;
 
     // Set proper permissions on key file (0600)
     #[cfg(unix)]
@@ -786,7 +800,10 @@ fn tmt_provision(sh: &Shell, args: &[String]) -> Result<()> {
     println!("    --user root --key {} \\", key_path);
     println!("    plan --name <PLAN_NAME>");
     println!("\nTo connect via SSH:");
-    println!("  ssh -i {} -p {} -o IdentitiesOnly=yes root@localhost", key_path, ssh_port);
+    println!(
+        "  ssh -i {} -p {} -o IdentitiesOnly=yes root@localhost",
+        key_path, ssh_port
+    );
     println!("\nTo cleanup:");
     println!("  bcvk libvirt rm --stop --force {}", vm_name);
     println!("========================================\n");

hack/provision-derived.sh

@@ -45,8 +45,11 @@ dnf clean all
 cat <<KARGEOF >> /usr/lib/bootc/kargs.d/20-console.toml
 kargs = ["console=ttyS0,115200n8"]
 KARGEOF
-# And cloud-init stuff
-ln -s ../cloud-init.target /usr/lib/systemd/system/default.target.wants
+# And cloud-init stuff, unless we're doing a UKI which is always
+# tested with bcvk
+if test '!' -d /boot/EFI; then
+  ln -s ../cloud-init.target /usr/lib/systemd/system/default.target.wants
+fi
 
 # Allow root SSH login for testing with bcvk/tmt
 mkdir -p /etc/cloud/cloud.cfg.d

tests/build-sealed

@@ -2,6 +2,8 @@
 set -euo pipefail
 # This should turn into https://github.com/bootc-dev/bootc/issues/1498
 
+variant=$1
+shift
 # The un-sealed container image we want to use
 input_image=$1
 shift
@@ -17,6 +19,19 @@ runv() {
   "$@"
 }
 
+case $variant in
+  ostree)
+    # Nothing to do
+    podman tag $input_image $output_image
+    ;;
+  composefs-sealeduki*)
+    ;;
+  *) 
+    echo "Unknown variant=$variant" 1>&2; exit 1
+    ;;
+esac
+
+
 graphroot=$(podman system info -f '{{.Store.GraphRoot}}')
 echo "Computing composefs digest..."
 cfs_digest=$(podman run --rm --privileged --read-only --security-opt=label=disable -v /sys:/sys:ro --net=none \

tmt/tests/booted/readonly/001-test-status.nu

@@ -11,15 +11,23 @@ assert equal $st.apiVersion org.containers.bootc/v1
 
 let st = bootc status --format=yaml | from yaml
 assert equal $st.apiVersion org.containers.bootc/v1
-assert ($st.status.booted.image.timestamp != null)
+# Detect composefs by checking if composefs field is present
+let is_composefs = ($st.status.booted.composefs? != null)
+if not $is_composefs {
+    assert ($st.status.booted.image.timestamp != null)
+} # else { TODO composefs: timestamp is not populated with composefs }
 let ostree = $st.status.booted.ostree
 if $ostree != null {
     assert ($ostree.stateroot != null)
 }
 
 let st = bootc status --json --booted | from json
 assert equal $st.apiVersion org.containers.bootc/v1
-assert ($st.status.booted.image.timestamp != null)
+# Detect composefs by checking if composefs field is present
+let is_composefs = ($st.status.booted.composefs? != null)
+if not $is_composefs {
+    assert ($st.status.booted.image.timestamp != null)
+} # else { TODO composefs: timestamp is not populated with composefs }
 assert (($st.status | get rollback | default null) == null)
 assert (($st.status | get staged | default null) == null)
 

tmt/tests/booted/readonly/010-test-bootc-container-store.nu

@@ -3,10 +3,18 @@ use tap.nu
 
 tap begin "verify bootc-owned container storage"
 
-# Just verifying that the additional store works
-podman --storage-opt=additionalimagestore=/usr/lib/bootc/storage images
+# Detect composefs by checking if composefs field is present
+let st = bootc status --json | from json
+let is_composefs = ($st.status.booted.composefs? != null)
 
-# And verify this works
-bootc image cmd list -q o>/dev/null
+if $is_composefs {
+    print "# TODO composefs: skipping test - /usr/lib/bootc/storage doesn't exist with composefs"
+} else {
+    # Just verifying that the additional store works
+    podman --storage-opt=additionalimagestore=/usr/lib/bootc/storage images
+
+    # And verify this works
+    bootc image cmd list -q o>/dev/null
+}
 
 tap ok

tmt/tests/booted/readonly/011-test-ostree-ext-cli.nu

@@ -7,7 +7,15 @@ tap begin "verify bootc wrapping ostree-ext"
 
 # Parse the status and get the booted image
 let st = bootc status --json | from json
-let booted = $st.status.booted.image
-# Then verify we can extract its metadata via the ostree-container code.
-let metadata = bootc internals ostree-container image metadata --repo=/ostree/repo $"($booted.image.transport):($booted.image.image)" | from json
-assert equal $metadata.mediaType "application/vnd.oci.image.manifest.v1+json"
+# Detect composefs by checking if composefs field is present
+let is_composefs = ($st.status.booted.composefs? != null)
+if $is_composefs {
+    print "# TODO composefs: skipping test - ostree-container commands don't work with composefs"
+} else {
+    let booted = $st.status.booted.image
+    # Then verify we can extract its metadata via the ostree-container code.
+    let metadata = bootc internals ostree-container image metadata --repo=/ostree/repo $"($booted.image.transport):($booted.image.image)" | from json
+    assert equal $metadata.mediaType "application/vnd.oci.image.manifest.v1+json"
+}
+
+tap ok