[WIP] test: Add integration test running on github runner

.github/workflows/integration.yml

@@ -0,0 +1,100 @@
+name: bootc integration test
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        test_os: [fedora-41, fedora-42, fedora-43, centos-9]
+        test_runner: [ubuntu-latest, ubuntu-24.04-arm]
+
+    runs-on: ${{ matrix.test_runner }}
+
+    steps:
+      - name: Install podman for heredoc support
+        run: |
+          set -eux
+          echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list
+          sudo apt update
+          sudo apt install -y crun/testing podman/testing
+
+      - uses: actions/checkout@v4
+
+      - name: Build bootc and bootc image
+        env:
+          TEST_OS: ${{ matrix.test_os }}
+        run: sudo -E TEST_OS=$TEST_OS tests/build.sh
+
+      - name: Grant sudo user permission to archive files
+        run: |
+          sudo chmod 0755 /tmp/tmp-bootc-build/id_rsa
+
+      - name: Archive bootc disk image - disk.raw
+        if: matrix.test_runner == 'ubuntu-latest'
+        uses: actions/upload-artifact@v4
+        with:
+          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-disk
+          path: /tmp/tmp-bootc-build/disk.raw
+          retention-days: 1
+
+      - name: Archive SSH private key - id_rsa
+        if: matrix.test_runner == 'ubuntu-latest'
+        uses: actions/upload-artifact@v4
+        with:
+          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-id_rsa
+          path: /tmp/tmp-bootc-build/id_rsa
+          retention-days: 1
+
+  test:
+    needs: build
+    strategy:
+      matrix:
+        test_os: [fedora-41, fedora-42, fedora-43, centos-9]
+        tmt_plan: [test-01-readonly, test-20-local-upgrade, test-21-logically-bound-switch, test-22-logically-bound-install, test-23-install-outside-container, test-24-local-upgrade-reboot]
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install dependence
+        run: |
+          sudo apt-get update
+          sudo apt install -y qemu-kvm qemu-system
+          pip install --user tmt
+
+      - name: Create folder to save disk image
+        run: mkdir -p /tmp/tmp-bootc-build
+
+      - name: Download disk.raw
+        uses: actions/download-artifact@v4
+        with:
+          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-disk
+          path: /tmp/tmp-bootc-build
+
+      - name: Download id_rsa
+        uses: actions/download-artifact@v4
+        with:
+          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-id_rsa
+          path: /tmp/tmp-bootc-build
+
+      - name: Enable KVM group perms
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+          ls -l /dev/kvm
+
+      - name: Run test
+        env:
+          TMT_PLAN_NAME: ${{ matrix.tmt_plan }}
+        run: chmod 600 /tmp/tmp-bootc-build/id_rsa && tests/test.sh
+
+      - name: Archive TMT logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ matrix.tmt_plan }}
+          path: /var/tmp/tmt

Makefile

@@ -66,6 +66,9 @@ test-bin-archive: all
 test-tmt:
 	cargo xtask test-tmt
 
+test:
+	tests/build.sh && tests/test.sh
+
 # This gates CI by default. Note that for clippy, we gate on
 # only the clippy correctness and suspicious lints, plus a select
 # set of default rustc warnings.

tests/build.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+set -exuo pipefail
+
+# This script basically builds bootc from source using the provided base image,
+# then runs the target tests.
+
+mkdir -p /tmp/tmp-bootc-build
+BOOTC_TEMPDIR="/tmp/tmp-bootc-build"
+
+# Get OS info from TEST_OS env
+OS_ID=$(echo "$TEST_OS" | cut -d '-' -f 1)
+OS_VERSION_ID=$(echo "$TEST_OS" | cut -d '-' -f 2)
+
+# Base image
+case "$OS_ID" in
+    "centos")
+        TIER1_IMAGE_URL="quay.io/centos-bootc/centos-bootc:stream${OS_VERSION_ID}"
+        ;;
+    "fedora")
+        TIER1_IMAGE_URL="quay.io/fedora/fedora-bootc:${OS_VERSION_ID}"
+        ;;
+esac
+
+CONTAINERFILE="${BOOTC_TEMPDIR}/Containerfile"
+tee "$CONTAINERFILE" > /dev/null << CONTAINERFILEOF
+FROM $TIER1_IMAGE_URL as build
+
+WORKDIR /code
+
+RUN <<EORUN
+set -xeuo pipefail
+. /usr/lib/os-release
+case \$ID in
+    centos|rhel) dnf config-manager --set-enabled crb;;
+    fedora) dnf -y install dnf-utils 'dnf5-command(builddep)';;
+esac
+dnf -y builddep contrib/packaging/bootc.spec
+dnf -y install git-core
+EORUN
+
+RUN mkdir -p /build/target/dev-rootfs
+# git config --global --add safe.directory /code to fix "fatal: detected dubious ownership in repository at '/code'" error
+RUN --mount=type=cache,target=/build/target --mount=type=cache,target=/var/roothome git config --global --add safe.directory /code && make test-bin-archive && mkdir -p /out && cp target/bootc.tar.zst /out
+
+FROM $TIER1_IMAGE_URL
+
+# Inject our built code
+COPY --from=build /out/bootc.tar.zst /tmp
+RUN tar -C / --zstd -xvf /tmp/bootc.tar.zst && rm -vrf /tmp/*
+
+RUN <<EORUN
+set -xeuo pipefail
+
+# Provision test requirement
+/code/hack/provision-derived.sh
+# Also copy in some default install configs we use for testing
+cp -a /code/hack/install-test-configs/* /usr/lib/bootc/install/
+# And some test kargs
+cp -a /code/hack/test-kargs/* /usr/lib/bootc/kargs.d/
+
+# For testing farm
+mkdir -p -m 0700 /var/roothome
+
+# Enable ttyS0 console
+mkdir -p /usr/lib/bootc/kargs.d/
+cat <<KARGEOF >> /usr/lib/bootc/kargs.d/20-console.toml
+kargs = ["console=ttyS0,115200n8"]
+KARGEOF
+
+# For test-22-logically-bound-install
+cp -a /code/tmt/tests/lbi/usr/. /usr
+ln -s /usr/share/containers/systemd/curl.container /usr/lib/bootc/bound-images.d/curl.container
+ln -s /usr/share/containers/systemd/curl-base.image /usr/lib/bootc/bound-images.d/curl-base.image
+ln -s /usr/share/containers/systemd/podman.image /usr/lib/bootc/bound-images.d/podman.image
+
+# Install rsync which is required by tmt
+dnf -y install cloud-init rsync
+dnf -y clean all
+
+rm -rf /var/cache /var/lib/dnf
+EORUN
+CONTAINERFILEOF
+
+LOCAL_IMAGE="localhost/bootc:test"
+podman build \
+    --retry 5 \
+    --retry-delay 5s \
+    -v "$(pwd)":/code:z \
+    -t "$LOCAL_IMAGE" \
+    -f "$CONTAINERFILE" \
+    "$BOOTC_TEMPDIR"
+
+SSH_KEY=${BOOTC_TEMPDIR}/id_rsa
+ssh-keygen -f "${SSH_KEY}" -N "" -q -t rsa-sha2-256 -b 2048
+
+truncate -s 10G "${BOOTC_TEMPDIR}/disk.raw"
+
+# For test-22-logically-bound-install
+podman pull --retry 5 --retry-delay 5s quay.io/curl/curl:latest
+podman pull --retry 5 --retry-delay 5s quay.io/curl/curl-base:latest
+podman pull --retry 5 --retry-delay 5s registry.access.redhat.com/ubi9/podman:latest
+
+podman run \
+  --rm \
+  --privileged \
+  --pid=host \
+  --security-opt label=type:unconfined_t \
+  -v /var/lib/containers:/var/lib/containers \
+  -v /dev:/dev \
+  -v "$BOOTC_TEMPDIR":/output \
+  "$LOCAL_IMAGE" \
+  bootc install to-disk \
+  --filesystem "xfs" \
+  --root-ssh-authorized-keys "/output/id_rsa.pub" \
+  --karg=console=ttyS0,115200n8 \
+  --generic-image \
+  --via-loopback \
+  /output/disk.raw

tests/test.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+set -exuo pipefail
+
+# This script runs disk image with qemu-system and run tmt against this vm.
+
+BOOTC_TEMPDIR="/tmp/tmp-bootc-build"
+SSH_OPTIONS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5)
+SSH_KEY=${BOOTC_TEMPDIR}/id_rsa
+
+ARCH=$(uname -m)
+case "$ARCH" in
+"aarch64")
+  qemu-system-aarch64 \
+    -name bootc-vm \
+    -enable-kvm \
+    -machine virt \
+    -cpu host \
+    -m 2G \
+    -bios /usr/share/AAVMF/AAVMF_CODE.fd \
+    -drive file="${BOOTC_TEMPDIR}/disk.raw",if=virtio,format=raw \
+    -net nic,model=virtio \
+    -net user,hostfwd=tcp::2222-:22 \
+    -display none \
+    -daemonize
+  ;;
+"x86_64")
+  qemu-system-x86_64 \
+    -name bootc-vm \
+    -enable-kvm \
+    -cpu host \
+    -m 2G \
+    -drive file="${BOOTC_TEMPDIR}/disk.raw",if=virtio,format=raw \
+    -net nic,model=virtio \
+    -net user,hostfwd=tcp::2222-:22 \
+    -display none \
+    -daemonize
+  ;;
+*)
+  echo "Only support x86_64 and aarch64" >&2
+  exit 1
+  ;;
+esac
+
+wait_for_ssh_up() {
+  SSH_STATUS=$(ssh "${SSH_OPTIONS[@]}" -i "$SSH_KEY" -p 2222 root@"${1}" '/bin/bash -c "echo -n READY"')
+  if [[ $SSH_STATUS == READY ]]; then
+    echo 1
+  else
+    echo 0
+  fi
+}
+
+for _ in $(seq 0 30); do
+  RESULT=$(wait_for_ssh_up "localhost")
+  if [[ $RESULT == 1 ]]; then
+    echo "SSH is ready now! 🥳"
+    break
+  fi
+  sleep 10
+done
+
+# Make sure VM is ready for testing
+ssh "${SSH_OPTIONS[@]}" \
+  -i "$SSH_KEY" \
+  -p 2222 \
+  root@localhost \
+  "bootc status"
+
+# TMT will rsync tmt-* scripts to TMT_SCRIPTS_DIR=/var/lib/tmt/scripts
+tmt run --all --verbose -e TMT_SCRIPTS_DIR=/var/lib/tmt/scripts provision --how connect --guest localhost --port 2222 --user root --key "$SSH_KEY" plan --name "/tmt/plans/bootc-integration/${TMT_PLAN_NAME}"

tmt/plans/bootc-integration.fmf

@@ -0,0 +1,46 @@
+execute:
+  how: tmt
+
+/test-01-readonly:
+  summary: Execute booted readonly/nondestructive tests
+  discover:
+    how: fmf
+    test:
+        - /tmt/tests/test-01-readonly
+
+/test-20-local-upgrade:
+  summary: Execute local upgrade tests
+  discover:
+    how: fmf
+    test:
+        - /tmt/tests/test-20-local-upgrade
+
+/test-21-logically-bound-switch:
+  summary: Execute logically bound images tests for switching images
+  discover:
+    how: fmf
+    test:
+        - /tmt/tests/test-21-logically-bound-switch
+
+/test-22-logically-bound-install:
+  summary: Execute logically bound images tests for switching images
+  environment+:
+    LBI: enabled
+  discover:
+    how: fmf
+    test:
+        - /tmt/tests/test-22-logically-bound-install
+
+/test-23-install-outside-container:
+  summary: Execute tests for installing outside of a container
+  discover:
+    how: fmf
+    test:
+        - /tmt/tests/test-23-install-outside-container
+
+/test-24-local-upgrade-reboot:
+  summary: Execute local upgrade tests with automated reboot
+  discover:
+    how: fmf
+    test:
+        - /tmt/tests/test-24-local-upgrade-reboot