install: Honor composefs.enabled=verity

Key off the ostree prepare-root config to require fsverity
on all objects.

As part of this:

- Add a dependency on composefs-rs just for the fsverity querying
  APIs, and as prep for further integration.
- Add `bootc internals fsck`, which verifies the expected
  fsverity state.

Signed-off-by: Colin Walters <[email protected]>

Author: cgwalters

.github/workflows/ci.yml

@@ -75,12 +75,21 @@ jobs:
         uses: actions/checkout@v4
       - name: Free up disk space on runner
         run: sudo ./ci/clean-gha-runner.sh
+      - name: Enable fsverity for /
+        run: sudo tune2fs -O verity $(findmnt -vno SOURCE /)
+      - name: Install utils
+        run: sudo apt -y install fsverity
       - name: Integration tests
         run: |
           set -xeu
+          # Build images to test; TODO investigate doing single container builds
+          # via GHA and pushing to a temporary registry to share among workflows?
           sudo podman build -t localhost/bootc -f hack/Containerfile .
+          sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity
+
           export CARGO_INCREMENTAL=0  # because we aren't caching the test runner bits
           cargo build --release -p tests-integration
+
           df -h /
           sudo install -m 0755 target/release/tests-integration /usr/bin/bootc-integration-tests
           rm target -rf
@@ -90,8 +99,16 @@ jobs:
             -v /run/dbus:/run/dbus -v /run/systemd:/run/systemd localhost/bootc /src/ostree-ext/ci/priv-integration.sh
           # Nondestructive but privileged tests
           sudo bootc-integration-tests host-privileged localhost/bootc
-          # Finally the install-alongside suite
+          # Install tests
           sudo bootc-integration-tests install-alongside localhost/bootc
+
+          # And the fsverity case
+          sudo podman run --privileged --pid=host localhost/bootc-fsverity bootc install to-existing-root --stateroot=other \
+            --acknowledge-destructive --skip-fetch-check
+          # Crude cross check
+          sudo find /ostree/repo/objects -name '*.file' -type f | while read f; do
+            sudo fsverity measure $f >/dev/null
+          done
   docs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'documentation') }}
     runs-on: ubuntu-latest

Cargo.lock

@@ -0,0 +1,14 @@
+# Enable fsverity at install time
+FROM localhost/bootc
+RUN <<EORUN
+set -xeuo pipefail
+cat > /usr/lib/ostree/prepare-root.conf <<EOF
+[composefs]
+enabled = verity
+EOF
+cat > /usr/lib/bootc/install/90-ext4.toml <<EOF
+[install.filesystem.root]
+type = "ext4"
+EOF
+bootc container lint
+EORUN

ci/Containerfile.install-fsverity

@@ -26,7 +26,6 @@ chrono = { workspace = true, features = ["serde"] }
 clap = { workspace = true, features = ["derive","cargo"] }
 clap_mangen = { workspace = true, optional = true }
 #composefs = "0.2.0"
-composefs = { git = "https://github.com/containers/composefs-rs", rev = "55ae2e9ba72f6afda4887d746e6b98f0a1875ac4" }
 cap-std-ext = { workspace = true, features = ["fs_utf8"] }
 hex = { workspace = true }
 fn-error-context = { workspace = true }

lib/Cargo.toml

@@ -13,10 +13,10 @@ use cap_std_ext::cap_std;
 use cap_std_ext::cap_std::fs::Dir;
 use clap::Parser;
 use clap::ValueEnum;
-use composefs::fsverity;
 use fn_error_context::context;
 use ostree::gio;
 use ostree_container::store::PrepareResult;
+use ostree_ext::composefs::fsverity;
 use ostree_ext::container as ostree_container;
 use ostree_ext::container_utils::ostree_booted;
 use ostree_ext::keyfileext::KeyFileExt;

lib/src/cli.rs

@@ -6,17 +6,26 @@
 // Unfortunately needed here to work with linkme
 #![allow(unsafe_code)]
 
+use std::fmt::Write as _;
 use std::future::Future;
 use std::pin::Pin;
 use std::process::Command;
 
+use bootc_utils::iterator_split_nonempty_rest_count;
+use camino::Utf8PathBuf;
 use cap_std::fs::{Dir, MetadataExt as _};
 use cap_std_ext::cap_std;
 use cap_std_ext::dirext::CapStdExtDirExt;
+use fn_error_context::context;
 use linkme::distributed_slice;
+use ostree_ext::ostree_prepareroot::Tristate;
+use ostree_ext::{composefs, ostree};
+use serde::{Deserialize, Serialize};
 
 use crate::store::Storage;
 
+use std::os::fd::AsFd;
+
 /// A lint check has failed.
 #[derive(thiserror::Error, Debug)]
 struct FsckError(String);
@@ -112,6 +121,153 @@ fn check_resolvconf(storage: &Storage) -> FsckResult {
     fsck_ok()
 }
 
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) enum VerityState {
+    Enabled,
+    Disabled,
+    Inconsistent((u64, u64)),
+}
+
+#[derive(Debug, Default)]
+struct ObjectsVerityState {
+    /// Count of objects with fsverity
+    enabled: u64,
+    /// Count of objects without fsverity
+    disabled: u64,
+    /// Objects which should have fsverity but do not
+    missing: Vec<String>,
+}
+
+/// Check the fsverity state of all regular files in this object directory.
+#[context("Computing verity state")]
+fn verity_state_of_objects(
+    d: &Dir,
+    prefix: &str,
+    expected: bool,
+) -> anyhow::Result<ObjectsVerityState> {
+    let mut enabled = 0;
+    let mut disabled = 0;
+    let mut missing = Vec::new();
+    for ent in d.entries()? {
+        let ent = ent?;
+        if !ent.file_type()?.is_file() {
+            continue;
+        }
+        let name = ent.file_name();
+        let name = name
+            .into_string()
+            .map(Utf8PathBuf::from)
+            .map_err(|_| anyhow::anyhow!("Invalid UTF-8"))?;
+        let Some("file") = name.extension() else {
+            continue;
+        };
+        let f = d.open(&name)?;
+        let r: Option<composefs::fsverity::Sha256HashValue> =
+            composefs::fsverity::ioctl::fs_ioc_measure_verity(f.as_fd())?;
+        drop(f);
+        if r.is_some() {
+            enabled += 1;
+        } else {
+            disabled += 1;
+            if expected {
+                missing.push(format!("{prefix}{name}"));
+            }
+        }
+    }
+    let r = ObjectsVerityState {
+        enabled,
+        disabled,
+        missing,
+    };
+    Ok(r)
+}
+
+async fn verity_state_of_all_objects(
+    repo: &ostree::Repo,
+    expected: bool,
+) -> anyhow::Result<ObjectsVerityState> {
+    // Limit concurrency here
+    const MAX_CONCURRENT: usize = 3;
+
+    let repodir = Dir::reopen_dir(&repo.dfd_borrow())?;
+
+    // It's convenient here to reuse tokio's spawn_blocking as a threadpool basically.
+    let mut joinset = tokio::task::JoinSet::new();
+    let mut results = Vec::new();
+
+    for ent in repodir.read_dir("objects")? {
+        // Block here if the queue is full
+        while joinset.len() >= MAX_CONCURRENT {
+            results.push(joinset.join_next().await.unwrap()??);
+        }
+        let ent = ent?;
+        if !ent.file_type()?.is_dir() {
+            continue;
+        }
+        let name = ent.file_name();
+        let name = name
+            .into_string()
+            .map(Utf8PathBuf::from)
+            .map_err(|_| anyhow::anyhow!("Invalid UTF-8"))?;
+
+        let objdir = ent.open_dir()?;
+        let expected = expected.clone();
+        joinset.spawn_blocking(move || verity_state_of_objects(&objdir, name.as_str(), expected));
+    }
+
+    // Drain the remaining tasks.
+    while let Some(output) = joinset.join_next().await {
+        results.push(output??);
+    }
+    // Fold the results.
+    let r = results
+        .into_iter()
+        .fold(ObjectsVerityState::default(), |mut acc, v| {
+            acc.enabled += v.enabled;
+            acc.disabled += v.disabled;
+            acc.missing.extend(v.missing);
+            acc
+        });
+    Ok(r)
+}
+
+#[distributed_slice(FSCK_CHECKS)]
+static CHECK_FSVERITY: FsckCheck =
+    FsckCheck::new("fsverity", 10, FsckFnImpl::Async(check_fsverity));
+fn check_fsverity(storage: &Storage) -> Pin<Box<dyn Future<Output = FsckResult> + '_>> {
+    Box::pin(check_fsverity_inner(storage))
+}
+
+async fn check_fsverity_inner(storage: &Storage) -> FsckResult {
+    let repo = &storage.repo();
+    let verity_state = ostree_ext::fsverity::is_verity_enabled(repo)?;
+    tracing::debug!(
+        "verity: expected={:?} found={:?}",
+        verity_state.desired,
+        verity_state.enabled
+    );
+
+    let verity_found_state =
+        verity_state_of_all_objects(&storage.repo(), verity_state.desired == Tristate::Enabled)
+            .await?;
+    let Some((missing, rest)) =
+        iterator_split_nonempty_rest_count(verity_found_state.missing.iter(), 5)
+    else {
+        return fsck_ok();
+    };
+    let mut err = String::from("fsverity enabled, but objects without fsverity:\n");
+    for obj in missing {
+        // SAFETY: Writing into a String
+        writeln!(err, "  {obj}").unwrap();
+    }
+    if rest > 0 {
+        // SAFETY: Writing into a String
+        writeln!(err, "  ...and {rest} more").unwrap();
+    }
+    fsck_err(err)
+}
+
 pub(crate) async fn fsck(storage: &Storage, mut output: impl std::io::Write) -> anyhow::Result<()> {
     let mut checks = FSCK_CHECKS.static_slice().iter().collect::<Vec<_>>();
     checks.sort_by(|a, b| a.ordering.cmp(&b.ordering));

lib/src/fsck.rs

@@ -40,6 +40,7 @@ use fn_error_context::context;
 use ostree::gio;
 use ostree_ext::oci_spec;
 use ostree_ext::ostree;
+use ostree_ext::ostree_prepareroot::{ComposefsState, Tristate};
 use ostree_ext::prelude::Cast;
 use ostree_ext::sysroot::SysrootLock;
 use ostree_ext::{container as ostree_container, ostree_prepareroot};
@@ -77,6 +78,15 @@ const SELINUXFS: &str = "/sys/fs/selinux";
 const EFIVARFS: &str = "/sys/firmware/efi/efivars";
 pub(crate) const ARCH_USES_EFI: bool = cfg!(any(target_arch = "x86_64", target_arch = "aarch64"));
 
+const DEFAULT_REPO_CONFIG: &[(&str, &str)] = &[
+    // Default to avoiding grub2-mkconfig etc.
+    ("sysroot.bootloader", "none"),
+    // Always flip this one on because we need to support alongside installs
+    // to systems without a separate boot partition.
+    ("sysroot.bootprefix", "true"),
+    ("sysroot.readonly", "true"),
+];
+
 /// Kernel argument used to specify we want the rootfs mounted read-write by default
 const RW_KARG: &str = "rw";
 
@@ -638,14 +648,7 @@ async fn initialize_ostree_root(state: &State, root_setup: &RootSetup) -> Result
         crate::lsm::ensure_dir_labeled(rootfs_dir, "boot", None, 0o755.into(), sepolicy)?;
     }
 
-    for (k, v) in [
-        // Default to avoiding grub2-mkconfig etc.
-        ("sysroot.bootloader", "none"),
-        // Always flip this one on because we need to support alongside installs
-        // to systems without a separate boot partition.
-        ("sysroot.bootprefix", "true"),
-        ("sysroot.readonly", "true"),
-    ] {
+    for (k, v) in DEFAULT_REPO_CONFIG.iter() {
         Command::new("ostree")
             .args(["config", "--repo", "ostree/repo", "set", k, v])
             .cwd_dir(rootfs_dir.try_clone()?)
@@ -657,6 +660,19 @@ async fn initialize_ostree_root(state: &State, root_setup: &RootSetup) -> Result
         ostree::Sysroot::new(Some(&gio::File::for_path(path)))
     };
     sysroot.load(cancellable)?;
+    let repo = &sysroot.repo();
+
+    let repo_verity_state = ostree_ext::fsverity::is_verity_enabled(&repo)?;
+    let prepare_root_composefs = state
+        .prepareroot_config
+        .get("composefs.enabled")
+        .map(|v| ComposefsState::from_str(&v))
+        .transpose()?
+        .unwrap_or(ComposefsState::default());
+    if prepare_root_composefs.requires_fsverity() || repo_verity_state.desired == Tristate::Enabled
+    {
+        ostree_ext::fsverity::ensure_verity(repo).await?;
+    }
 
     let stateroot_exists = rootfs_dir.try_exists(format!("ostree/deploy/{stateroot}"))?;
     ensure!(