ephemeral: Replace systemd.volatile=overlay with fine-grained mounts

Instead of using systemd.volatile=overlay which overlaid all of / with
a single tmpfs-backed overlayfs, set up /etc and /var separately:

- /etc: overlayfs with tmpfs upper (transient changes, lost on reboot)
- /var: real tmpfs with content copied from image (not overlayfs)

The key benefit is that /var is now a real tmpfs, allowing podman to
use overlayfs for container storage inside /var/lib/containers. With
the old approach, the nested overlayfs caused "too many levels of
symbolic links" errors.

Implementation:

The initramfs units are embedded in a CPIO archive that gets appended
to the existing initramfs. This uses the Linux kernel's ability to
concatenate multiple CPIO archives.

Services running in initramfs (before switch-root):
- bcvk-etc-overlay.service: Sets up overlay on /sysroot/etc using a
  bind-mounted lowerdir to avoid self-referential mount issues on
  older kernels. Uses index=off,metacopy=off for virtiofs compat.
- bcvk-var-ephemeral.service: Copies /sysroot/var to tmpfs and bind
  mounts it back.
- bcvk-copy-units.service: Copies bcvk-journal-stream.service to
  /run/systemd/system/ for systemd <256 compatibility. The /run tmpfs
  is preserved across switch-root via MS_MOVE.

For systemd 256+, the journal-stream unit is created via SMBIOS
credentials (systemd.extra-unit.*). For older versions like CentOS
Stream 9 (systemd 252), the unit is copied from the initramfs since
credential-based unit creation isn't supported.

The execute service target is changed from default.target to
multi-user.target with ConditionPathExists=!/etc/initrd-release to
ensure it runs after switch-root, not in the initramfs.

This is Phase 1 of issue #22, making ephemeral VMs more bootc-like.
SELinux is still disabled (selinux=0); Phase 2 will add composefs
support to enable proper SELinux labeling.

Closes: #22 (Phase 1)
Assisted-by: OpenCode (Sonnet 4)

Author: cgwalters

crates/integration-tests/src/tests/run_ephemeral.rs

@@ -299,3 +299,77 @@ fn test_run_ephemeral_instancetype_invalid() -> Result<()> {
     Ok(())
 }
 integration_test!(test_run_ephemeral_instancetype_invalid);
+
+/// Test that ephemeral VMs have the expected mount layout:
+/// - / is read-only virtiofs
+/// - /etc is overlayfs with tmpfs upper (writable)
+/// - /var is tmpfs (not overlayfs, so podman can use overlayfs inside)
+fn test_run_ephemeral_mount_layout() -> Result<()> {
+    // Check each mount point individually using findmnt
+    // Running all three at once with -J can hang on some configurations
+
+    // Check root mount
+    let output = run_bcvk(&[
+        "ephemeral",
+        "run",
+        "--rm",
+        "--label",
+        INTEGRATION_TEST_LABEL,
+        "--execute",
+        "findmnt -n -o FSTYPE,OPTIONS /",
+        &get_test_image(),
+    ])?;
+    output.assert_success("check root mount");
+    let root_line = output.stdout.trim();
+    assert!(
+        root_line.starts_with("virtiofs"),
+        "Root should be virtiofs, got: {}",
+        root_line
+    );
+    assert!(
+        root_line.contains("ro"),
+        "Root should be read-only, got: {}",
+        root_line
+    );
+
+    // Check /etc mount
+    let output = run_bcvk(&[
+        "ephemeral",
+        "run",
+        "--rm",
+        "--label",
+        INTEGRATION_TEST_LABEL,
+        "--execute",
+        "findmnt -n -o FSTYPE /etc",
+        &get_test_image(),
+    ])?;
+    output.assert_success("check /etc mount");
+    assert_eq!(
+        output.stdout.trim(),
+        "overlay",
+        "/etc should be overlay, got: {}",
+        output.stdout
+    );
+
+    // Check /var mount - should be tmpfs, NOT overlay
+    let output = run_bcvk(&[
+        "ephemeral",
+        "run",
+        "--rm",
+        "--label",
+        INTEGRATION_TEST_LABEL,
+        "--execute",
+        "findmnt -n -o FSTYPE /var",
+        &get_test_image(),
+    ])?;
+    output.assert_success("check /var mount");
+    assert_eq!(
+        output.stdout.trim(),
+        "tmpfs",
+        "/var should be tmpfs (not overlay), got: {}",
+        output.stdout
+    );
+
+    Ok(())
+}
+integration_test!(test_run_ephemeral_mount_layout);

crates/kit/src/cpio.rs

@@ -0,0 +1,221 @@
+//! Minimal CPIO archive creation for initramfs appending
+//!
+//! This module implements the "newc" CPIO format for appending files to
+//! an initramfs. The Linux kernel supports concatenating multiple CPIO
+//! archives, so we can simply append our files to an existing initramfs.
+
+use std::io::{self, BufWriter, Write};
+
+/// CPIO "newc" format magic number
+const CPIO_MAGIC: &str = "070701";
+
+/// Write a CPIO archive entry header
+fn write_header<W: Write>(
+    writer: &mut BufWriter<W>,
+    name: &str,
+    mode: u32,
+    file_size: u32,
+) -> io::Result<()> {
+    let name_with_nul = format!("{}\0", name);
+    // SAFETY: name length should fit within 32 bits
+    let namesize: u32 = name_with_nul.len().try_into().unwrap();
+
+    // newc header format: all fields are 8-char hex ASCII
+    let ino = 0u32;
+    write!(
+        writer,
+        "{CPIO_MAGIC}{ino:08x}{mode:08x}{uid:08x}{gid:08x}{nlink:08x}{mtime:08x}{filesize:08x}{devmajor:08x}{devminor:08x}{rdevmajor:08x}{rdevminor:08x}{namesize:08x}{check:08x}",
+        uid = 0u32,
+        gid = 0u32,
+        nlink = 1u32,
+        mtime = 0u32,
+        filesize = file_size,
+        devmajor = 0u32,
+        devminor = 0u32,
+        rdevmajor = 0u32,
+        rdevminor = 0u32,
+        check = 0u32,
+    )?;
+
+    // Write filename (with NUL terminator)
+    writer.write_all(name_with_nul.as_bytes())?;
+
+    // Pad to 4-byte boundary after header + filename
+    // Header is 110 bytes, so total is 110 + namesize
+    let header_plus_name = 110 + namesize;
+    let padding = (4 - (header_plus_name % 4)) % 4;
+    for _ in 0..padding {
+        writer.write_all(b"\0")?;
+    }
+
+    Ok(())
+}
+
+/// Pad output to 4-byte boundary
+fn write_data_padding<W: Write>(writer: &mut BufWriter<W>, data_len: u32) -> io::Result<()> {
+    let padding = (4 - (data_len % 4)) % 4;
+    for _ in 0..padding {
+        writer.write_all(b"\0")?;
+    }
+    Ok(())
+}
+
+/// Write a directory entry to a CPIO archive
+fn write_directory<W: Write>(writer: &mut BufWriter<W>, path: &str) -> io::Result<()> {
+    // Directory mode: 0755 + S_IFDIR (0o40000)
+    let mode = 0o40755;
+    write_header(writer, path, mode, 0)?;
+    Ok(())
+}
+
+/// Write a regular file entry to a CPIO archive
+fn write_file<W: Write>(
+    writer: &mut BufWriter<W>,
+    path: &str,
+    content: &[u8],
+    mode: u32,
+) -> io::Result<()> {
+    // Add S_IFREG (0o100000) to mode
+    let full_mode = 0o100000 | mode;
+    // SAFETY: content length should fit within 32 bits
+    let content_len: u32 = content.len().try_into().unwrap();
+    write_header(writer, path, full_mode, content_len)?;
+    writer.write_all(content)?;
+    write_data_padding(writer, content_len)?;
+    Ok(())
+}
+
+/// Write the CPIO trailer (end of archive marker)
+fn write_trailer<W: Write>(writer: &mut BufWriter<W>) -> io::Result<()> {
+    write_header(writer, "TRAILER!!!", 0, 0)?;
+    Ok(())
+}
+
+/// Create a CPIO archive with bcvk initramfs units
+///
+/// This creates a minimal CPIO archive containing:
+/// - The /etc overlay service unit (runs in initramfs)
+/// - The /var ephemeral service unit (runs in initramfs)  
+/// - The copy-units service (copies journal-stream to /sysroot/etc for systemd <256)
+/// - The journal-stream service (to be copied for systemd <256 compatibility)
+/// - Drop-in files to pull units into appropriate targets
+///
+/// On systemd v256+, the journal-stream unit is created via SMBIOS credentials.
+/// On older versions, bcvk-copy-units.service copies the embedded unit to
+/// /sysroot/etc/systemd/system/ before switch-root.
+pub fn create_initramfs_units_cpio() -> Vec<u8> {
+    let mut buf = Vec::new();
+    let mut writer = BufWriter::new(&mut buf);
+
+    // Include the initramfs service units
+    let etc_overlay_content = include_str!("units/bcvk-etc-overlay.service");
+    let var_ephemeral_content = include_str!("units/bcvk-var-ephemeral.service");
+    let copy_units_content = include_str!("units/bcvk-copy-units.service");
+
+    // Include the journal-stream service (copied to /sysroot/etc on systemd <256)
+    let journal_stream_content = include_str!("units/bcvk-journal-stream.service");
+
+    // Create directory structure
+    write_directory(&mut writer, "usr").unwrap();
+    write_directory(&mut writer, "usr/lib").unwrap();
+    write_directory(&mut writer, "usr/lib/systemd").unwrap();
+    write_directory(&mut writer, "usr/lib/systemd/system").unwrap();
+
+    // Write the initramfs service units (mode 0644)
+    write_file(
+        &mut writer,
+        "usr/lib/systemd/system/bcvk-etc-overlay.service",
+        etc_overlay_content.as_bytes(),
+        0o644,
+    )
+    .unwrap();
+
+    write_file(
+        &mut writer,
+        "usr/lib/systemd/system/bcvk-var-ephemeral.service",
+        var_ephemeral_content.as_bytes(),
+        0o644,
+    )
+    .unwrap();
+
+    write_file(
+        &mut writer,
+        "usr/lib/systemd/system/bcvk-copy-units.service",
+        copy_units_content.as_bytes(),
+        0o644,
+    )
+    .unwrap();
+
+    // Write the journal-stream service (will be copied to /sysroot/etc on systemd <256)
+    write_file(
+        &mut writer,
+        "usr/lib/systemd/system/bcvk-journal-stream.service",
+        journal_stream_content.as_bytes(),
+        0o644,
+    )
+    .unwrap();
+
+    // Create drop-in directories and files to pull units into initrd-fs.target
+    write_directory(&mut writer, "usr/lib/systemd/system/initrd-fs.target.d").unwrap();
+
+    let etc_dropin = "[Unit]\nWants=bcvk-etc-overlay.service\n";
+    write_file(
+        &mut writer,
+        "usr/lib/systemd/system/initrd-fs.target.d/bcvk-etc-overlay.conf",
+        etc_dropin.as_bytes(),
+        0o644,
+    )
+    .unwrap();
+
+    let var_dropin = "[Unit]\nWants=bcvk-var-ephemeral.service\n";
+    write_file(
+        &mut writer,
+        "usr/lib/systemd/system/initrd-fs.target.d/bcvk-var-ephemeral.conf",
+        var_dropin.as_bytes(),
+        0o644,
+    )
+    .unwrap();
+
+    let copy_dropin = "[Unit]\nWants=bcvk-copy-units.service\n";
+    write_file(
+        &mut writer,
+        "usr/lib/systemd/system/initrd-fs.target.d/bcvk-copy-units.conf",
+        copy_dropin.as_bytes(),
+        0o644,
+    )
+    .unwrap();
+
+    // Write trailer
+    write_trailer(&mut writer).unwrap();
+
+    // Flush and return the buffer
+    writer.into_inner().unwrap();
+    buf
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_create_initramfs_units_cpio() {
+        let cpio = create_initramfs_units_cpio();
+
+        // Should start with CPIO magic
+        assert!(cpio.starts_with(CPIO_MAGIC.as_bytes()));
+
+        let cpio_str = std::str::from_utf8(&cpio).unwrap();
+
+        // Should contain the embedded service units
+        assert!(cpio_str.contains("bcvk-etc-overlay.service"));
+        assert!(cpio_str.contains("bcvk-var-ephemeral.service"));
+        assert!(cpio_str.contains("bcvk-copy-units.service"));
+        assert!(cpio_str.contains("bcvk-journal-stream.service"));
+
+        // Should contain the drop-in configs
+        assert!(cpio_str.contains("initrd-fs.target.d"));
+
+        // Should end with TRAILER!!!
+        assert!(cpio_str.contains("TRAILER!!!"));
+    }
+}

crates/kit/src/credentials.rs

@@ -173,6 +173,10 @@ pub fn storage_opts_tmpfiles_d_lines() -> String {
         "f /etc/systemd/system.conf.d/90-bcvk-storage.conf 0644 root root - [Manager]\\nDefaultEnvironment=STORAGE_OPTS=additionalimagestore=/run/host-container-storage\n"
     ).to_string()
 }
+// Note: The /etc overlay and /var ephemeral units are now embedded directly in the
+// initramfs CPIO archive (see cpio.rs) rather than being injected via SMBIOS credentials.
+// This ensures they work on systemd <256 where credential import happens too late for
+// generators to process the credentials.
 
 /// Generate SMBIOS credential string for root SSH access
 ///

crates/kit/src/lib.rs

@@ -1,4 +1,5 @@
 //! bcvk library - exposes internal modules for testing
 
+pub mod cpio;
 pub mod qemu_img;
 pub mod xml_utils;

crates/kit/src/main.rs

@@ -10,6 +10,7 @@ mod cache_metadata;
 mod cli_json;
 mod common_opts;
 mod container_entrypoint;
+mod cpio;
 mod credentials;
 mod domain_list;
 mod ephemeral;