ephemeral: Replace systemd.volatile=overlay with fine-grained mounts

Instead of using systemd.volatile=overlay which overlaid all of / with
a single tmpfs-backed overlayfs, set up /etc and /var separately:

- /etc: overlayfs with tmpfs upper (transient changes, lost on reboot)
- /var: real tmpfs with content copied from image (not overlayfs)

The key benefit is that /var is now a real tmpfs, allowing podman to
use overlayfs for container storage inside /var/lib/containers. With
the old approach, the nested overlayfs caused "too many levels of
symbolic links" errors.

Implementation uses systemd credentials to inject units that run in the
initramfs before switch-root:
- bcvk-etc-overlay.service: overlay on /sysroot/etc with index=off,metacopy=off
  to avoid virtiofs contention; ordered after initrd-parse-etc.service
- bcvk-var-ephemeral.service: copies /sysroot/var to tmpfs and bind mounts

Both units use ConditionPathExists=/etc/initrd-release to only run in
the initramfs context.

The execute service target is changed from default.target to
multi-user.target with ConditionPathExists=!/etc/initrd-release to
ensure it runs after switch-root, not in the initramfs.

This is Phase 1 of issue #22, making ephemeral VMs more bootc-like.
SELinux is still disabled (selinux=0); Phase 2 will add composefs
support to enable proper SELinux labeling.

Closes: #22 (Phase 1)
Assisted-by: OpenCode (Sonnet 4)

Author: cgwalters

crates/integration-tests/src/tests/run_ephemeral.rs

@@ -299,3 +299,77 @@ fn test_run_ephemeral_instancetype_invalid() -> Result<()> {
     Ok(())
 }
 integration_test!(test_run_ephemeral_instancetype_invalid);
+
+/// Test that ephemeral VMs have the expected mount layout:
+/// - / is read-only virtiofs
+/// - /etc is overlayfs with tmpfs upper (writable)
+/// - /var is tmpfs (not overlayfs, so podman can use overlayfs inside)
+fn test_run_ephemeral_mount_layout() -> Result<()> {
+    // Check each mount point individually using findmnt
+    // Running all three at once with -J can hang on some configurations
+
+    // Check root mount
+    let output = run_bcvk(&[
+        "ephemeral",
+        "run",
+        "--rm",
+        "--label",
+        INTEGRATION_TEST_LABEL,
+        "--execute",
+        "findmnt -n -o FSTYPE,OPTIONS /",
+        &get_test_image(),
+    ])?;
+    output.assert_success("check root mount");
+    let root_line = output.stdout.trim();
+    assert!(
+        root_line.starts_with("virtiofs"),
+        "Root should be virtiofs, got: {}",
+        root_line
+    );
+    assert!(
+        root_line.contains("ro"),
+        "Root should be read-only, got: {}",
+        root_line
+    );
+
+    // Check /etc mount
+    let output = run_bcvk(&[
+        "ephemeral",
+        "run",
+        "--rm",
+        "--label",
+        INTEGRATION_TEST_LABEL,
+        "--execute",
+        "findmnt -n -o FSTYPE /etc",
+        &get_test_image(),
+    ])?;
+    output.assert_success("check /etc mount");
+    assert_eq!(
+        output.stdout.trim(),
+        "overlay",
+        "/etc should be overlay, got: {}",
+        output.stdout
+    );
+
+    // Check /var mount - should be tmpfs, NOT overlay
+    let output = run_bcvk(&[
+        "ephemeral",
+        "run",
+        "--rm",
+        "--label",
+        INTEGRATION_TEST_LABEL,
+        "--execute",
+        "findmnt -n -o FSTYPE /var",
+        &get_test_image(),
+    ])?;
+    output.assert_success("check /var mount");
+    assert_eq!(
+        output.stdout.trim(),
+        "tmpfs",
+        "/var should be tmpfs (not overlay), got: {}",
+        output.stdout
+    );
+
+    Ok(())
+}
+integration_test!(test_run_ephemeral_mount_layout);

crates/kit/src/credentials.rs

@@ -174,6 +174,60 @@ pub fn storage_opts_tmpfiles_d_lines() -> String {
     ).to_string()
 }
 
+/// Generate SMBIOS credentials for transient /etc overlay
+///
+/// Creates a systemd service that runs in the initramfs to set up /etc as an
+/// overlayfs with tmpfs upper. Changes are stored in tmpfs (lost on reboot),
+/// while the base /etc content comes from the container image.
+///
+/// Uses a service instead of a mount unit because systemd mount units can
+/// hang when mounting overlayfs on virtiofs subdirectories.
+pub fn smbios_creds_for_etc_overlay() -> Vec<String> {
+    // Systemd unit that sets up /etc overlay in initramfs
+    let unit_content = include_str!("units/bcvk-etc-overlay.service");
+    let encoded_unit = data_encoding::BASE64.encode(unit_content.as_bytes());
+    let unit_cred = format!(
+        "io.systemd.credential.binary:systemd.extra-unit.bcvk-etc-overlay.service={encoded_unit}"
+    );
+
+    // Create dropin for initrd-fs.target to pull in our unit
+    let dropin_content = "[Unit]\nWants=bcvk-etc-overlay.service\n";
+    let encoded_dropin = data_encoding::BASE64.encode(dropin_content.as_bytes());
+    let dropin_cred = format!(
+        "io.systemd.credential.binary:systemd.unit-dropin.initrd-fs.target~bcvk-etc-overlay={encoded_dropin}"
+    );
+
+    vec![unit_cred, dropin_cred]
+}
+
+/// Generate SMBIOS credentials for ephemeral /var with image content
+///
+/// Creates a systemd service that runs in the initramfs to:
+/// 1. Creates /run/var-ephemeral directory  
+/// 2. Copies all content from /sysroot/var to /run/var-ephemeral (one-time)
+/// 3. Bind mounts /run/var-ephemeral over /sysroot/var
+///
+/// This runs before switch-root so /var is ready as a real tmpfs (not overlayfs).
+/// This allows podman to use overlayfs inside /var/lib/containers.
+pub fn smbios_creds_for_var_ephemeral() -> Vec<String> {
+    // Systemd unit that sets up ephemeral /var in initramfs
+    // Must run after sysroot is mounted but before switch-root
+    let unit_content = include_str!("units/bcvk-var-ephemeral.service");
+    let encoded_unit = data_encoding::BASE64.encode(unit_content.as_bytes());
+    let unit_cred = format!(
+        "io.systemd.credential.binary:systemd.extra-unit.bcvk-var-ephemeral.service={encoded_unit}"
+    );
+
+    // Create dropin for initrd-fs.target to pull in our unit
+    let dropin_content = "[Unit]\nWants=bcvk-var-ephemeral.service\n";
+    let encoded_dropin = data_encoding::BASE64.encode(dropin_content.as_bytes());
+    let dropin_cred = format!(
+        "io.systemd.credential.binary:systemd.unit-dropin.initrd-fs.target~bcvk-var-ephemeral={encoded_dropin}"
+    );
+
+    vec![unit_cred, dropin_cred]
+}
+
 /// Generate SMBIOS credential string for root SSH access
 ///
 /// Creates a systemd credential for QEMU's SMBIOS interface. Preferred method

crates/kit/src/run_ephemeral.rs

@@ -1064,6 +1064,13 @@ pub(crate) async fn run_impl(opts: RunEphemeralOpts) -> Result<()> {
         );
     }
 
+    // Add /etc overlay and /var ephemeral credentials
+    // These replace systemd.volatile=overlay with more fine-grained control:
+    // - /etc: overlay with tmpfs upper (writable, changes lost on reboot)
+    // - /var: real tmpfs (allows podman to use overlayfs inside)
+    mount_unit_smbios_creds.extend(crate::credentials::smbios_creds_for_etc_overlay());
+    mount_unit_smbios_creds.extend(crate::credentials::smbios_creds_for_var_ephemeral());
+
     // Handle --execute: pipes will be created when adding to qemu_config later
     // No need to create files anymore as we're using pipes
 
@@ -1110,6 +1117,8 @@ WantedBy=sysinit.target
 Description=Execute Script Service
 Requires=dev-virtio\x2dports-execute.device
 After=dev-virtio\x2dports-execute.device
+# Ensure we only run after switch-root in the real root filesystem
+ConditionPathExists=!/etc/initrd-release
 
 [Service]
 Type=oneshot
@@ -1127,6 +1136,8 @@ Description=Execute Script Service Completion
 After=bootc-execute.service
 Requires=dev-virtio\x2dports-executestatus.device
 After=dev-virtio\x2dports-executestatus.device
+# Ensure we only run after switch-root in the real root filesystem
+ConditionPathExists=!/etc/initrd-release
 
 [Service]
 Type=oneshot
@@ -1148,12 +1159,14 @@ StandardOutput=file:/dev/virtio-ports/executestatus
             );
             mount_unit_smbios_creds.push(finish_cred);
 
-            // Create dropin for default.target to enable execute services
+            // Create dropin for multi-user.target to enable execute services
+            // Using multi-user.target instead of default.target ensures these only run
+            // after switch-root in the real root filesystem (not in initramfs)
             let execute_dropin =
                 "[Unit]\nWants=bootc-execute.service bootc-execute-finish.service\n";
             let encoded_dropin = data_encoding::BASE64.encode(execute_dropin.as_bytes());
             let dropin_cred = format!(
-                "io.systemd.credential.binary:systemd.unit-dropin.default.target~bcvk-execute={encoded_dropin}"
+                "io.systemd.credential.binary:systemd.unit-dropin.multi-user.target~bcvk-execute={encoded_dropin}"
             );
             mount_unit_smbios_creds.push(dropin_cred);
             debug!("Generated SMBIOS credentials for execute units");
@@ -1199,10 +1212,10 @@ StandardOutput=file:/dev/virtio-ports/executestatus
         // At the core we boot from the mounted container's root,
         "rootfstype=virtiofs",
         "root=rootfs",
-        // But read-only, with an overlayfs in the VM backed
-        // by tmpfs
+        // But read-only. We set up /etc overlay and /var copyup via
+        // systemd credentials rather than systemd.volatile=overlay
+        // to have more control over individual directories.
         "rootflags=ro",
-        "systemd.volatile=overlay",
         // This avoids having journald interact with the rootfs
         // at all, which lessens the I/O traffic for virtiofs
         "systemd.journald.storage=volatile",

crates/kit/src/units/bcvk-etc-overlay.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Setup ephemeral /etc overlay
+DefaultDependencies=no
+ConditionPathExists=/etc/initrd-release
+Before=initrd-fs.target
+# Must run after initrd-parse-etc.service which scans /sysroot/etc/fstab
+# to avoid virtiofs contention that causes mount to hang
+After=sysroot.mount initrd-parse-etc.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+TimeoutStartSec=30
+ExecStart=/usr/bin/mkdir -p /run/etc-upper /run/etc-work
+# Use index=off,metacopy=off to avoid extended attr operations on virtiofs
+ExecStart=/usr/bin/mount -t overlay overlay -o lowerdir=/sysroot/etc,upperdir=/run/etc-upper,workdir=/run/etc-work,index=off,metacopy=off /sysroot/etc

crates/kit/src/units/bcvk-var-ephemeral.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Setup ephemeral /var from image content
+DefaultDependencies=no
+ConditionPathExists=/etc/initrd-release
+Before=initrd-fs.target
+After=sysroot.mount
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/bin/mkdir -p /run/var-ephemeral
+ExecStart=/usr/bin/cp -a /sysroot/var/. /run/var-ephemeral/
+ExecStart=/usr/bin/mount --bind /run/var-ephemeral /sysroot/var