feat(repx-rs): add bwrap overlay fallback for kernels < 5.11 (#10)

Author: hakan-demirli

.github/workflows/main.yml

@@ -64,6 +64,7 @@ jobs:
           - e2e-mount-paths-docker
           - incremental-sync
           - large_lab
+          - e2e-bwrap-overlay-fallback
     steps:
       - uses: actions/checkout@v4
       - uses: DeterminateSystems/nix-installer-action@v11

Cargo.lock

@@ -12,3 +12,4 @@ nix = { workspace = true, features = ["fs"] }
 tempfile = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+chrono = { workspace = true }

crates/repx-executor/Cargo.toml

@@ -129,6 +129,14 @@ impl<'a> RuntimeContext<'a> {
         }
     }
 
+    pub fn get_capabilities_cache_dir(&self) -> PathBuf {
+        if let Some(local) = &self.request.node_local_path {
+            local.join("repx").join("cache").join("capabilities")
+        } else {
+            self.request.base_path.join("cache").join("capabilities")
+        }
+    }
+
     pub fn calculate_restricted_path(
         &self,
         required_system_binaries: &[&str],

crates/repx-executor/src/context.rs

@@ -2,11 +2,19 @@ use crate::context::RuntimeContext;
 use crate::error::{ExecutorError, Result};
 use crate::ExecutionRequest;
 use nix::fcntl::{Flock, FlockArg};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use std::path::{Path, PathBuf};
 use std::process::Stdio;
 use tokio::process::Command as TokioCommand;
 
+const EXCLUDED_ROOTFS_DIRS: &[&str] = &["dev", "proc", "tmp"];
+
+#[derive(Debug, Serialize, Deserialize)]
+struct OverlayCapabilityCache {
+    tmp_overlay_supported: bool,
+    checked_at: String,
+}
+
 pub struct BwrapRuntime;
 
 const EXCLUDED_HOST_DIRS: &[&str] = &["dev", "proc", "sys", "nix"];
@@ -212,7 +220,20 @@ impl BwrapRuntime {
         Ok(union_dir)
     }
 
-    pub async fn check_overlay_support(ctx: &RuntimeContext<'_>, lower_dir: &Path) -> bool {
+    pub async fn check_overlay_support(ctx: &RuntimeContext<'_>, _lower_dir: &Path) -> bool {
+        let cache_dir = ctx.get_capabilities_cache_dir();
+        let cache_file = cache_dir.join("overlay_support.json");
+
+        if let Ok(content) = tokio::fs::read_to_string(&cache_file).await {
+            if let Ok(cached) = serde_json::from_str::<OverlayCapabilityCache>(&content) {
+                return cached.tmp_overlay_supported;
+            }
+        }
+
+        Self::run_overlay_check(ctx).await
+    }
+
+    async fn run_overlay_check(ctx: &RuntimeContext<'_>) -> bool {
         let bwrap_path = match ctx.get_host_tool_path("bwrap") {
             Ok(p) => p,
             Err(_) => return false,
@@ -234,10 +255,12 @@ impl BwrapRuntime {
         let upper = start_path.join("upper");
         let work = start_path.join("work");
         let merged = start_path.join("merged");
+        let lower = start_path.join("lower");
 
         if tokio::fs::create_dir(&upper).await.is_err()
             || tokio::fs::create_dir(&work).await.is_err()
             || tokio::fs::create_dir(&merged).await.is_err()
+            || tokio::fs::create_dir(&lower).await.is_err()
         {
             return false;
         }
@@ -249,7 +272,7 @@ impl BwrapRuntime {
             .arg("/")
             .arg("/")
             .arg("--overlay-src")
-            .arg(lower_dir)
+            .arg(&lower)
             .arg("--overlay")
             .arg(&upper)
             .arg(&work)
@@ -265,6 +288,103 @@ impl BwrapRuntime {
         }
     }
 
+    pub async fn check_tmp_overlay_support(ctx: &RuntimeContext<'_>, rootfs_path: &Path) -> bool {
+        let cache_dir = ctx.get_capabilities_cache_dir();
+        let cache_file = cache_dir.join("overlay_support.json");
+
+        if let Ok(content) = tokio::fs::read_to_string(&cache_file).await {
+            if let Ok(cached) = serde_json::from_str::<OverlayCapabilityCache>(&content) {
+                tracing::debug!(
+                    "Using cached overlay support result: supported={}",
+                    cached.tmp_overlay_supported
+                );
+                return cached.tmp_overlay_supported;
+            }
+        }
+
+        let supported = Self::run_tmp_overlay_check(ctx, rootfs_path).await;
+
+        if let Err(e) = tokio::fs::create_dir_all(&cache_dir).await {
+            tracing::debug!("Failed to create capabilities cache dir: {}", e);
+        } else {
+            let cache_entry = OverlayCapabilityCache {
+                tmp_overlay_supported: supported,
+                checked_at: chrono::Utc::now().to_rfc3339(),
+            };
+            if let Ok(json) = serde_json::to_string_pretty(&cache_entry) {
+                if let Err(e) = tokio::fs::write(&cache_file, json).await {
+                    tracing::debug!("Failed to write overlay capability cache: {}", e);
+                }
+            }
+        }
+
+        supported
+    }
+
+    async fn run_tmp_overlay_check(ctx: &RuntimeContext<'_>, rootfs_path: &Path) -> bool {
+        let bwrap_path = match ctx.get_host_tool_path("bwrap") {
+            Ok(p) => p,
+            Err(_) => return false,
+        };
+
+        let temp_base = ctx.get_temp_path();
+        let temp_dir = match tempfile::Builder::new()
+            .prefix(".repx-tmp-overlay-check-")
+            .tempdir_in(&temp_base)
+        {
+            Ok(t) => t,
+            Err(e) => {
+                tracing::debug!("Failed to create temp dir for tmp-overlay check: {}", e);
+                return false;
+            }
+        };
+
+        let test_lower = if rootfs_path.join("bin").exists() {
+            rootfs_path.join("bin")
+        } else if rootfs_path.join("etc").exists() {
+            rootfs_path.join("etc")
+        } else {
+            rootfs_path.to_path_buf()
+        };
+
+        let test_mount_point = temp_dir.path().join("test");
+        if tokio::fs::create_dir(&test_mount_point).await.is_err() {
+            return false;
+        }
+
+        let mut cmd = TokioCommand::new(&bwrap_path);
+
+        cmd.arg("--unshare-user")
+            .arg("--dev-bind")
+            .arg("/")
+            .arg("/")
+            .arg("--overlay-src")
+            .arg(&test_lower)
+            .arg("--tmp-overlay")
+            .arg(&test_mount_point)
+            .arg("true");
+
+        ctx.restrict_command_environment(&mut cmd, &[]);
+        cmd.stdout(Stdio::null()).stderr(Stdio::null());
+
+        match cmd.status().await {
+            Ok(status) => {
+                let supported = status.success();
+                if !supported {
+                    tracing::debug!(
+                        "tmp-overlay check failed for {:?} - kernel may not support userxattr overlay",
+                        test_lower
+                    );
+                }
+                supported
+            }
+            Err(e) => {
+                tracing::debug!("tmp-overlay check command failed: {}", e);
+                false
+            }
+        }
+    }
+
     pub async fn build_command(
         ctx: &RuntimeContext<'_>,
         rootfs_path: &Path,
@@ -290,12 +410,25 @@ impl BwrapRuntime {
         } else {
             cmd.arg("--unshare-all")
                 .arg("--hostname")
-                .arg("repx-container")
-                .arg("--overlay-src")
-                .arg(rootfs_path)
-                .arg("--tmp-overlay")
-                .arg("/")
-                .arg("--dev")
+                .arg("repx-container");
+
+            let overlay_supported = Self::check_tmp_overlay_support(ctx, rootfs_path).await;
+
+            if overlay_supported {
+                cmd.arg("--overlay-src")
+                    .arg(rootfs_path)
+                    .arg("--tmp-overlay")
+                    .arg("/");
+            } else {
+                tracing::info!(
+                    "Overlay filesystem not supported on target (kernel may lack userxattr support). \
+                     Using read-only bind mounts for rootfs."
+                );
+
+                Self::configure_readonly_rootfs_mounts(&mut cmd, rootfs_path).await?;
+            }
+
+            cmd.arg("--dev")
                 .arg("/dev")
                 .arg("--proc")
                 .arg("/proc")
@@ -497,4 +630,35 @@ impl BwrapRuntime {
 
         Ok(())
     }
+
+    async fn configure_readonly_rootfs_mounts(
+        cmd: &mut TokioCommand,
+        rootfs_path: &Path,
+    ) -> Result<()> {
+        let entries = match std::fs::read_dir(rootfs_path) {
+            Ok(e) => e,
+            Err(e) => {
+                return Err(ExecutorError::Io(std::io::Error::new(
+                    std::io::ErrorKind::NotFound,
+                    format!("Failed to read rootfs directory {:?}: {}", rootfs_path, e),
+                )));
+            }
+        };
+
+        for entry in entries.flatten() {
+            let file_name = entry.file_name();
+            let file_name_str = file_name.to_string_lossy();
+
+            if EXCLUDED_ROOTFS_DIRS.contains(&file_name_str.as_ref()) {
+                continue;
+            }
+
+            let source_path = entry.path();
+            let target_path = format!("/{}", file_name_str);
+
+            cmd.arg("--ro-bind").arg(&source_path).arg(&target_path);
+        }
+
+        Ok(())
+    }
 }

crates/repx-executor/src/runtime/bwrap.rs

@@ -3,6 +3,87 @@
 mod harness;
 use harness::TestHarness;
 use std::fs;
+
+#[test]
+fn test_bwrap_overlay_fallback_when_disabled() {
+    let harness = TestHarness::with_execution_type("bwrap");
+    harness.stage_lab();
+
+    let base_path = &harness.cache_dir;
+
+    let capabilities_dir = base_path.join("cache").join("capabilities");
+    fs::create_dir_all(&capabilities_dir).expect("Failed to create capabilities dir");
+
+    let cache_content = r#"{
+        "tmp_overlay_supported": false,
+        "checked_at": "2025-01-01T00:00:00Z"
+    }"#;
+    fs::write(capabilities_dir.join("overlay_support.json"), cache_content)
+        .expect("Failed to write overlay cache");
+
+    let mut cmd = harness.cmd();
+    cmd.arg("run").arg("simulation-run");
+
+    let output = cmd.output().expect("Failed to execute command");
+
+    println!("STDOUT: {}", String::from_utf8_lossy(&output.stdout));
+    println!("STDERR: {}", String::from_utf8_lossy(&output.stderr));
+
+    let stage_e_job_id = harness.get_job_id_by_name("stage-E-total-sum");
+    let stage_e_path = harness.get_job_output_path(&stage_e_job_id);
+
+    assert!(
+        stage_e_path.join("repx/SUCCESS").exists(),
+        "Job should succeed with overlay fallback"
+    );
+
+    let total_sum_content = fs::read_to_string(stage_e_path.join("out/total_sum.txt")).unwrap();
+    let val = total_sum_content.trim();
+    assert!(
+        val == "400" || val == "415",
+        "Expected 400 or 415, got {}",
+        val
+    );
+
+    let cache_content_after =
+        fs::read_to_string(capabilities_dir.join("overlay_support.json")).unwrap();
+    assert!(
+        cache_content_after.contains("\"tmp_overlay_supported\": false"),
+        "Cache should still show overlay as unsupported"
+    );
+}
+
+#[test]
+fn test_bwrap_overlay_capability_detection() {
+    let harness = TestHarness::with_execution_type("bwrap");
+    harness.stage_lab();
+
+    let base_path = &harness.cache_dir;
+    let capabilities_dir = base_path.join("cache").join("capabilities");
+
+    let _ = fs::remove_dir_all(&capabilities_dir);
+
+    let mut cmd = harness.cmd();
+    cmd.arg("run").arg("simulation-run");
+    cmd.assert().success();
+
+    let cache_file = capabilities_dir.join("overlay_support.json");
+    assert!(
+        cache_file.exists(),
+        "Overlay capability cache should be created after first run"
+    );
+
+    let cache_content = fs::read_to_string(&cache_file).unwrap();
+    assert!(
+        cache_content.contains("tmp_overlay_supported"),
+        "Cache should contain overlay support status"
+    );
+    assert!(
+        cache_content.contains("checked_at"),
+        "Cache should contain timestamp"
+    );
+}
+
 #[test]
 fn test_full_run_local_bwrap() {
     let harness = TestHarness::with_execution_type("bwrap");

crates/repx-runner/tests/bwrap_tests.rs

@@ -49,6 +49,9 @@ let
     incremental-sync = import ./checks/runtime/incremental-sync-test.nix {
       inherit pkgs repx referenceLab;
     };
+    e2e-bwrap-overlay-fallback = import ./checks/runtime/e2e-bwrap-overlay-fallback.nix {
+      inherit pkgs repx referenceLab;
+    };
   };
 
   libChecks = {