Add additional sandboxing to compose and k8s backends (#78)

Author: nhynes

README.md

@@ -139,6 +139,11 @@ Direct output only supports components that use `program.path`.
 - Linux: `bwrap` and `slirp4netns`
 - macOS: `/usr/bin/sandbox-exec`
 
+Current enforcement notes:
+- Direct/native on Linux has the strongest capability mediation today: Amber runs each component behind a sidecar/router, isolates sidecar networking, joins the component into that namespace, shapes the filesystem with curated read-only mounts plus explicit writable storage, and drops all Linux capabilities for Amber-owned sidecars.
+- Docker Compose and Kubernetes now default generated containers to non-escalating privilege settings, run Amber-owned internal routers/provisioners non-root where their images already guarantee it, make those internal root filesystems read-only where possible, and reject external slot targets that resolve to loopback or link-local IPs.
+- Docker Compose and Kubernetes do not yet transparently redirect all arbitrary container egress through the router. Amber strongly mediates declared capability paths, but shared pod/service networking still means generic outbound traffic is not yet fully non-bypassable on those backends.
+
 ### 3c) Generate VM output and run
 
 ```sh

cli/src/main.rs

@@ -954,6 +954,8 @@ struct ProcessSpec {
     env: BTreeMap<String, String>,
     work_dir: PathBuf,
     #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+    drop_all_caps: bool,
+    #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
     read_only_mounts: Vec<ReadOnlyMount>,
     writable_dirs: Vec<PathBuf>,
     bind_dirs: Vec<PathBuf>,
@@ -1180,6 +1182,7 @@ async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> {
                 args: Vec::new(),
                 env,
                 work_dir,
+                drop_all_caps: true,
                 read_only_mounts: vec![ReadOnlyMount {
                     source: runtime_root.join("mesh"),
                     dest: runtime_root.join("mesh"),
@@ -1231,6 +1234,7 @@ async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> {
                 args: Vec::new(),
                 env,
                 work_dir,
+                drop_all_caps: true,
                 read_only_mounts: vec![ReadOnlyMount {
                     source: runtime_root.join("mesh"),
                     dest: runtime_root.join("mesh"),
@@ -1771,6 +1775,7 @@ fn component_program_spec(
                 args,
                 env: env.clone(),
                 work_dir,
+                drop_all_caps: false,
                 read_only_mounts,
                 writable_dirs,
                 bind_dirs: Vec::new(),
@@ -1820,6 +1825,7 @@ fn component_program_spec(
                 args: vec!["run".to_string()],
                 env,
                 work_dir,
+                drop_all_caps: false,
                 read_only_mounts,
                 writable_dirs,
                 bind_dirs: Vec::new(),
@@ -2981,6 +2987,10 @@ impl DirectSandbox {
                     "--chdir".to_string(),
                     spec.work_dir.display().to_string(),
                 ];
+                if spec.drop_all_caps {
+                    args.push("--cap-drop".to_string());
+                    args.push("ALL".to_string());
+                }
                 if matches!(spec.network, ProcessNetwork::Isolated) {
                     args.push("--unshare-net".to_string());
                 }
@@ -4847,6 +4857,7 @@ mod tests {
             args: vec!["ok".to_string()],
             env: BTreeMap::new(),
             work_dir: PathBuf::from("/tmp/amber-work"),
+            drop_all_caps: false,
             read_only_mounts: Vec::new(),
             writable_dirs: Vec::new(),
             bind_dirs: Vec::new(),
@@ -4985,6 +4996,27 @@ mod tests {
         );
     }
 
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn bubblewrap_can_drop_all_caps_for_internal_processes() {
+        let mut sandbox = DirectSandbox::Bubblewrap {
+            binary: PathBuf::from("/usr/bin/bwrap"),
+        };
+        let spec = ProcessSpec {
+            drop_all_caps: true,
+            ..linux_test_process_spec()
+        };
+
+        let (_, args) = sandbox
+            .wrap_command(&spec)
+            .expect("command should be wrapped");
+        assert!(
+            args.windows(2)
+                .any(|window| window[0] == "--cap-drop" && window[1] == "ALL"),
+            "bubblewrap args should drop all caps when requested: {args:?}"
+        );
+    }
+
     #[cfg(target_os = "linux")]
     #[test]
     fn bubblewrap_uses_curated_linux_mounts() {

compiler/src/targets/mesh/docker_compose/mod.rs

@@ -184,6 +184,8 @@ struct Service {
     image: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     user: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    read_only: Option<bool>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     cap_add: Vec<String>,
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
@@ -504,6 +506,8 @@ fn render_docker_compose_inner(scenario: &Scenario) -> DcResult<DockerComposeArt
             .volumes
             .push(format!("{HELPER_VOLUME_NAME}:{HELPER_BIN_DIR}"));
         helper_init.restart = Some("no".to_string());
+        apply_default_service_hardening(&mut helper_init);
+        apply_internal_service_rootfs_hardening(&mut helper_init);
         compose
             .services
             .insert(HELPER_INIT_SERVICE.to_string(), helper_init);
@@ -516,10 +520,7 @@ fn render_docker_compose_inner(scenario: &Scenario) -> DcResult<DockerComposeArt
 
         let mut provisioner_service = Service::new(images.provisioner.clone());
         provisioner_service.user = Some("0:0".to_string());
-        provisioner_service.cap_drop.push("ALL".to_string());
-        provisioner_service
-            .security_opt
-            .push("no-new-privileges:true".to_string());
+        apply_default_service_hardening(&mut provisioner_service);
         provisioner_service.environment = Some(Environment::List(vec![format!(
             "AMBER_MESH_PROVISION_PLAN_PATH={PROVISIONER_PLAN_PATH}"
         )]));
@@ -583,6 +584,7 @@ fn render_docker_compose_inner(scenario: &Scenario) -> DcResult<DockerComposeArt
         "${{AMBER_DOCKER_CONTAINER_LOGS_DIR:-{DOCKER_CONTAINER_LOGS_DIR}}}:\
          {DOCKER_CONTAINER_LOGS_DIR}:ro"
     ));
+    apply_default_service_hardening(&mut otelcol_service);
     compose
         .services
         .insert(OTELCOL_SERVICE_NAME.to_string(), otelcol_service);
@@ -625,6 +627,7 @@ fn render_docker_compose_inner(scenario: &Scenario) -> DcResult<DockerComposeArt
         ));
         push_router_observability_env(&mut env_entries);
         let mut router_service = Service::new(images.router.clone());
+        router_service.user = Some(format!("{ROUTER_RUNTIME_UID}:{ROUTER_RUNTIME_GID}"));
         router_service.environment = Some(Environment::List(env_entries));
         router_service
             .extra_hosts
@@ -667,6 +670,8 @@ fn render_docker_compose_inner(scenario: &Scenario) -> DcResult<DockerComposeArt
                 ),
             ],
         );
+        apply_default_service_hardening(&mut router_service);
+        apply_internal_service_rootfs_hardening(&mut router_service);
 
         compose
             .services
@@ -678,6 +683,7 @@ fn render_docker_compose_inner(scenario: &Scenario) -> DcResult<DockerComposeArt
         let svc = names.get(id).unwrap();
 
         let mut sidecar_service = Service::new(images.router.clone());
+        sidecar_service.user = Some(format!("{ROUTER_RUNTIME_UID}:{ROUTER_RUNTIME_GID}"));
         let mut sidecar_env_entries = vec![
             format!("AMBER_ROUTER_CONFIG_PATH={}", mesh_config_path()),
             format!("AMBER_ROUTER_IDENTITY_PATH={}", mesh_identity_path()),
@@ -702,6 +708,8 @@ fn render_docker_compose_inner(scenario: &Scenario) -> DcResult<DockerComposeArt
                 "service_completed_successfully",
             )],
         );
+        apply_default_service_hardening(&mut sidecar_service);
+        apply_internal_service_rootfs_hardening(&mut sidecar_service);
         compose
             .services
             .insert(svc.sidecar.clone(), sidecar_service);
@@ -845,6 +853,7 @@ fn render_docker_compose_inner(scenario: &Scenario) -> DcResult<DockerComposeArt
         }
         if Some(*id) == docker_gateway_component {
             configure_injected_docker_gateway_service(&mut program_service, s, *id, svc)?;
+            apply_internal_service_rootfs_hardening(&mut program_service);
         }
         for storage_mount in storage_mounts {
             program_service.volumes.push(format!(
@@ -858,6 +867,7 @@ fn render_docker_compose_inner(scenario: &Scenario) -> DcResult<DockerComposeArt
             &label,
             &compose_component_service_name(&label),
         );
+        apply_default_service_hardening(&mut program_service);
 
         compose
             .services
@@ -1012,6 +1022,25 @@ service:
     )
 }
 
+fn apply_default_service_hardening(service: &mut Service) {
+    if !service.cap_drop.iter().any(|cap| cap == "ALL") {
+        service.cap_drop.push("ALL".to_string());
+    }
+    if !service
+        .security_opt
+        .iter()
+        .any(|opt| opt == "no-new-privileges:true")
+    {
+        service
+            .security_opt
+            .push("no-new-privileges:true".to_string());
+    }
+}
+
+fn apply_internal_service_rootfs_hardening(service: &mut Service) {
+    service.read_only = Some(true);
+}
+
 fn push_router_observability_env(env_entries: &mut Vec<String>) {
     env_entries.push(format!(
         "{SCENARIO_RUN_ID_ENV}=${{{COMPOSE_PROJECT_NAME_ENV}:-default}}"

compiler/src/targets/mesh/docker_compose/tests.rs

@@ -376,6 +376,21 @@ fn env_value(service: &super::Service, key: &str) -> Option<String> {
     }
 }
 
+fn assert_service_hardened(service: &super::Service, yaml: &str) {
+    assert!(service.cap_drop.iter().any(|cap| cap == "ALL"), "{yaml}");
+    assert!(
+        service
+            .security_opt
+            .iter()
+            .any(|opt| opt == "no-new-privileges:true"),
+        "{yaml}"
+    );
+}
+
+fn assert_internal_service_rootfs_hardened(service: &super::Service, yaml: &str) {
+    assert_eq!(service.read_only, Some(true), "{yaml}");
+}
+
 fn injected_docker_gateway_service(compose: &super::DockerComposeFile) -> (&str, &super::Service) {
     compose
         .services
@@ -414,7 +429,7 @@ fn assert_depends_on(service: &super::Service, name: &str, condition: &str) {
 }
 
 fn storage_scenario(version: &str, initial_state: &str) -> Scenario {
-    let provide_http =
+    let provide_http: ProvideDecl =
         serde_json::from_value(json!({ "kind": "http", "endpoint": "http" })).unwrap();
 
     let root = Component {
@@ -447,7 +462,7 @@ fn storage_scenario(version: &str, initial_state: &str) -> Scenario {
             }),
         )),
         slots: BTreeMap::new(),
-        provides: BTreeMap::from([("http".to_string(), provide_http)]),
+        provides: BTreeMap::from([("http".to_string(), provide_http.clone())]),
         resources: BTreeMap::from([("state".to_string(), storage_resource_decl(None))]),
         metadata: None,
         children: Vec::new(),
@@ -457,7 +472,14 @@ fn storage_scenario(version: &str, initial_state: &str) -> Scenario {
         root: ComponentId(0),
         components: vec![Some(root)],
         bindings: Vec::new(),
-        exports: Vec::new(),
+        exports: vec![ScenarioExport {
+            name: "http".to_string(),
+            capability: provide_http.decl.clone(),
+            from: ProvideRef {
+                component: ComponentId(0),
+                name: "http".to_string(),
+            },
+        }],
     }
 }
 
@@ -864,6 +886,7 @@ fn docker_compose_emits_gateway_for_framework_docker_binding() {
         gateway.network_mode.as_deref(),
         Some(expected_network_mode.as_str()),
     );
+    assert_internal_service_rootfs_hardened(gateway, yaml.as_ref());
 
     let gateway_config = env_value(gateway, super::DOCKER_GATEWAY_CONFIG_ENV)
         .expect("gateway config env should be present");
@@ -962,6 +985,10 @@ fn docker_compose_emits_framework_docker_mount_proxy_wiring() {
         super::HELPER_INIT_SERVICE,
         "service_completed_successfully",
     );
+    assert_internal_service_rootfs_hardened(
+        service(&compose, super::HELPER_INIT_SERVICE),
+        yaml.as_ref(),
+    );
     assert_depends_on(program_service, gateway_name, "service_started");
 
     let mount_proxy_b64 = env_value(program_service, super::DOCKER_MOUNT_PROXY_SPEC_ENV)
@@ -1109,6 +1136,20 @@ fn compose_emits_sidecars_and_programs_and_slot_urls() {
     assert!(compose.services.contains_key("c2-client-net"), "{yaml}");
     assert!(compose.services.contains_key("c2-client"), "{yaml}");
 
+    for name in [
+        "amber-provisioner",
+        "amber-otelcol",
+        "c1-server-net",
+        "c1-server",
+        "c2-client-net",
+        "c2-client",
+    ] {
+        assert_service_hardened(service(&compose, name), yaml.as_ref());
+    }
+    for name in ["c1-server-net", "c2-client-net"] {
+        assert_internal_service_rootfs_hardened(service(&compose, name), yaml.as_ref());
+    }
+
     // Program uses sidecar netns.
     assert_eq!(
         service(&compose, "c2-client").network_mode.as_deref(),
@@ -1626,6 +1667,12 @@ fn compose_routes_external_slots_through_router() {
     assert!(compose.services.contains_key("amber-router"));
     let router_service = service(&compose, "amber-router");
     assert!(env_value(router_service, "AMBER_EXTERNAL_SLOT_API_URL").is_some());
+    assert_eq!(
+        router_service.user.as_deref(),
+        Some("65532:65532"),
+        "{yaml}"
+    );
+    assert_internal_service_rootfs_hardened(router_service, yaml.as_ref());
     assert!(
         router_service.ports.is_empty(),
         "slot-only scenarios should not publish the router mesh port on the host"