From 6a017a2a870723b9ba7d0f10348f17408ad58f2b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 4 Nov 2025 16:32:17 +0100 Subject: [PATCH 1/2] Enable socket-only daemon operations without filesystem access Allows the Nix daemon to serve store paths purely over Unix domain sockets without requiring the client to have filesystem access to the store directory. This can be useful for VM setups where the host serves paths to the guest via socket. Tests verify socket-only operations work for copying, substitution, and remote building (tested on Linux), with both local and binary cache stores. --- doc/manual/rl-next/socket-only-daemon.md | 10 ++ src/libstore/daemon.cc | 2 +- .../include/nix/store/uds-remote-store.hh | 2 +- tests/functional/meson.build | 2 + .../socket-only-daemon-binary-cache.sh | 11 ++ tests/functional/socket-only-daemon.sh | 149 ++++++++++++++++++ 6 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 doc/manual/rl-next/socket-only-daemon.md create mode 100755 tests/functional/socket-only-daemon-binary-cache.sh create mode 100755 tests/functional/socket-only-daemon.sh diff --git a/doc/manual/rl-next/socket-only-daemon.md b/doc/manual/rl-next/socket-only-daemon.md new file mode 100644 index 00000000000..02b3e111cff --- /dev/null +++ b/doc/manual/rl-next/socket-only-daemon.md @@ -0,0 +1,10 @@ +--- +synopsis: "Nix daemon can serve store paths over sockets without filesystem access" +--- + +The Nix daemon can now serve store paths purely over Unix domain sockets without +requiring the client to have filesystem access to the store directory. This can be +useful for VM setups where the host serves store paths to the guest via socket, +with the guest having no direct access to the host's `/nix/store`. + +This works for copying paths, substitution, and building. diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e6efd6c0981..937946134bd 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -896,7 +896,7 @@ static void performOp( auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); logger->stopWork(); - dumpPath(store->toRealPath(path), conn.to); + store->narFromPath(path, conn.to); break; } diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index 764e8768a32..1f3b1696679 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -68,7 +68,7 @@ struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore void narFromPath(const StorePath & path, Sink & sink) override { - Store::narFromPath(path, sink); + RemoteStore::narFromPath(path, sink); } /** diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 6f649c8360b..bc541401067 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -84,6 +84,8 @@ suites = [ 'referrers.sh', 'optimise-store.sh', 'substitute-with-invalid-ca.sh', + 'socket-only-daemon.sh', + 'socket-only-daemon-binary-cache.sh', 'signing.sh', 'hash-convert.sh', 'hash-path.sh', diff --git a/tests/functional/socket-only-daemon-binary-cache.sh b/tests/functional/socket-only-daemon-binary-cache.sh new file mode 100755 index 00000000000..78d300f6a53 --- /dev/null +++ b/tests/functional/socket-only-daemon-binary-cache.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# Run the socket-only-daemon test with a binary cache backend +# This tests that +# - The daemon can serve a binary cache store; not just a local store +# - Client's store operations do not reach into the file system secretly, +# because the files don't exist in the places where a local store would +# put them. (We have NARs instead) + +daemon_backing_store_is_binary_cache=1 +source socket-only-daemon.sh diff --git a/tests/functional/socket-only-daemon.sh b/tests/functional/socket-only-daemon.sh new file mode 100755 index 00000000000..7d3cfd56916 --- /dev/null +++ b/tests/functional/socket-only-daemon.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + +# Test that a daemon can serve store paths purely over the socket, +# without requiring filesystem access to the store directory. +# This is important for VM setups where the host serves paths to +# the guest via socket, but the store directory is not shared. +# +# Can be called with daemon_backing_store_is_binary_cache=1 to test with a binary cache +# instead of a regular local store. See socket-only-daemon-binary-cache.sh. + +source common.sh + +needLocalStore "This test requires starting a separate daemon" + +# Create state and cache locations +# Note: We use the same NIX_STORE_DIR (logical store path) as the test environment so paths are compatible +remote_cache_dir="$TEST_ROOT/remote-cache-$RANDOM" +remote_state_dir="$TEST_ROOT/remote-state-$RANDOM" +remote_real_store_dir="$TEST_ROOT/remote-real-store-$RANDOM" +remote_socket="$TEST_ROOT/remote-socket-initial" +moved_socket="$TEST_ROOT/moved-socket" + +# Set up the remote store URI based on store type +if [[ "${daemon_backing_store_is_binary_cache:-0}" == "1" ]]; then + echo "Using binary cache store" + mkdir -p "$remote_cache_dir" + remote_full_store_uri="file://$remote_cache_dir" +else + echo "Using local store with different physical location" + mkdir -p "$remote_state_dir" + mkdir -p "$remote_real_store_dir" + remote_full_store_uri="local?store=$NIX_STORE_DIR&real=$remote_real_store_dir&state=$remote_state_dir" +fi + +# Create a test derivation file +cat > "$TEST_ROOT/test-derivation.nix" </dev/null || true + wait "$remote_daemon_pid" 2>/dev/null || true + fi +} +trap cleanup_daemon EXIT + +# Wait for socket to appear +for ((i = 0; i < 60; i++)); do + if [[ -S "$remote_socket" ]]; then + daemon_started=1 + break + fi + if ! kill -0 "$remote_daemon_pid"; then + fail "Remote daemon died unexpectedly" + fi + sleep 0.1 +done +[[ -n "${daemon_started:-}" ]] || fail "Remote daemon didn't start" + +echo "Remote daemon started with PID $remote_daemon_pid" + +# Move the socket to a different location to prevent any path-based +# assumptions from accidentally working (mildly paranoid, mildly effective; +# ideally we'd use a namespace, but that level of complexity is not actually +# needed) +mv "$remote_socket" "$moved_socket" + +echo "Socket moved to: $moved_socket" + +# Clear our local store so we need to substitute +clearStore + +# Try to copy the path from the daemon via the moved socket +# NOTE: We do NOT pass the store location to the client - only the socket! +# The daemon must be able to serve paths knowing only what's in its own configuration. +nix copy --from "unix://$moved_socket" --no-require-sigs "$out" + +# Verify the content +[[ -f "$out" ]] || fail "Output path doesn't exist" +[[ "$(cat "$out")" == "hello-from-remote" ]] || fail "Output content is wrong" + +echo "Socket-only copy test PASSED" + +# Clear the store again to test substituters mechanism +clearStore + +# First verify that --max-jobs 0 without substituters fails (test our assumption) +if nix-build --max-jobs 0 --no-out-link "$TEST_ROOT/test-derivation.nix" 2>/dev/null; then + fail "Building with --max-jobs 0 should have failed without substituters" +fi + +echo "Confirmed: --max-jobs 0 without substituters fails as expected" + +# Now test using the socket as a substituter with --max-jobs 0 (no building allowed) +# This ensures the substituter mechanism works, not just nix copy +nix-build --max-jobs 0 \ + --option substituters "unix://$moved_socket" \ + --option require-sigs false \ + --no-out-link \ + "$TEST_ROOT/test-derivation.nix" + +echo "Socket-only substituter test PASSED" + +# Test builders mechanism (only on Linux with daemon backed by local store) +# - Builders need sandboxing with namespace support to mount the correct store path +# - Binary cache stores can't build, only serve files +if [[ $(uname) == Linux && "${daemon_backing_store_is_binary_cache:-0}" == "0" ]]; then + # Clear the store again to test builders mechanism + clearStore + + # Test using the socket as a remote builder + # This ensures the builders mechanism can also use socket-only connections + nix-build \ + --option builders "unix://$moved_socket" \ + --option require-sigs false \ + --max-jobs 0 \ + --no-out-link \ + "$TEST_ROOT/test-derivation.nix" + + echo "Socket-only builder test PASSED" +fi From 624e0d247c9c0449151f98935ff1839f093468e8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 4 Nov 2025 22:17:45 +0100 Subject: [PATCH 2/2] Add proof of concept daemon forwarding VM test I was hoping for a more obviously correct solution in terms of security, but this is still a nice addition to the socket-only-daemon* functional tests. This could be used as a starting point for building out two things - Another method for running the functional tests, where the local Nix client is relocated and dependent on its remote builder. - An alternative, simpler solution to the SSH-based "darwin" linux-builder solution. It would still need a means for entering a shell for troubleshooting tasks, but presumably this could also be managed through a unix socket or something. --- tests/nixos/default.nix | 2 + tests/nixos/remote-builds-plain-daemon.nix | 217 +++++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 tests/nixos/remote-builds-plain-daemon.nix diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index edfa4124f3f..a1e853f2981 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -115,6 +115,8 @@ in remoteBuildsSshNg = runNixOSTest ./remote-builds-ssh-ng.nix; + remoteBuildsPlainDaemon = runNixOSTest ./remote-builds-plain-daemon.nix; + } // lib.concatMapAttrs ( nixVersion: diff --git a/tests/nixos/remote-builds-plain-daemon.nix b/tests/nixos/remote-builds-plain-daemon.nix new file mode 100644 index 00000000000..2f7ad21ba8d --- /dev/null +++ b/tests/nixos/remote-builds-plain-daemon.nix @@ -0,0 +1,217 @@ +# Test Nix's remote build feature with host-to-VM socket forwarding. +# This tests that the host (test driver) can perform remote builds in a VM +# using a socket connection, demonstrating the socket-only daemon functionality. + +{ + config, + hostPkgs, + ... +}: + +let + # TCP port for the Nix daemon inside the VM + daemonPort = 3049; + + # The configuration of the VM builder. + builder = + { + config, + pkgs, + lib, + ... + }: + { + environment.systemPackages = [ pkgs.netcat ]; + virtualisation.writableStore = true; + nix.settings.sandbox = true; + + # Forward TCP port from host to guest + # We'll use socat on the host to bridge Unix socket → localhost TCP + # QEMU forwards localhost:daemonPort → guest:10.0.2.15:daemonPort + # Note: QEMU will support Unix socket forwarding natively (hostfwd=unix:...) + # once https://gitlab.com/qemu-project/qemu/-/commit/6d10e021318b16e3e90f98b7b2fa187826e26c0a + # is released, which would eliminate the need for socat + virtualisation.forwardPorts = [ + { + from = "host"; + host.port = daemonPort; + guest.port = daemonPort; + } + ]; + + # Configure nix-daemon to listen on TCP directly + # Empty string clears the default Unix socket, then we add TCP + # Note: We bind to the eth0 IP address. In QEMU user networking, this is typically + # 10.0.2.15 but we use FreeBind to allow binding before the network is fully configured. + # Binding to a specific IP (not 0.0.0.0) prevents listening on localhost. + # NOTE: This is similar to what is proposed in https://github.com/systemd/systemd/issues/32795 + # (using a separate network interface only accessible to systemd), but that's not yet + # implemented. + systemd.sockets.nix-daemon = { + listenStreams = [ + "" + # QEMU user networking assigns 10.0.2.15 by default + "10.0.2.15:${toString daemonPort}" + ]; + # FreeBind allows binding to IPs that don't exist yet + socketConfig = { + FreeBind = true; + }; + }; + + # Restrict access to the daemon port: only allow connections from QEMU gateway + # In QEMU user networking, forwarded connections appear to come from the gateway (10.0.2.2) + # This should prevent unprivileged guest processes from accessing the daemon. + # For production use, consider additional isolation mechanisms (see systemd.sockets comment above). + # This has not been audited. + networking.firewall.extraCommands = '' + # Insert in reverse order since -I inserts at position 1 + # Drop all connections to daemon port (inserted first, will be at position 2) + iptables -I nixos-fw -p tcp --dport ${toString daemonPort} -j nixos-fw-log-refuse + # Allow connections from QEMU gateway only (inserted second, will be at position 1) + iptables -I nixos-fw -p tcp --dport ${toString daemonPort} -s 10.0.2.2 -j nixos-fw-accept + ''; + }; + +in + +{ + config = { + name = "remote-builds-plain-daemon"; + + nodes = { + builder = builder; + }; + + testScript = + { nodes }: + '' + # fmt: off + import subprocess + import os + import time + + start_all() + + # Wait for the VM to be ready + builder.wait_for_unit("nix-daemon.socket") + + # Verify the daemon is listening on TCP + builder.succeed("ss -tlnp | grep ${toString daemonPort}") + + print("VM builder is ready with TCP daemon") + + # Start socat to bridge Unix socket → localhost TCP + # QEMU forwards localhost:daemonPort → VM's 10.0.2.15:daemonPort + # (See virtualisation.forwardPorts comment for future QEMU native Unix socket support) + socket_path = os.environ.get('TMPDIR', '/tmp') + '/nix-builder.sock' + + print(f"Starting socat to forward {socket_path} -> localhost:${toString daemonPort}") + socat_proc = subprocess.Popen( + ["${hostPkgs.socat}/bin/socat", + f"UNIX-LISTEN:{socket_path},fork", + "TCP:127.0.0.1:${toString daemonPort}"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + # Wait for socket to be created + for i in range(30): + if os.path.exists(socket_path): + break + if socat_proc.poll() is not None: + stdout, stderr = socat_proc.communicate() + raise Exception(f"socat died unexpectedly: {stderr.decode()}") + time.sleep(0.1) + else: + socat_proc.terminate() + raise Exception(f"Socket {socket_path} was not created by socat") + + print(f"Host socket {socket_path} ready (socat -> QEMU -> VM)") + + # Test the connection by trying a simple operation + print("Testing connection to VM daemon...") + subprocess.run( + ["${hostPkgs.nix}/bin/nix", + "--extra-experimental-features", "nix-command", + "--store", f"unix://{socket_path}", + "store", "ping"], + check=True, + timeout=60 + ) + + # Create a simple derivation to build + # Use a fixed system instead of builtins.currentSystem + test_expr = """ + derivation { + name = "socket-forward-host-build-test"; + system = "x86_64-linux"; + builder = "/bin/sh"; + args = [ "-c" "echo 'Built via forwarded socket from host!' > $out" ]; + } + """ + + expr_file = os.environ.get('TMPDIR', '/tmp') + '/test-expr.nix' + with open(expr_file, 'w') as f: + f.write(test_expr) + + # Create a fresh store for the host + host_store = os.environ.get('TMPDIR', '/tmp') + '/host-store' + os.makedirs(host_store, exist_ok=True) + + # Perform a build from the host using the VM as a builder + # Builders format: + print("Host performing remote build in VM via socket->TCP bridge...") + result = subprocess.run( + ["${hostPkgs.nix}/bin/nix-build", + "--store", host_store, + expr_file, + "--no-out-link", + "--option", "builders", f"unix://{socket_path} x86_64-linux - 1", + "--option", "require-sigs", "false", + "--max-jobs", "0"], # Force remote building + stdout=subprocess.PIPE, + text=True + ) + + if result.returncode != 0: + print(f"Build failed with exit code {result.returncode}") + raise Exception("Build failed") + + out_path = result.stdout.strip() + print(f"Build succeeded! Output: {out_path}") + + # Verify the build happened in the VM by checking it exists there + builder.succeed(f"test -e {out_path}") + + # Verify the build output was copied to the host's physical store + host_physical_path = f"{host_store}{out_path}" + print(f"Checking host physical store at: {host_physical_path}") + if not os.path.exists(host_physical_path): + raise Exception(f"Build output not found in host store: {host_physical_path}") + + # Verify the build output content + with open(host_physical_path, 'r') as f: + content = f.read() + expected = "Built via forwarded socket from host!\n" + if content != expected: + raise Exception(f"Build output has wrong content. Expected: {repr(expected)}, got: {repr(content)}") + + print("Socket-forwarded remote build from host test PASSED") + + + # Test that guest processes CANNOT connect (firewall enabled) + print("Testing that guest user CANNOT connect to daemon port (firewall enabled)...") + builder.fail("timeout 5 nc -z 127.0.0.1 ${toString daemonPort}") + print("Confirmed: guest user cannot connect to localhost") + + builder.fail("timeout 5 nc -z 10.0.2.15 ${toString daemonPort}") + print("Confirmed: guest user cannot connect to interface IP") + + # Clean up socat + print("Cleaning up socat...") + socat_proc.terminate() + socat_proc.wait(timeout=5) + ''; + }; +}