diff --git a/doc/manual/rl-next/socket-only-daemon.md b/doc/manual/rl-next/socket-only-daemon.md new file mode 100644 index 00000000000..02b3e111cff --- /dev/null +++ b/doc/manual/rl-next/socket-only-daemon.md @@ -0,0 +1,10 @@ +--- +synopsis: "Nix daemon can serve store paths over sockets without filesystem access" +--- + +The Nix daemon can now serve store paths purely over Unix domain sockets without +requiring the client to have filesystem access to the store directory. This can be +useful for VM setups where the host serves store paths to the guest via socket, +with the guest having no direct access to the host's `/nix/store`. + +This works for copying paths, substitution, and building. diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e6efd6c0981..937946134bd 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -896,7 +896,7 @@ static void performOp( auto path = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); logger->stopWork(); - dumpPath(store->toRealPath(path), conn.to); + store->narFromPath(path, conn.to); break; } diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index 764e8768a32..1f3b1696679 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -68,7 +68,7 @@ struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore void narFromPath(const StorePath & path, Sink & sink) override { - Store::narFromPath(path, sink); + RemoteStore::narFromPath(path, sink); } /** diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 6f649c8360b..bc541401067 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -84,6 +84,8 @@ suites = [ 'referrers.sh', 'optimise-store.sh', 'substitute-with-invalid-ca.sh', + 'socket-only-daemon.sh', + 'socket-only-daemon-binary-cache.sh', 'signing.sh', 'hash-convert.sh', 'hash-path.sh', diff --git a/tests/functional/socket-only-daemon-binary-cache.sh b/tests/functional/socket-only-daemon-binary-cache.sh new file mode 100755 index 00000000000..78d300f6a53 --- /dev/null +++ b/tests/functional/socket-only-daemon-binary-cache.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# Run the socket-only-daemon test with a binary cache backend +# This tests that +# - The daemon can serve a binary cache store; not just a local store +# - Client's store operations do not reach into the file system secretly, +# because the files don't exist in the places where a local store would +# put them. (We have NARs instead) + +daemon_backing_store_is_binary_cache=1 +source socket-only-daemon.sh diff --git a/tests/functional/socket-only-daemon.sh b/tests/functional/socket-only-daemon.sh new file mode 100755 index 00000000000..7d3cfd56916 --- /dev/null +++ b/tests/functional/socket-only-daemon.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + +# Test that a daemon can serve store paths purely over the socket, +# without requiring filesystem access to the store directory. +# This is important for VM setups where the host serves paths to +# the guest via socket, but the store directory is not shared. +# +# Can be called with daemon_backing_store_is_binary_cache=1 to test with a binary cache +# instead of a regular local store. See socket-only-daemon-binary-cache.sh. + +source common.sh + +needLocalStore "This test requires starting a separate daemon" + +# Create state and cache locations +# Note: We use the same NIX_STORE_DIR (logical store path) as the test environment so paths are compatible +remote_cache_dir="$TEST_ROOT/remote-cache-$RANDOM" +remote_state_dir="$TEST_ROOT/remote-state-$RANDOM" +remote_real_store_dir="$TEST_ROOT/remote-real-store-$RANDOM" +remote_socket="$TEST_ROOT/remote-socket-initial" +moved_socket="$TEST_ROOT/moved-socket" + +# Set up the remote store URI based on store type +if [[ "${daemon_backing_store_is_binary_cache:-0}" == "1" ]]; then + echo "Using binary cache store" + mkdir -p "$remote_cache_dir" + remote_full_store_uri="file://$remote_cache_dir" +else + echo "Using local store with different physical location" + mkdir -p "$remote_state_dir" + mkdir -p "$remote_real_store_dir" + remote_full_store_uri="local?store=$NIX_STORE_DIR&real=$remote_real_store_dir&state=$remote_state_dir" +fi + +# Create a test derivation file +cat > "$TEST_ROOT/test-derivation.nix" </dev/null || true + wait "$remote_daemon_pid" 2>/dev/null || true + fi +} +trap cleanup_daemon EXIT + +# Wait for socket to appear +for ((i = 0; i < 60; i++)); do + if [[ -S "$remote_socket" ]]; then + daemon_started=1 + break + fi + if ! kill -0 "$remote_daemon_pid"; then + fail "Remote daemon died unexpectedly" + fi + sleep 0.1 +done +[[ -n "${daemon_started:-}" ]] || fail "Remote daemon didn't start" + +echo "Remote daemon started with PID $remote_daemon_pid" + +# Move the socket to a different location to prevent any path-based +# assumptions from accidentally working (mildly paranoid, mildly effective; +# ideally we'd use a namespace, but that level of complexity is not actually +# needed) +mv "$remote_socket" "$moved_socket" + +echo "Socket moved to: $moved_socket" + +# Clear our local store so we need to substitute +clearStore + +# Try to copy the path from the daemon via the moved socket +# NOTE: We do NOT pass the store location to the client - only the socket! +# The daemon must be able to serve paths knowing only what's in its own configuration. +nix copy --from "unix://$moved_socket" --no-require-sigs "$out" + +# Verify the content +[[ -f "$out" ]] || fail "Output path doesn't exist" +[[ "$(cat "$out")" == "hello-from-remote" ]] || fail "Output content is wrong" + +echo "Socket-only copy test PASSED" + +# Clear the store again to test substituters mechanism +clearStore + +# First verify that --max-jobs 0 without substituters fails (test our assumption) +if nix-build --max-jobs 0 --no-out-link "$TEST_ROOT/test-derivation.nix" 2>/dev/null; then + fail "Building with --max-jobs 0 should have failed without substituters" +fi + +echo "Confirmed: --max-jobs 0 without substituters fails as expected" + +# Now test using the socket as a substituter with --max-jobs 0 (no building allowed) +# This ensures the substituter mechanism works, not just nix copy +nix-build --max-jobs 0 \ + --option substituters "unix://$moved_socket" \ + --option require-sigs false \ + --no-out-link \ + "$TEST_ROOT/test-derivation.nix" + +echo "Socket-only substituter test PASSED" + +# Test builders mechanism (only on Linux with daemon backed by local store) +# - Builders need sandboxing with namespace support to mount the correct store path +# - Binary cache stores can't build, only serve files +if [[ $(uname) == Linux && "${daemon_backing_store_is_binary_cache:-0}" == "0" ]]; then + # Clear the store again to test builders mechanism + clearStore + + # Test using the socket as a remote builder + # This ensures the builders mechanism can also use socket-only connections + nix-build \ + --option builders "unix://$moved_socket" \ + --option require-sigs false \ + --max-jobs 0 \ + --no-out-link \ + "$TEST_ROOT/test-derivation.nix" + + echo "Socket-only builder test PASSED" +fi diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index edfa4124f3f..a1e853f2981 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -115,6 +115,8 @@ in remoteBuildsSshNg = runNixOSTest ./remote-builds-ssh-ng.nix; + remoteBuildsPlainDaemon = runNixOSTest ./remote-builds-plain-daemon.nix; + } // lib.concatMapAttrs ( nixVersion: diff --git a/tests/nixos/remote-builds-plain-daemon.nix b/tests/nixos/remote-builds-plain-daemon.nix new file mode 100644 index 00000000000..2f7ad21ba8d --- /dev/null +++ b/tests/nixos/remote-builds-plain-daemon.nix @@ -0,0 +1,217 @@ +# Test Nix's remote build feature with host-to-VM socket forwarding. +# This tests that the host (test driver) can perform remote builds in a VM +# using a socket connection, demonstrating the socket-only daemon functionality. + +{ + config, + hostPkgs, + ... +}: + +let + # TCP port for the Nix daemon inside the VM + daemonPort = 3049; + + # The configuration of the VM builder. + builder = + { + config, + pkgs, + lib, + ... + }: + { + environment.systemPackages = [ pkgs.netcat ]; + virtualisation.writableStore = true; + nix.settings.sandbox = true; + + # Forward TCP port from host to guest + # We'll use socat on the host to bridge Unix socket → localhost TCP + # QEMU forwards localhost:daemonPort → guest:10.0.2.15:daemonPort + # Note: QEMU will support Unix socket forwarding natively (hostfwd=unix:...) + # once https://gitlab.com/qemu-project/qemu/-/commit/6d10e021318b16e3e90f98b7b2fa187826e26c0a + # is released, which would eliminate the need for socat + virtualisation.forwardPorts = [ + { + from = "host"; + host.port = daemonPort; + guest.port = daemonPort; + } + ]; + + # Configure nix-daemon to listen on TCP directly + # Empty string clears the default Unix socket, then we add TCP + # Note: We bind to the eth0 IP address. In QEMU user networking, this is typically + # 10.0.2.15 but we use FreeBind to allow binding before the network is fully configured. + # Binding to a specific IP (not 0.0.0.0) prevents listening on localhost. + # NOTE: This is similar to what is proposed in https://github.com/systemd/systemd/issues/32795 + # (using a separate network interface only accessible to systemd), but that's not yet + # implemented. + systemd.sockets.nix-daemon = { + listenStreams = [ + "" + # QEMU user networking assigns 10.0.2.15 by default + "10.0.2.15:${toString daemonPort}" + ]; + # FreeBind allows binding to IPs that don't exist yet + socketConfig = { + FreeBind = true; + }; + }; + + # Restrict access to the daemon port: only allow connections from QEMU gateway + # In QEMU user networking, forwarded connections appear to come from the gateway (10.0.2.2) + # This should prevent unprivileged guest processes from accessing the daemon. + # For production use, consider additional isolation mechanisms (see systemd.sockets comment above). + # This has not been audited. + networking.firewall.extraCommands = '' + # Insert in reverse order since -I inserts at position 1 + # Drop all connections to daemon port (inserted first, will be at position 2) + iptables -I nixos-fw -p tcp --dport ${toString daemonPort} -j nixos-fw-log-refuse + # Allow connections from QEMU gateway only (inserted second, will be at position 1) + iptables -I nixos-fw -p tcp --dport ${toString daemonPort} -s 10.0.2.2 -j nixos-fw-accept + ''; + }; + +in + +{ + config = { + name = "remote-builds-plain-daemon"; + + nodes = { + builder = builder; + }; + + testScript = + { nodes }: + '' + # fmt: off + import subprocess + import os + import time + + start_all() + + # Wait for the VM to be ready + builder.wait_for_unit("nix-daemon.socket") + + # Verify the daemon is listening on TCP + builder.succeed("ss -tlnp | grep ${toString daemonPort}") + + print("VM builder is ready with TCP daemon") + + # Start socat to bridge Unix socket → localhost TCP + # QEMU forwards localhost:daemonPort → VM's 10.0.2.15:daemonPort + # (See virtualisation.forwardPorts comment for future QEMU native Unix socket support) + socket_path = os.environ.get('TMPDIR', '/tmp') + '/nix-builder.sock' + + print(f"Starting socat to forward {socket_path} -> localhost:${toString daemonPort}") + socat_proc = subprocess.Popen( + ["${hostPkgs.socat}/bin/socat", + f"UNIX-LISTEN:{socket_path},fork", + "TCP:127.0.0.1:${toString daemonPort}"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + # Wait for socket to be created + for i in range(30): + if os.path.exists(socket_path): + break + if socat_proc.poll() is not None: + stdout, stderr = socat_proc.communicate() + raise Exception(f"socat died unexpectedly: {stderr.decode()}") + time.sleep(0.1) + else: + socat_proc.terminate() + raise Exception(f"Socket {socket_path} was not created by socat") + + print(f"Host socket {socket_path} ready (socat -> QEMU -> VM)") + + # Test the connection by trying a simple operation + print("Testing connection to VM daemon...") + subprocess.run( + ["${hostPkgs.nix}/bin/nix", + "--extra-experimental-features", "nix-command", + "--store", f"unix://{socket_path}", + "store", "ping"], + check=True, + timeout=60 + ) + + # Create a simple derivation to build + # Use a fixed system instead of builtins.currentSystem + test_expr = """ + derivation { + name = "socket-forward-host-build-test"; + system = "x86_64-linux"; + builder = "/bin/sh"; + args = [ "-c" "echo 'Built via forwarded socket from host!' > $out" ]; + } + """ + + expr_file = os.environ.get('TMPDIR', '/tmp') + '/test-expr.nix' + with open(expr_file, 'w') as f: + f.write(test_expr) + + # Create a fresh store for the host + host_store = os.environ.get('TMPDIR', '/tmp') + '/host-store' + os.makedirs(host_store, exist_ok=True) + + # Perform a build from the host using the VM as a builder + # Builders format: + print("Host performing remote build in VM via socket->TCP bridge...") + result = subprocess.run( + ["${hostPkgs.nix}/bin/nix-build", + "--store", host_store, + expr_file, + "--no-out-link", + "--option", "builders", f"unix://{socket_path} x86_64-linux - 1", + "--option", "require-sigs", "false", + "--max-jobs", "0"], # Force remote building + stdout=subprocess.PIPE, + text=True + ) + + if result.returncode != 0: + print(f"Build failed with exit code {result.returncode}") + raise Exception("Build failed") + + out_path = result.stdout.strip() + print(f"Build succeeded! Output: {out_path}") + + # Verify the build happened in the VM by checking it exists there + builder.succeed(f"test -e {out_path}") + + # Verify the build output was copied to the host's physical store + host_physical_path = f"{host_store}{out_path}" + print(f"Checking host physical store at: {host_physical_path}") + if not os.path.exists(host_physical_path): + raise Exception(f"Build output not found in host store: {host_physical_path}") + + # Verify the build output content + with open(host_physical_path, 'r') as f: + content = f.read() + expected = "Built via forwarded socket from host!\n" + if content != expected: + raise Exception(f"Build output has wrong content. Expected: {repr(expected)}, got: {repr(content)}") + + print("Socket-forwarded remote build from host test PASSED") + + + # Test that guest processes CANNOT connect (firewall enabled) + print("Testing that guest user CANNOT connect to daemon port (firewall enabled)...") + builder.fail("timeout 5 nc -z 127.0.0.1 ${toString daemonPort}") + print("Confirmed: guest user cannot connect to localhost") + + builder.fail("timeout 5 nc -z 10.0.2.15 ${toString daemonPort}") + print("Confirmed: guest user cannot connect to interface IP") + + # Clean up socat + print("Cleaning up socat...") + socat_proc.terminate() + socat_proc.wait(timeout=5) + ''; + }; +}