Skip to content

Commit 0adad01

Browse files
roberthroberth
committed
Enable socket-only daemon operations without filesystem access
Allows the Nix daemon to serve store paths purely over Unix domain sockets without requiring the client to have filesystem access to the store directory. This can be useful for VM setups where the host serves paths to the guest via socket. Tests verify socket-only operations work for copying, substitution, and remote building (tested on Linux), with both local and binary cache stores.
1 parent 341c42f commit 0adad01

File tree

7 files changed

+176
-4
lines changed

7 files changed

+176
-4
lines changed

‎doc/manual/rl-next/socket-only-daemon.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
synopsis: "Nix daemon can serve store paths over sockets without filesystem access"
3+
---
4+
5+
The Nix daemon can now serve store paths purely over Unix domain sockets without
6+
requiring the client to have filesystem access to the store directory. This can be
7+
useful for VM setups where the host serves store paths to the guest via socket,
8+
with the guest having no direct access to the host's `/nix/store`.
9+
10+
This works for copying paths, substitution, and building.

‎src/libstore/daemon.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -896,7 +896,7 @@ static void performOp(
896896
auto path = WorkerProto::Serialise<StorePath>::read(*store, rconn);
897897
logger->startWork();
898898
logger->stopWork();
899-
dumpPath(store->toRealPath(path), conn.to);
899+
store->narFromPath(path, conn.to);
900900
break;
901901
}
902902

‎src/libstore/include/nix/store/uds-remote-store.hh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore
6868

6969
void narFromPath(const StorePath & path, Sink & sink) override
7070
{
71-
Store::narFromPath(path, sink);
71+
RemoteStore::narFromPath(path, sink);
7272
}
7373

7474
/**

‎src/libstore/worker-protocol-connection.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,13 @@ WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * so
6969
} else {
7070
auto error = readString(from);
7171
unsigned int status = readInt(from);
72-
ex = std::make_exception_ptr(Error(status, error));
72+
ex = std::make_exception_ptr(Error(status, "[daemon] " + error));
7373
}
7474
break;
7575
}
7676

7777
else if (msg == STDERR_NEXT)
78-
printError(chomp(readString(from)));
78+
printError("[daemon] " + chomp(readString(from)));
7979

8080
else if (msg == STDERR_START_ACTIVITY) {
8181
auto act = readNum<ActivityId>(from);

‎tests/functional/meson.build‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,8 @@ suites = [
8484
'referrers.sh',
8585
'optimise-store.sh',
8686
'substitute-with-invalid-ca.sh',
87+
'socket-only-daemon.sh',
88+
'socket-only-daemon-binary-cache.sh',
8789
'signing.sh',
8890
'hash-convert.sh',
8991
'hash-path.sh',

‎tests/functional/socket-only-daemon-binary-cache.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/usr/bin/env bash
2+
3+
# Run the socket-only-daemon test with a binary cache backend
4+
# This tests that
5+
# - The daemon can serve a binary cache store; not just a local store
6+
# - Client's store operations do not reach into the file system secretly,
7+
# because the files don't exist in the places where a local store would
8+
# put them. (We have NARs instead)
9+
10+
daemon_backing_store_is_binary_cache=1
11+
source socket-only-daemon.sh

‎tests/functional/socket-only-daemon.sh‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
#!/usr/bin/env bash
2+
3+
# Test that a daemon can serve store paths purely over the socket,
4+
# without requiring filesystem access to the store directory.
5+
# This is important for VM setups where the host serves paths to
6+
# the guest via socket, but the store directory is not shared.
7+
#
8+
# Can be called with daemon_backing_store_is_binary_cache=1 to test with a binary cache
9+
# instead of a regular local store. See socket-only-daemon-binary-cache.sh.
10+
11+
source common.sh
12+
13+
needLocalStore "This test requires starting a separate daemon"
14+
15+
# Create state and cache locations
16+
# Note: We use the same NIX_STORE_DIR (logical store path) as the test environment so paths are compatible
17+
remote_cache_dir="$TEST_ROOT/remote-cache-$RANDOM"
18+
remote_state_dir="$TEST_ROOT/remote-state-$RANDOM"
19+
remote_real_store_dir="$TEST_ROOT/remote-real-store-$RANDOM"
20+
remote_socket="$TEST_ROOT/remote-socket-initial"
21+
moved_socket="$TEST_ROOT/moved-socket"
22+
23+
# Set up the remote store URI based on store type
24+
if [[ "${daemon_backing_store_is_binary_cache:-0}" == "1" ]]; then
25+
echo "Using binary cache store"
26+
mkdir -p "$remote_cache_dir"
27+
remote_full_store_uri="file://$remote_cache_dir"
28+
else
29+
echo "Using local store with different physical location"
30+
mkdir -p "$remote_state_dir"
31+
mkdir -p "$remote_real_store_dir"
32+
remote_full_store_uri="local?store=$NIX_STORE_DIR&real=$remote_real_store_dir&state=$remote_state_dir"
33+
fi
34+
35+
# Create a test derivation file
36+
cat > "$TEST_ROOT/test-derivation.nix" <<EOF
37+
with import ${config_nix};
38+
mkDerivation {
39+
name = "socket-test-path";
40+
buildCommand = "echo hello-from-remote > \$out";
41+
}
42+
EOF
43+
44+
# Build a test path in our local store
45+
out=$(nix-build --no-out-link "$TEST_ROOT/test-derivation.nix")
46+
47+
echo "Built path: $out"
48+
49+
# Copy the path to the remote store
50+
nix copy --to "$remote_full_store_uri" --no-check-sigs "$out"
51+
52+
# Verify it exists in the remote store
53+
if [[ "${daemon_backing_store_is_binary_cache:-0}" == "1" ]]; then
54+
cache_hash=$(basename "$out" | cut -d- -f1)
55+
[[ -f "$remote_cache_dir/$cache_hash.narinfo" ]] || fail "Path not in binary cache"
56+
else
57+
[[ -f "$out" ]] || fail "Path not in remote store"
58+
fi
59+
60+
# Start a daemon for the remote store
61+
rm -f "$remote_socket"
62+
NIX_DAEMON_SOCKET_PATH="$remote_socket" \
63+
nix --extra-experimental-features 'nix-command' daemon --store "$remote_full_store_uri" &
64+
remote_daemon_pid=$!
65+
66+
# Ensure daemon is cleaned up on exit
67+
cleanup_daemon() {
68+
if [[ -n "${remote_daemon_pid:-}" ]]; then
69+
kill "$remote_daemon_pid" 2>/dev/null || true
70+
wait "$remote_daemon_pid" 2>/dev/null || true
71+
fi
72+
}
73+
trap cleanup_daemon EXIT
74+
75+
# Wait for socket to appear
76+
for ((i = 0; i < 60; i++)); do
77+
if [[ -S "$remote_socket" ]]; then
78+
daemon_started=1
79+
break
80+
fi
81+
if ! kill -0 "$remote_daemon_pid"; then
82+
fail "Remote daemon died unexpectedly"
83+
fi
84+
sleep 0.1
85+
done
86+
[[ -n "${daemon_started:-}" ]] || fail "Remote daemon didn't start"
87+
88+
echo "Remote daemon started with PID $remote_daemon_pid"
89+
90+
# Move the socket to a different location to prevent any path-based
91+
# assumptions from accidentally working (mildly paranoid, mildly effective;
92+
# ideally we'd use a namespace, but that level of complexity is not actually
93+
# needed)
94+
mv "$remote_socket" "$moved_socket"
95+
96+
echo "Socket moved to: $moved_socket"
97+
98+
# Clear our local store so we need to substitute
99+
clearStore
100+
101+
# Try to copy the path from the daemon via the moved socket
102+
# NOTE: We do NOT pass the store location to the client - only the socket!
103+
# The daemon must be able to serve paths knowing only what's in its own configuration.
104+
nix copy --from "unix://$moved_socket" --no-require-sigs "$out"
105+
106+
# Verify the content
107+
[[ -f "$out" ]] || fail "Output path doesn't exist"
108+
[[ "$(cat "$out")" == "hello-from-remote" ]] || fail "Output content is wrong"
109+
110+
echo "Socket-only copy test PASSED"
111+
112+
# Clear the store again to test substituters mechanism
113+
clearStore
114+
115+
# First verify that --max-jobs 0 without substituters fails (test our assumption)
116+
if nix-build --max-jobs 0 --no-out-link "$TEST_ROOT/test-derivation.nix" 2>/dev/null; then
117+
fail "Building with --max-jobs 0 should have failed without substituters"
118+
fi
119+
120+
echo "Confirmed: --max-jobs 0 without substituters fails as expected"
121+
122+
# Now test using the socket as a substituter with --max-jobs 0 (no building allowed)
123+
# This ensures the substituter mechanism works, not just nix copy
124+
nix-build --max-jobs 0 \
125+
--option substituters "unix://$moved_socket" \
126+
--option require-sigs false \
127+
--no-out-link \
128+
"$TEST_ROOT/test-derivation.nix"
129+
130+
echo "Socket-only substituter test PASSED"
131+
132+
# Test builders mechanism (only on Linux with daemon backed by local store)
133+
# - Builders need sandboxing with namespace support to mount the correct store path
134+
# - Binary cache stores can't build, only serve files
135+
if [[ $(uname) == Linux && "${daemon_backing_store_is_binary_cache:-0}" == "0" ]]; then
136+
# Clear the store again to test builders mechanism
137+
clearStore
138+
139+
# Test using the socket as a remote builder
140+
# This ensures the builders mechanism can also use socket-only connections
141+
nix-build \
142+
--option builders "unix://$moved_socket" \
143+
--option require-sigs false \
144+
--max-jobs 0 \
145+
--no-out-link \
146+
"$TEST_ROOT/test-derivation.nix"
147+
148+
echo "Socket-only builder test PASSED"
149+
fi

0 commit comments

Comments
 (0)