Skip to content

Commit 1547160

Browse files
astroastro
committed
nixos-modules: move virtiofsd from host to microvm
1 parent e8d5f12 commit 1547160

File tree

7 files changed

+169
-95
lines changed

7 files changed

+169
-95
lines changed

‎lib/runner.nix

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
let
77
inherit (pkgs) lib;
88

9+
inherit (microvmConfig) virtiofsdScripts;
10+
911
inherit (import ./. { inherit lib; }) createVolumesScript makeMacvtap;
1012
inherit (makeMacvtap {
1113
inherit microvmConfig hypervisorConfig;
@@ -54,7 +56,7 @@ pkgs.buildPackages.runCommand "microvm-${microvmConfig.hypervisor}-${microvmConf
5456
# for `nix run`
5557
meta.mainProgram = "microvm-run";
5658
passthru = {
57-
inherit canShutdown supportsNotifySocket;
59+
inherit canShutdown supportsNotifySocket tapMultiQueue;
5860
inherit (microvmConfig) hypervisor;
5961
};
6062
} ''
@@ -68,6 +70,16 @@ pkgs.buildPackages.runCommand "microvm-${microvmConfig.hypervisor}-${microvmConf
6870
ln -s ${balloonScriptBin}/bin/microvm-balloon $out/bin/microvm-balloon
6971
''}
7072
73+
${lib.optionalString (virtiofsdScripts.run != null) ''
74+
ln -s ${lib.getExe virtiofsdScripts.run} $out/bin/virtiofsd-run
75+
''}
76+
${lib.optionalString (virtiofsdScripts.reload != null) ''
77+
ln -s ${lib.getExe virtiofsdScripts.reload} $out/bin/virtiofsd-reload
78+
''}
79+
${lib.optionalString (virtiofsdScripts.shutdown != null) ''
80+
ln -s ${lib.getExe virtiofsdScripts.shutdown} $out/bin/virtiofsd-shutdown
81+
''}
82+
7183
mkdir -p $out/share/microvm
7284
ln -s ${toplevel} $out/share/microvm/system
7385

‎nixos-modules/host/default.nix

Lines changed: 33 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -250,68 +250,41 @@ in
250250
'';
251251
};
252252

253-
"microvm-virtiofsd@" = let
254-
inherit (pkgs.python3Packages) supervisor;
255-
in rec {
256-
description = "VirtioFS daemons for MicroVM '%i'";
257-
before = [ "microvm@%i.service" ];
258-
after = [ "local-fs.target" ];
259-
partOf = [ "microvm@%i.service" ];
260-
unitConfig.ConditionPathExists = "${stateDir}/%i/current/share/microvm/virtiofs";
261-
restartIfChanged = false;
262-
serviceConfig = {
263-
ExecReload = "${lib.getExe' supervisor "supervisorctl"} reload";
264-
ExecStop = "${lib.getExe' supervisor "supervisorctl"} shutdown";
265-
LimitNOFILE = 1048576;
266-
NotifyAccess = "all";
267-
PrivateTmp = "yes";
268-
Restart = "always";
269-
RestartSec = "5s";
270-
SyslogIdentifier = "microvm-virtiofsd@%i";
271-
Type = "notify";
272-
WorkingDirectory = "${stateDir}/%i";
273-
};
274-
script = ''
275-
echo "[supervisord]
276-
nodaemon=true
277-
user=root
253+
"microvm-virtiofsd@" =
254+
let
255+
runFromBootedOrCurrent = pkgs.writeShellScript "microvm-runFromBootedOrCurrent" ''
256+
NAME="$1"
257+
VM="$2"
258+
cd "${stateDir}/$VM"
278259
279-
[eventlistener:notify]
280-
command=${pkgs.writers.writePython3 "supervisord-event-handler" { } (lib.readFile ./supervisord-event-handler.py)}
281-
events=PROCESS_STATE
282-
" > /tmp/supervisord.conf
283-
284-
virtiofsd_count=0
285-
286-
for d in $PWD/current/share/microvm/virtiofs/*; do
287-
SOCKET="$(realpath "$(cat $d/socket)")"
288-
SOURCE="$(cat $d/source)"
289-
mkdir -p "$SOURCE"
290-
291-
group_programs+="virtiofsd-$(basename "$d"),"
292-
virtiofsd_count=$((virtiofsd_count+1))
293-
294-
echo "[program:virtiofsd-$(basename "$d")]
295-
stderr_syslog=true
296-
stdout_syslog=true
297-
command=${lib.getExe pkgs.virtiofsd} \
298-
--socket-path=$SOCKET \
299-
--socket-group=${config.users.users.microvm.group} \
300-
--shared-dir=\"$SOURCE\" \
301-
--rlimit-nofile ${toString serviceConfig.LimitNOFILE} \
302-
--thread-pool-size ${toString config.microvm.virtiofsd.threadPoolSize} \
303-
--posix-acl --xattr \
304-
${lib.optionalString (config.microvm.virtiofsd.inodeFileHandles != null)
305-
"--inode-file-handles=${config.microvm.virtiofsd.inodeFileHandles}"
306-
} \
307-
${lib.concatStringsSep " " config.microvm.virtiofsd.extraArgs}
308-
" >> /tmp/supervisord.conf
309-
done
260+
if [ -e booted ]; then
261+
exec booted/bin/$NAME
262+
else
263+
exec current/bin/$NAME
264+
fi
265+
'';
310266

311-
echo -n $virtiofsd_count > /tmp/virtiofsd_count
312-
exec ${lib.getExe' supervisor "supervisord"} --configuration /tmp/supervisord.conf
313-
'';
314-
};
267+
in {
268+
description = "VirtioFS daemons for MicroVM '%i'";
269+
before = [ "microvm@%i.service" ];
270+
after = [ "local-fs.target" ];
271+
partOf = [ "microvm@%i.service" ];
272+
unitConfig.ConditionPathExists = "${stateDir}/%i/current/bin/virtiofsd-run";
273+
restartIfChanged = false;
274+
serviceConfig = {
275+
WorkingDirectory = "${stateDir}/%i";
276+
ExecStart = "${stateDir}/%i/current/bin/virtiofsd-run";
277+
ExecReload = "${runFromBootedOrCurrent} virtiofsd-reload %i";
278+
ExecStop = "${runFromBootedOrCurrent} virtiofsd-shutdown %i";
279+
LimitNOFILE = 1048576;
280+
NotifyAccess = "all";
281+
PrivateTmp = "yes";
282+
Restart = "always";
283+
RestartSec = "5s";
284+
SyslogIdentifier = "microvm-virtiofsd@%i";
285+
Type = "notify";
286+
};
287+
};
315288

316289
"microvm@" = {
317290
description = "MicroVM '%i'";

‎nixos-modules/host/options.nix

Lines changed: 0 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -171,37 +171,5 @@
171171
This includes declarative `config.microvm.vms` as well as MicroVMs that are managed through the `microvm` command.
172172
'';
173173
};
174-
175-
virtiofsd.inodeFileHandles = mkOption {
176-
type = with types; nullOr (enum [
177-
"never" "prefer" "mandatory"
178-
]);
179-
default = null;
180-
description = ''
181-
When to use file handles to reference inodes instead of O_PATH file descriptors
182-
(never, prefer, mandatory)
183-
184-
Allows you to overwrite default behavior in case you hit "too
185-
many open files" on eg. ZFS.
186-
<https://gitlab.com/virtio-fs/virtiofsd/-/issues/121>
187-
'';
188-
};
189-
190-
virtiofsd.threadPoolSize = mkOption {
191-
type = with types; oneOf [ str ints.unsigned ];
192-
default = "`nproc`";
193-
description = ''
194-
The amounts of threads virtiofsd should spawn. This option also takes the special
195-
string `\`nproc\`` which spawns as many threads as the host has cores.
196-
'';
197-
};
198-
199-
virtiofsd.extraArgs = mkOption {
200-
type = with types; listOf str;
201-
default = [];
202-
description = ''
203-
Extra command-line switch to pass to virtiofsd.
204-
'';
205-
};
206174
};
207175
}

‎nixos-modules/microvm/default.nix

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ in
1515
./asserts.nix
1616
./system.nix
1717
./mounts.nix
18+
./virtiofsd
1819
./graphics.nix
1920
./optimization.nix
2021
./ssh-deploy.nix

‎nixos-modules/microvm/options.nix

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,38 @@ in
472472
'';
473473
};
474474

475+
virtiofsd.inodeFileHandles = mkOption {
476+
type = with types; nullOr (enum [
477+
"never" "prefer" "mandatory"
478+
]);
479+
default = null;
480+
description = ''
481+
When to use file handles to reference inodes instead of O_PATH file descriptors
482+
(never, prefer, mandatory)
483+
484+
Allows you to overwrite default behavior in case you hit "too
485+
many open files" on eg. ZFS.
486+
<https://gitlab.com/virtio-fs/virtiofsd/-/issues/121>
487+
'';
488+
};
489+
490+
virtiofsd.threadPoolSize = mkOption {
491+
type = with types; oneOf [ str ints.unsigned ];
492+
default = "`nproc`";
493+
description = ''
494+
The amounts of threads virtiofsd should spawn. This option also takes the special
495+
string `\`nproc\`` which spawns as many threads as the host has cores.
496+
'';
497+
};
498+
499+
virtiofsd.extraArgs = mkOption {
500+
type = with types; listOf str;
501+
default = [];
502+
description = ''
503+
Extra command-line switch to pass to virtiofsd.
504+
'';
505+
};
506+
475507
runner = mkOption {
476508
description = "Generated Hypervisor runner for this NixOS";
477509
type = with types; attrsOf package;
@@ -483,6 +515,26 @@ in
483515
default = config.microvm.runner.${config.microvm.hypervisor};
484516
defaultText = literalExpression ''"config.microvm.runner.''${config.microvm.hypervisor}"'';
485517
};
518+
519+
virtiofsdScripts = {
520+
run = mkOption {
521+
description = "Generated script to run required virtiofsd instances";
522+
type = with types; nullOr package;
523+
default = null;
524+
};
525+
reload = mkOption {
526+
description = ''
527+
Generated script to reload the supervisor configuration that runs the virtiofsd instances
528+
'';
529+
type = with types; nullOr package;
530+
default = null;
531+
};
532+
shutdown = mkOption {
533+
description = "Generated script to stop the virtiofsd instances";
534+
type = with types; nullOr package;
535+
default = null;
536+
};
537+
};
486538
};
487539

488540
config = lib.mkMerge [ {

‎nixos-modules/microvm/virtiofsd/default.nix

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
{ config, lib, pkgs, ... }:
2+
3+
let
4+
virtiofsShares = builtins.filter ({ proto, ... }:
5+
proto == "virtiofs"
6+
) config.microvm.shares;
7+
8+
requiresVirtiofsd = virtiofsShares != [];
9+
10+
inherit (pkgs.python3Packages) supervisor;
11+
supervisord = lib.getExe' supervisor "supervisord";
12+
supervisorctl = lib.getExe' supervisor "supervisorctl";
13+
14+
in
15+
{
16+
microvm.virtiofsdScripts = lib.mkIf requiresVirtiofsd {
17+
run =
18+
let
19+
supervisordConfig = pkgs.writeText "${config.networking.hostName}-virtiofsd-supervisord.conf" ''
20+
[supervisord]
21+
nodaemon=true
22+
23+
[eventlistener:notify]
24+
command=${pkgs.writers.writePython3 "supervisord-event-handler" { } (
25+
pkgs.substituteAll {
26+
src = ./supervisord-event-handler.py;
27+
virtiofsdCount = builtins.length virtiofsShares;
28+
}
29+
)}
30+
events=PROCESS_STATE
31+
32+
${lib.concatMapStrings ({ proto, tag, socket, source, ... }: ''
33+
[program:virtiofsd-${tag}]
34+
stderr_syslog=true
35+
stdout_syslog=true
36+
autorestart=true
37+
command=${pkgs.writeShellScript "virtiofsd-${tag}" ''
38+
if [ $(id -u) = 0 ]; then
39+
OPT_RLIMIT="--rlimit-nofile 1048576"
40+
else
41+
OPT_RLIMIT=""
42+
fi
43+
exec ${lib.getExe pkgs.virtiofsd} \
44+
--socket-path=${lib.escapeShellArg socket} \
45+
--socket-group=$(id -gn) \
46+
--shared-dir=${lib.escapeShellArg source} \
47+
$OPT_RLIMIT \
48+
--thread-pool-size ${toString config.microvm.virtiofsd.threadPoolSize} \
49+
--posix-acl --xattr \
50+
${lib.optionalString (config.microvm.virtiofsd.inodeFileHandles != null)
51+
"--inode-file-handles=${config.microvm.virtiofsd.inodeFileHandles}"
52+
} \
53+
${lib.concatStringsSep " " config.microvm.virtiofsd.extraArgs}
54+
''}
55+
'' ) virtiofsShares}
56+
'';
57+
in pkgs.writeShellScriptBin "run-virtiofsd" ''
58+
exec ${supervisord} --configuration ${supervisordConfig}
59+
'';
60+
61+
reload = pkgs.writeShellScriptBin "reload-virtiofsd" ''
62+
exec ${supervisorctl} reload
63+
'';
64+
65+
shutdown = pkgs.writeShellScriptBin "shutdown-virtiofsd" ''
66+
exec ${supervisorctl} stop
67+
'';
68+
};
69+
}

‎nixos-modules/host/supervisord-event-handler.py renamed to ‎nixos-modules/microvm/virtiofsd/supervisord-event-handler.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,7 @@ def write_stderr(s):
1515

1616
def main():
1717
count = 0
18-
with open('/tmp/virtiofsd_count') as f:
19-
expected_count = int(f.read())
18+
expected_count = @virtiofsdCount@
2019

2120
while True:
2221
write_stdout('READY\n')

0 commit comments

Comments
 (0)