Skip to content

Commit 93a5ef0

Browse files
edolstraedolstra
committed
nix run: Fix chroot execution
Running "nix run" with a diverted store, e.g. $ nix run --store local?root=/tmp/nix nixpkgs.hello stopped working when Nix became multithreaded, because unshare(CLONE_NEWUSER) doesn't work in multithreaded processes. The obvious solution is to terminate all other threads first, but 1) there is no way to terminate Boehm GC marker threads; and 2) it appears that the kernel has a race where unshare(CLONE_NEWUSER) will still fail for some indeterminate amount of time after joining other threads. So instead, "nix run" will now exec() a single-threaded helper ("nix __run_in_chroot") that performs the actual unshare()/chroot()/exec().
1 parent 1c58e13 commit 93a5ef0

File tree

2 files changed

+110
-60
lines changed

2 files changed

+110
-60
lines changed

‎src/nix/main.cc‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@
99
#include "store-api.hh"
1010
#include "progress-bar.hh"
1111

12+
extern std::string chrootHelperName;
13+
14+
void chrootHelper(int argc, char * * argv);
15+
1216
namespace nix {
1317

1418
struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
@@ -57,6 +61,13 @@ void mainWrapped(int argc, char * * argv)
5761
verbosity = lvlError;
5862
settings.verboseBuild = false;
5963

64+
/* The chroot helper needs to be run before any threads have been
65+
started. */
66+
if (argc > 0 && argv[0] == chrootHelperName) {
67+
chrootHelper(argc, argv);
68+
return;
69+
}
70+
6071
initNix();
6172
initGC();
6273

‎src/nix/run.cc‎

Lines changed: 99 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,16 @@
55
#include "derivations.hh"
66
#include "local-store.hh"
77
#include "finally.hh"
8+
#include "fs-accessor.hh"
89

910
#if __linux__
1011
#include <sys/mount.h>
1112
#endif
1213

1314
using namespace nix;
1415

16+
std::string chrootHelperName = "__run_in_chroot";
17+
1518
struct CmdRun : InstallablesCommand
1619
{
1720
CmdRun()
@@ -32,73 +35,109 @@ struct CmdRun : InstallablesCommand
3235
{
3336
auto outPaths = toStorePaths(store, Build);
3437

35-
auto store2 = store.dynamic_pointer_cast<LocalStore>();
36-
37-
if (store2 && store->storeDir != store2->realStoreDir) {
38-
#if __linux__
39-
uid_t uid = getuid();
40-
uid_t gid = getgid();
41-
42-
if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1)
43-
throw SysError("setting up a private mount namespace");
44-
45-
/* Bind-mount realStoreDir on /nix/store. If the latter
46-
mount point doesn't already exists, we have to create a
47-
chroot environment containing the mount point and bind
48-
mounts for the children of /. Would be nice if we could
49-
use overlayfs here, but that doesn't work in a user
50-
namespace yet (Ubuntu has a patch for this:
51-
https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */
52-
if (!pathExists(store->storeDir)) {
53-
// FIXME: Use overlayfs?
54-
55-
Path tmpDir = createTempDir();
56-
57-
createDirs(tmpDir + store->storeDir);
58-
59-
if (mount(store2->realStoreDir.c_str(), (tmpDir + store->storeDir).c_str(), "", MS_BIND, 0) == -1)
60-
throw SysError(format("mounting '%s' on '%s'") % store2->realStoreDir % store->storeDir);
61-
62-
for (auto entry : readDirectory("/")) {
63-
Path dst = tmpDir + "/" + entry.name;
64-
if (pathExists(dst)) continue;
65-
if (mkdir(dst.c_str(), 0700) == -1)
66-
throw SysError(format("creating directory '%s'") % dst);
67-
if (mount(("/" + entry.name).c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1)
68-
throw SysError(format("mounting '%s' on '%s'") % ("/" + entry.name) % dst);
69-
}
70-
71-
char * cwd = getcwd(0, 0);
72-
if (!cwd) throw SysError("getting current directory");
73-
Finally freeCwd([&]() { free(cwd); });
74-
75-
if (chroot(tmpDir.c_str()) == -1)
76-
throw SysError(format("chrooting into '%s'") % tmpDir);
77-
78-
if (chdir(cwd) == -1)
79-
throw SysError(format("chdir to '%s' in chroot") % cwd);
80-
} else
81-
if (mount(store2->realStoreDir.c_str(), store->storeDir.c_str(), "", MS_BIND, 0) == -1)
82-
throw SysError(format("mounting '%s' on '%s'") % store2->realStoreDir % store->storeDir);
83-
84-
writeFile("/proc/self/setgroups", "deny");
85-
writeFile("/proc/self/uid_map", (format("%d %d %d") % uid % uid % 1).str());
86-
writeFile("/proc/self/gid_map", (format("%d %d %d") % gid % gid % 1).str());
87-
#else
88-
throw Error(format("mounting the Nix store on '%s' is not supported on this platform") % store->storeDir);
89-
#endif
90-
}
91-
38+
auto accessor = store->getFSAccessor();
9239

9340
auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":");
9441
for (auto & path : outPaths)
95-
if (pathExists(path + "/bin"))
42+
if (accessor->stat(path + "/bin").type != FSAccessor::tMissing)
9643
unixPath.push_front(path + "/bin");
9744
setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);
9845

99-
if (execlp("bash", "bash", nullptr) == -1)
100-
throw SysError("unable to exec 'bash'");
46+
std::string cmd = "bash";
47+
Strings args = { cmd };
48+
49+
/* If this is a diverted store (i.e. its "logical" location
50+
(typically /nix/store) differs from its "physical" location
51+
(e.g. /home/eelco/nix/store), then run the command in a
52+
chroot. For non-root users, this requires running it in new
53+
mount and user namespaces. Unfortunately,
54+
unshare(CLONE_NEWUSER) doesn't work in a multithreaded
55+
program (which "nix" is), so we exec() a single-threaded
56+
helper program (chrootHelper() below) to do the work. */
57+
auto store2 = store.dynamic_pointer_cast<LocalStore>();
58+
59+
if (store2 && store->storeDir != store2->realStoreDir) {
60+
Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd };
61+
for (auto & arg : args) helperArgs.push_back(arg);
62+
63+
execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());
64+
65+
throw SysError("could not execute chroot helper");
66+
}
67+
68+
execvp(cmd.c_str(), stringsToCharPtrs(args).data());
69+
70+
throw SysError("unable to exec '%s'", cmd);
10171
}
10272
};
10373

10474
static RegisterCommand r1(make_ref<CmdRun>());
75+
76+
void chrootHelper(int argc, char * * argv)
77+
{
78+
int p = 1;
79+
std::string storeDir = argv[p++];
80+
std::string realStoreDir = argv[p++];
81+
std::string cmd = argv[p++];
82+
Strings args;
83+
while (p < argc)
84+
args.push_back(argv[p++]);
85+
86+
#if __linux__
87+
uid_t uid = getuid();
88+
uid_t gid = getgid();
89+
90+
if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1)
91+
throw SysError("setting up a private mount namespace");
92+
93+
/* Bind-mount realStoreDir on /nix/store. If the latter mount
94+
point doesn't already exists, we have to create a chroot
95+
environment containing the mount point and bind mounts for the
96+
children of /. Would be nice if we could use overlayfs here,
97+
but that doesn't work in a user namespace yet (Ubuntu has a
98+
patch for this:
99+
https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */
100+
if (true /* !pathExists(storeDir) */) {
101+
// FIXME: Use overlayfs?
102+
103+
Path tmpDir = createTempDir();
104+
105+
createDirs(tmpDir + storeDir);
106+
107+
if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1)
108+
throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir);
109+
110+
for (auto entry : readDirectory("/")) {
111+
Path dst = tmpDir + "/" + entry.name;
112+
if (pathExists(dst)) continue;
113+
if (mkdir(dst.c_str(), 0700) == -1)
114+
throw SysError(format("creating directory '%s'") % dst);
115+
if (mount(("/" + entry.name).c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1)
116+
throw SysError(format("mounting '%s' on '%s'") % ("/" + entry.name) % dst);
117+
}
118+
119+
char * cwd = getcwd(0, 0);
120+
if (!cwd) throw SysError("getting current directory");
121+
Finally freeCwd([&]() { free(cwd); });
122+
123+
if (chroot(tmpDir.c_str()) == -1)
124+
throw SysError(format("chrooting into '%s'") % tmpDir);
125+
126+
if (chdir(cwd) == -1)
127+
throw SysError(format("chdir to '%s' in chroot") % cwd);
128+
} else
129+
if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1)
130+
throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir);
131+
132+
writeFile("/proc/self/setgroups", "deny");
133+
writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1));
134+
writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1));
135+
136+
execvp(cmd.c_str(), stringsToCharPtrs(args).data());
137+
138+
throw SysError("unable to exec '%s'", cmd);
139+
140+
#else
141+
throw Error("mounting the Nix store on '%s' is not supported on this platform", >storeDir);
142+
#endif
143+
}

0 commit comments

Comments
 (0)