Skip to content

Commit 5e65584

Browse files
authored
Merge pull request #14145 from NixOS/external-derivation-builder
External derivation builders
2 parents e5ae81c + e7e2ac9 commit 5e65584

File tree

8 files changed

+310
-18
lines changed

8 files changed

+310
-18
lines changed

src/libstore/globals.cc

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -341,10 +341,15 @@ PathsInChroot BaseSetting<PathsInChroot>::parse(const std::string & str) const
341341
i.pop_back();
342342
}
343343
size_t p = i.find('=');
344-
if (p == std::string::npos)
345-
pathsInChroot[i] = {.source = i, .optional = optional};
346-
else
347-
pathsInChroot[i.substr(0, p)] = {.source = i.substr(p + 1), .optional = optional};
344+
std::string inside, outside;
345+
if (p == std::string::npos) {
346+
inside = i;
347+
outside = i;
348+
} else {
349+
inside = i.substr(0, p);
350+
outside = i.substr(p + 1);
351+
}
352+
pathsInChroot[inside] = {.source = outside, .optional = optional};
348353
}
349354
return pathsInChroot;
350355
}
@@ -374,6 +379,24 @@ unsigned int MaxBuildJobsSetting::parse(const std::string & str) const
374379
}
375380
}
376381

382+
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Settings::ExternalBuilder, systems, program, args);
383+
384+
template<>
385+
Settings::ExternalBuilders BaseSetting<Settings::ExternalBuilders>::parse(const std::string & str) const
386+
{
387+
try {
388+
return nlohmann::json::parse(str).template get<Settings::ExternalBuilders>();
389+
} catch (std::exception & e) {
390+
throw UsageError("parsing setting '%s': %s", name, e.what());
391+
}
392+
}
393+
394+
template<>
395+
std::string BaseSetting<Settings::ExternalBuilders>::to_string() const
396+
{
397+
return nlohmann::json(value).dump();
398+
}
399+
377400
template<>
378401
void BaseSetting<PathsInChroot>::appendOrSet(PathsInChroot newValue, bool append)
379402
{

src/libstore/include/nix/store/globals.hh

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1372,6 +1372,77 @@ public:
13721372
Default is 0, which disables the warning.
13731373
Set it to 1 to warn on all paths.
13741374
)"};
1375+
1376+
struct ExternalBuilder
1377+
{
1378+
std::vector<std::string> systems;
1379+
Path program;
1380+
std::vector<std::string> args;
1381+
};
1382+
1383+
using ExternalBuilders = std::vector<ExternalBuilder>;
1384+
1385+
Setting<ExternalBuilders> externalBuilders{
1386+
this,
1387+
{},
1388+
"external-builders",
1389+
R"(
1390+
Helper programs that execute derivations.
1391+
1392+
The program is passed a JSON document that describes the build environment as the final argument.
1393+
The JSON document looks like this:
1394+
1395+
{
1396+
"args": [
1397+
"-e",
1398+
"/nix/store/vj1c3wf9…-source-stdenv.sh",
1399+
"/nix/store/shkw4qm9…-default-builder.sh"
1400+
],
1401+
"builder": "/nix/store/s1qkj0ph…-bash-5.2p37/bin/bash",
1402+
"env": {
1403+
"HOME": "/homeless-shelter",
1404+
"builder": "/nix/store/s1qkj0ph…-bash-5.2p37/bin/bash",
1405+
"nativeBuildInputs": "/nix/store/l31j72f1…-version-check-hook",
1406+
"out": "/nix/store/2yx2prgx…-hello-2.12.2"
1407+
1408+
},
1409+
"inputPaths": [
1410+
"/nix/store/14dciax3…-glibc-2.32-54-dev",
1411+
"/nix/store/1azs5s8z…-gettext-0.21",
1412+
1413+
],
1414+
"outputs": {
1415+
"out": "/nix/store/2yx2prgx…-hello-2.12.2"
1416+
},
1417+
"realStoreDir": "/nix/store",
1418+
"storeDir": "/nix/store",
1419+
"system": "aarch64-linux",
1420+
"tmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0/build",
1421+
"tmpDirInSandbox": "/build",
1422+
"topTmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0",
1423+
"version": 1
1424+
}
1425+
)",
1426+
{}, // aliases
1427+
true, // document default
1428+
// NOTE(cole-h): even though we can make the experimental feature required here, the errors
1429+
// are not as good (it just becomes a warning if you try to use this setting without the
1430+
// experimental feature)
1431+
//
1432+
// With this commented out:
1433+
//
1434+
// error: experimental Nix feature 'external-builders' is disabled; add '--extra-experimental-features
1435+
// external-builders' to enable it
1436+
//
1437+
// With this uncommented:
1438+
//
1439+
// warning: Ignoring setting 'external-builders' because experimental feature 'external-builders' is not enabled
1440+
// error: Cannot build '/nix/store/vwsp4qd8…-opentofu-1.10.2.drv'.
1441+
// Reason: required system or feature not available
1442+
// Required system: 'aarch64-linux' with features {}
1443+
// Current system: 'aarch64-darwin' with features {apple-virt, benchmark, big-parallel, nixos-test}
1444+
// Xp::ExternalBuilders
1445+
};
13751446
};
13761447

13771448
// FIXME: don't use a global variable.

src/libstore/unix/build/derivation-builder.cc

Lines changed: 29 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,12 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder
229229
return acquireUserLock(1, false);
230230
}
231231

232+
/**
233+
* Throw an exception if we can't do this derivation because of
234+
* missing system features.
235+
*/
236+
virtual void checkSystem();
237+
232238
/**
233239
* Return the paths that should be made available in the sandbox.
234240
* This includes:
@@ -666,21 +672,8 @@ static bool checkNotWorldWritable(std::filesystem::path path)
666672
return true;
667673
}
668674

669-
std::optional<Descriptor> DerivationBuilderImpl::startBuild()
675+
void DerivationBuilderImpl::checkSystem()
670676
{
671-
if (useBuildUsers()) {
672-
if (!buildUser)
673-
buildUser = getBuildUser();
674-
675-
if (!buildUser)
676-
return std::nullopt;
677-
}
678-
679-
/* Make sure that no other processes are executing under the
680-
sandbox uids. This must be done before any chownToBuilder()
681-
calls. */
682-
prepareUser();
683-
684677
/* Right platform? */
685678
if (!drvOptions.canBuildLocally(store, drv)) {
686679
auto msg =
@@ -704,6 +697,24 @@ std::optional<Descriptor> DerivationBuilderImpl::startBuild()
704697

705698
throw BuildError(BuildResult::Failure::InputRejected, msg);
706699
}
700+
}
701+
702+
std::optional<Descriptor> DerivationBuilderImpl::startBuild()
703+
{
704+
if (useBuildUsers()) {
705+
if (!buildUser)
706+
buildUser = getBuildUser();
707+
708+
if (!buildUser)
709+
return std::nullopt;
710+
}
711+
712+
checkSystem();
713+
714+
/* Make sure that no other processes are executing under the
715+
sandbox uids. This must be done before any chownToBuilder()
716+
calls. */
717+
prepareUser();
707718

708719
auto buildDir = store.config->getBuildDir();
709720

@@ -1904,12 +1915,16 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path)
19041915
#include "chroot-derivation-builder.cc"
19051916
#include "linux-derivation-builder.cc"
19061917
#include "darwin-derivation-builder.cc"
1918+
#include "external-derivation-builder.cc"
19071919

19081920
namespace nix {
19091921

19101922
std::unique_ptr<DerivationBuilder> makeDerivationBuilder(
19111923
LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> miscMethods, DerivationBuilderParams params)
19121924
{
1925+
if (auto builder = ExternalDerivationBuilder::newIfSupported(store, miscMethods, params))
1926+
return builder;
1927+
19131928
bool useSandbox = false;
19141929

19151930
/* Are we doing a sandboxed build? */
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
namespace nix {
2+
3+
struct ExternalDerivationBuilder : DerivationBuilderImpl
4+
{
5+
Settings::ExternalBuilder externalBuilder;
6+
7+
ExternalDerivationBuilder(
8+
LocalStore & store,
9+
std::unique_ptr<DerivationBuilderCallbacks> miscMethods,
10+
DerivationBuilderParams params,
11+
Settings::ExternalBuilder externalBuilder)
12+
: DerivationBuilderImpl(store, std::move(miscMethods), std::move(params))
13+
, externalBuilder(std::move(externalBuilder))
14+
{
15+
experimentalFeatureSettings.require(Xp::ExternalBuilders);
16+
}
17+
18+
static std::unique_ptr<ExternalDerivationBuilder> newIfSupported(
19+
LocalStore & store, std::unique_ptr<DerivationBuilderCallbacks> & miscMethods, DerivationBuilderParams & params)
20+
{
21+
for (auto & handler : settings.externalBuilders.get()) {
22+
for (auto & system : handler.systems)
23+
if (params.drv.platform == system)
24+
return std::make_unique<ExternalDerivationBuilder>(
25+
store, std::move(miscMethods), std::move(params), handler);
26+
}
27+
return {};
28+
}
29+
30+
Path tmpDirInSandbox() override
31+
{
32+
/* In a sandbox, for determinism, always use the same temporary
33+
directory. */
34+
return "/build";
35+
}
36+
37+
void setBuildTmpDir() override
38+
{
39+
tmpDir = topTmpDir + "/build";
40+
createDir(tmpDir, 0700);
41+
}
42+
43+
void checkSystem() override {}
44+
45+
void startChild() override
46+
{
47+
if (drvOptions.getRequiredSystemFeatures(drv).count("recursive-nix"))
48+
throw Error("'recursive-nix' is not supported yet by external derivation builders");
49+
50+
auto json = nlohmann::json::object();
51+
52+
json.emplace("version", 1);
53+
json.emplace("builder", drv.builder);
54+
{
55+
auto l = nlohmann::json::array();
56+
for (auto & i : drv.args)
57+
l.push_back(rewriteStrings(i, inputRewrites));
58+
json.emplace("args", std::move(l));
59+
}
60+
{
61+
auto j = nlohmann::json::object();
62+
for (auto & [name, value] : env)
63+
j.emplace(name, rewriteStrings(value, inputRewrites));
64+
json.emplace("env", std::move(j));
65+
}
66+
json.emplace("topTmpDir", topTmpDir);
67+
json.emplace("tmpDir", tmpDir);
68+
json.emplace("tmpDirInSandbox", tmpDirInSandbox());
69+
json.emplace("storeDir", store.storeDir);
70+
json.emplace("realStoreDir", store.config->realStoreDir.get());
71+
json.emplace("system", drv.platform);
72+
{
73+
auto l = nlohmann::json::array();
74+
for (auto & i : inputPaths)
75+
l.push_back(store.printStorePath(i));
76+
json.emplace("inputPaths", std::move(l));
77+
}
78+
{
79+
auto l = nlohmann::json::object();
80+
for (auto & i : scratchOutputs)
81+
l.emplace(i.first, store.printStorePath(i.second));
82+
json.emplace("outputs", std::move(l));
83+
}
84+
85+
// TODO(cole-h): writing this to stdin is too much effort right now, if we want to revisit
86+
// that, see this comment by Eelco about how to make it not suck:
87+
// https://github.com/DeterminateSystems/nix-src/pull/141#discussion_r2205493257
88+
auto jsonFile = std::filesystem::path{topTmpDir} / "build.json";
89+
writeFile(jsonFile, json.dump());
90+
91+
pid = startProcess([&]() {
92+
openSlave();
93+
try {
94+
commonChildInit();
95+
96+
Strings args = {externalBuilder.program};
97+
98+
if (!externalBuilder.args.empty()) {
99+
args.insert(args.end(), externalBuilder.args.begin(), externalBuilder.args.end());
100+
}
101+
102+
args.insert(args.end(), jsonFile);
103+
104+
if (chdir(tmpDir.c_str()) == -1)
105+
throw SysError("changing into '%1%'", tmpDir);
106+
107+
chownToBuilder(topTmpDir);
108+
109+
setUser();
110+
111+
debug("executing external builder: %s", concatStringsSep(" ", args));
112+
execv(externalBuilder.program.c_str(), stringsToCharPtrs(args).data());
113+
114+
throw SysError("executing '%s'", externalBuilder.program);
115+
} catch (...) {
116+
handleChildException(true);
117+
_exit(1);
118+
}
119+
});
120+
}
121+
};
122+
123+
} // namespace nix

src/libutil/experimental-features.cc

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,14 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
304304
)",
305305
.trackingUrl = "https://github.com/NixOS/nix/milestone/55",
306306
},
307+
{
308+
.tag = Xp::ExternalBuilders,
309+
.name = "external-builders",
310+
.description = R"(
311+
Enables support for external builders / sandbox providers.
312+
)",
313+
.trackingUrl = "",
314+
},
307315
{
308316
.tag = Xp::BLAKE3Hashes,
309317
.name = "blake3-hashes",

src/libutil/include/nix/util/experimental-features.hh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ enum struct ExperimentalFeature {
3737
MountedSSHStore,
3838
VerifiedFetches,
3939
PipeOperators,
40+
ExternalBuilders,
4041
BLAKE3Hashes,
4142
};
4243

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/usr/bin/env bash
2+
3+
source common.sh
4+
5+
TODO_NixOS
6+
7+
needLocalStore "'--external-builders' can’t be used with the daemon"
8+
9+
expr="$TEST_ROOT/expr.nix"
10+
cat > "$expr" <<EOF
11+
with import ${config_nix};
12+
mkDerivation {
13+
name = "external";
14+
system = "x68_46-xunil";
15+
buildCommand = ''
16+
echo xyzzy
17+
printf foo > \$out
18+
'';
19+
}
20+
EOF
21+
22+
external_builder="$TEST_ROOT/external-builder.sh"
23+
cat > "$external_builder" <<EOF
24+
#! $SHELL -e
25+
26+
PATH=$PATH
27+
28+
[[ "\$1" = bla ]]
29+
30+
system="\$(jq -r .system < "\$2")"
31+
builder="\$(jq -r .builder < "\$2")"
32+
args="\$(jq -r '.args | join(" ")' < "\$2")"
33+
export buildCommand="\$(jq -r .env.buildCommand < "\$2")"
34+
export out="\$(jq -r .env.out < "\$2")"
35+
[[ \$system = x68_46-xunil ]]
36+
37+
printf "\2\n"
38+
39+
# In a real external builder, we would now call something like qemu to emulate the system.
40+
"\$builder" \$args
41+
42+
printf bar >> \$out
43+
EOF
44+
chmod +x "$external_builder"
45+
46+
nix build -L --file "$expr" --out-link "$TEST_ROOT/result" \
47+
--extra-experimental-features external-builders \
48+
--external-builders "[{\"systems\": [\"x68_46-xunil\"], \"args\": [\"bla\"], \"program\": \"$external_builder\"}]"
49+
50+
[[ $(cat "$TEST_ROOT/result") = foobar ]]

0 commit comments

Comments
 (0)