WIP libstore: control parallel builds with gnu make jobserver protocol

src/libstore/globals.cc

@@ -50,6 +50,9 @@ Settings settings;
 
 static GlobalConfig::Register rSettings(&settings);
 
+// Global jobserver FIFO path (set by daemon, used by derivation-builder)
+std::string jobserverFifoPath;
+
 Settings::Settings()
     : nixPrefix(NIX_PREFIX)
     , nixStore(

src/libstore/include/nix/store/globals.hh

@@ -207,6 +207,24 @@ public:
         )",
         {"build-cores"}};
 
+    Setting<bool> useJobserver{
+        this,
+        false,
+        "use-jobserver",
+        R"(
+          Enable the GNU Make jobserver protocol for coordinated parallelism across builds. This uses a token-based system to limit total parallelism.
+
+          When enabled, Nix creates a FIFO with a limited number of tokens. Build tools that support the jobserver protocol can coordinate parallelism through this shared resource.
+        )"};
+
+    Setting<unsigned int> jobserverTokens{
+        this,
+        0,
+        "jobserver-tokens",
+        R"(
+          Number of jobserver tokens to create. If set to 0 (default), uses the value of 'cores' or the number of available CPU cores.
+        )"};
+
     /**
      * Read-only mode.  Don't copy stuff to the store, don't change
      * the database.
@@ -1447,6 +1465,9 @@ public:
 // FIXME: don't use a global variable.
 extern Settings settings;
 
+// Global jobserver FIFO path (set by daemon, used by derivation-builder)
+extern std::string jobserverFifoPath;
+
 /**
  * Load the configuration (from `nix.conf`, `NIX_CONFIG`, etc.) into the
  * given configuration object.

src/libstore/unix/build/derivation-builder.cc

@@ -29,6 +29,8 @@
 #include <sys/mman.h>
 #include <sys/resource.h>
 #include <sys/socket.h>
+#include <sys/stat.h>
+#include <errno.h>
 
 #include "store-config-private.hh"
 
@@ -853,6 +855,15 @@ PathsInChroot DerivationBuilderImpl::getPathsInSandbox()
     }
     pathsInChroot[tmpDirInSandbox()] = {.source = tmpDir};
 
+    /* Add the jobserver FIFO to sandbox paths if jobserver is enabled */
+    if (settings.useJobserver && !jobserverFifoPath.empty()) {
+        struct stat st;
+        if (stat(jobserverFifoPath.c_str(), &st) == 0 && S_ISFIFO(st.st_mode)) {
+            pathsInChroot[jobserverFifoPath] = {.source = jobserverFifoPath};
+            printMsg(lvlDebug, "adding jobserver FIFO %s to sandbox paths", jobserverFifoPath);
+        }
+    }
+
     PathSet allowedPaths = settings.allowedImpureHostPrefixes;
 
     /* This works like the above, except on a per-derivation level */
@@ -1042,6 +1053,21 @@ void DerivationBuilderImpl::initEnv()
     /* The maximum number of cores to utilize for parallel building. */
     env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores ? settings.buildCores : settings.getDefaultCores());
 
+    /* GNU Make Jobserver Protocol Support
+     * Set MAKEFLAGS to enable jobserver for build processes. */
+    if (settings.useJobserver && !jobserverFifoPath.empty()) {
+        struct stat st;
+        if (stat(jobserverFifoPath.c_str(), &st) == 0 && S_ISFIFO(st.st_mode)) {
+            // Append to existing MAKEFLAGS (preserve any existing flags)
+            auto it = env.find("MAKEFLAGS");
+            std::string makeflags = (it != env.end() && !it->second.empty()) ? it->second + " " : "";
+            makeflags += fmt("--jobserver-auth=fifo:%s -j", jobserverFifoPath);
+            env["MAKEFLAGS"] = makeflags;
+
+            printMsg(lvlDebug, "jobserver: set MAKEFLAGS for build %s", store.printStorePath(drvPath));
+        }
+    }
+
     /* Write the final environment. Note that this is intentionally
        *not* `drv.env`, because we've desugared things like like
        "passAFile", "expandReferencesGraph", structured attrs, etc. */

src/nix/unix/daemon.cc

@@ -27,13 +27,15 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
+#include <fcntl.h>
 #include <sys/socket.h>
 #include <sys/un.h>
 #include <sys/select.h>
 #include <errno.h>
 #include <pwd.h>
 #include <grp.h>
 #include <fcntl.h>
+#include <sys/stat.h>
 
 #ifdef __linux__
 #  include "nix/util/cgroup.hh"
@@ -288,6 +290,10 @@ static std::pair<TrustedFlag, std::string> authPeer(const PeerInfo & peer)
     return {trusted, std::move(user)};
 }
 
+// Global FDs for jobserver (kept open for daemon lifetime)
+static AutoCloseFD jobserverReaderFd;
+static AutoCloseFD jobserverWriterFd;
+
 /**
  * Run a server. The loop opens a socket and accepts new connections from that
  * socket.
@@ -342,6 +348,72 @@ static void daemonLoop(std::optional<TrustedFlag> forceTrustClientOpt)
     }
 #endif
 
+    // Initialize GNU Make jobserver if enabled
+    if (settings.useJobserver) {
+        jobserverFifoPath = settings.nixStateDir + "/jobserver.fifo";
+
+        int totalTokens = settings.jobserverTokens;
+        if (totalTokens == 0) {
+            totalTokens = settings.buildCores ? settings.buildCores : settings.getDefaultCores();
+        }
+
+        /* GNU Make Protocol: Account for implicit slots
+         * Each concurrent build gets one free slot without touching the FIFO
+         * So if we want N total parallel jobs with M concurrent builds,
+         * we put N-M tokens in the FIFO */
+        int implicitSlots = settings.maxBuildJobs;
+        if (implicitSlots == 0)
+            implicitSlots = 1;
+
+        int fifoTokens = totalTokens - implicitSlots;
+        if (fifoTokens < 1) {
+            // Safety: ensure at least some tokens in FIFO
+            fifoTokens = totalTokens;
+            implicitSlots = 0;
+        }
+
+        // Remove old FIFO if it exists
+        unlink(jobserverFifoPath.c_str());
+        createDirs(dirOf(jobserverFifoPath));
+
+        // Create FIFO with permissions allowing build users to access
+        if (mkfifo(jobserverFifoPath.c_str(), 0660) == 0) {
+            // Open reader FD (non-blocking) - keeps FIFO alive, prevents EOF
+            jobserverReaderFd = open(jobserverFifoPath.c_str(), O_RDONLY | O_NONBLOCK);
+            if (jobserverReaderFd) {
+                // Open writer FD (non-blocking) - MUST stay open per protocol
+                jobserverWriterFd = open(jobserverFifoPath.c_str(), O_WRONLY | O_NONBLOCK);
+                if (jobserverWriterFd) {
+                    // Write tokens (each token is a single '+' character)
+                    std::string tokens(fifoTokens, '+');
+                    ssize_t written = write(jobserverWriterFd.get(), tokens.c_str(), fifoTokens);
+                    if (written == fifoTokens) {
+                        printInfo(
+                            "jobserver: initialized with %d tokens at %s (%d implicit slots reserved)",
+                            fifoTokens,
+                            jobserverFifoPath,
+                            implicitSlots);
+                    } else {
+                        warn("jobserver: only wrote %zd of %d tokens", written, fifoTokens);
+                    }
+                    // DO NOT CLOSE writer FD - must stay open for daemon lifetime
+                } else {
+                    warn("jobserver: failed to open writer: %s", strerror(errno));
+                    jobserverReaderFd.close();
+                    unlink(jobserverFifoPath.c_str());
+                    jobserverFifoPath.clear();
+                }
+            } else {
+                warn("jobserver: failed to open reader: %s", strerror(errno));
+                unlink(jobserverFifoPath.c_str());
+                jobserverFifoPath.clear();
+            }
+        } else {
+            warn("jobserver: failed to create FIFO: %s", strerror(errno));
+            jobserverFifoPath.clear();
+        }
+    }
+
     //  Loop accepting connections.
     while (1) {
 
@@ -410,6 +482,14 @@ static void daemonLoop(std::optional<TrustedFlag> forceTrustClientOpt)
                 options);
 
         } catch (Interrupted & e) {
+            // Clean up jobserver on interrupt (e.g., SIGINT)
+            if (jobserverWriterFd) {
+                jobserverWriterFd.close();
+                jobserverReaderFd.close();
+                if (!jobserverFifoPath.empty()) {
+                    unlink(jobserverFifoPath.c_str());
+                }
+            }
             return;
         } catch (Error & error) {
             auto ei = error.info();
@@ -418,6 +498,16 @@ static void daemonLoop(std::optional<TrustedFlag> forceTrustClientOpt)
             logError(ei);
         }
     }
+
+    // Clean up jobserver resources on daemon exit
+    if (jobserverWriterFd) {
+        printInfo("jobserver: cleaning up");
+        jobserverWriterFd.close();
+        jobserverReaderFd.close();
+        if (!jobserverFifoPath.empty()) {
+            unlink(jobserverFifoPath.c_str());
+        }
+    }
 }
 
 /**