Support enable_jobserver_pool = 1 in build.ninja.

Another way to enable jobserver pool mode is to set
`enable_jobserver_pool = 1` in the `build.ninja` file directly,
which can be determined by the generator directly.

This is equivalent to using `--jobserver-pool` on the command-line.
Note that:

- Any value other than 1 is ignored, intentionally (the size of
  the pool is determined when Ninja is invoked only).

- There is no way to disable the feature from the command-line
  when enabled in the build plan.

- Just like `--jobserver-pool`, this is ignored if a parent
  jobserver pool is detected when Ninja is invoked.

Author: digit-google

doc/manual.asciidoc

@@ -203,7 +203,8 @@ Ninja becomes a protocol client automatically if it detects the right
 values in the `MAKEFLAGS` environment variable (see exact conditions below).
 
 Since version 1.14, Ninja can also be a protocol server, if needed, using
-the `--jobserver-pool` command-line flag.
+the `--jobserver-pool` command-line flag, or if `enable_jobserver_pool = 1`
+is set in the Ninja build plan.
 
 In jobserver-enabled builds, there is one top-level "server" process which:
 
@@ -236,7 +237,8 @@ when a pipe-based jobserver is being used (i.e. when `MAKEFLAGS` contains
 otherwise ignore it.
 
 Since version 1.14, Ninja can also be used as a top-level protocol server
-by using the `--jobserver-pool` flag on the command-line. Note that this
+by using the `--jobserver-pool` flag on the command-line, or by setting
+`enable_jobserver_pool = 1` in the Ninja build file. Note that this
 feature is ignored if any of this is true:
 
 - An existing pool was detected, as this keeps all processes cooperating
@@ -984,7 +986,7 @@ previous one, it closes the previous scope.
 Top-level variables
 ~~~~~~~~~~~~~~~~~~~
 
-Two variables are significant when declared in the outermost file scope.
+Three variables are significant when declared in the outermost file scope.
 
 `builddir`:: a directory for some Ninja output files.  See <<ref_log,the
   discussion of the build log>>.  (You can also store other build output
@@ -993,6 +995,11 @@ Two variables are significant when declared in the outermost file scope.
 `ninja_required_version`:: the minimum version of Ninja required to process
   the build correctly.  See <<ref_versioning,the discussion of versioning>>.
 
+`enable_jobserver_pool`:: If set to `1` (any other value is ignored), enable
+  jobserver pool mode, as if `--jobserver-pool` was passed on the command
+  line. Note that `0` does not disable the feature, and that the size of
+  the pool is determined by the parallel job count that is either auto-detected
+  or controlled by the `-j<COUNT>` command-line option.
 
 [[ref_rule]]
 Rule variables

misc/jobserver_test.py

@@ -111,13 +111,13 @@ def span_output_file(span_n: int) -> str:
     return "out%02d" % span_n
 
 
-def generate_build_plan(command_count: int) -> str:
+def generate_build_plan(command_count: int, prefix: str = "") -> str:
     """Generate a Ninja build plan for |command_count| parallel tasks.
 
     Each task calls the test helper script which waits for 50ms
     then writes its own start and end time to its output file.
     """
-    result = f"""
+    result = prefix + f"""
 rule span
     command = {sys.executable} -S {_JOBSERVER_TEST_HELPER_SCRIPT} --duration-ms=50 $out
 
@@ -291,7 +291,7 @@ def run_ninja_with_jobserver_pipe(args):
             max_overlaps = compute_max_overlapped_spans(b.path, task_count)
             self.assertEqual(max_overlaps, 1)
 
-    def test_jobserver_pool_mode(self):
+    def test_jobserver_pool_mode_with_flag(self):
         task_count = 4
         build_plan = generate_build_plan(task_count)
         with BuildDir(build_plan) as b:
@@ -354,6 +354,57 @@ def test_jobserver_pool_mode_ignored_with_existing_pool(self):
             max_overlaps = compute_max_overlapped_spans(b.path, task_count)
             self.assertEqual(max_overlaps, 2)
 
+    def test_jobserver_pool_mode_with_variable(self):
+        task_count = 4
+        build_plan = generate_build_plan(task_count, prefix = "enable_jobserver_pool = 1\n")
+        with BuildDir(build_plan) as b:
+            # First, run the full tasks with with {task_count} tokens, this should allow all
+            # tasks to run in parallel.
+            ret = b.ninja_run(
+                ninja_args=["all"],
+            )
+            max_overlaps = compute_max_overlapped_spans(b.path, task_count)
+            self.assertEqual(max_overlaps, task_count)
+
+            # Second, use 2 tokens only, and verify that this was enforced by Ninja.
+            b.ninja_clean()
+            b.ninja_run(
+                ["-j2", "all"],
+            )
+            max_overlaps = compute_max_overlapped_spans(b.path, task_count)
+            self.assertEqual(max_overlaps, 2)
+
+            # Third, verify that --jobs=1 serializes all tasks.
+            b.ninja_clean()
+            b.ninja_run(
+                ["-j1", "all"],
+            )
+            max_overlaps = compute_max_overlapped_spans(b.path, task_count)
+            self.assertEqual(max_overlaps, 1)
+
+            # On Linux, use taskset to limit the number of available cores to 1
+            # and verify that the jobserver overrides the default Ninja parallelism
+            # and that {task_count} tasks are still spawned in parallel.
+            if platform.system() == "Linux":
+                # First, run without a jobserver, with a single CPU, Ninja will
+                # use a parallelism of 2 in this case (GuessParallelism() in ninja.cc)
+                b.ninja_clean()
+                b.ninja_run(
+                    ["all"],
+                    prefix_args=["taskset", "-c", "0"],
+                )
+                max_overlaps = compute_max_overlapped_spans(b.path, task_count)
+                self.assertEqual(max_overlaps, 2)
+
+                # Now with a jobserver with {task_count} tasks.
+                b.ninja_clean()
+                b.ninja_run(
+                    [f"-j{task_count}", "all"],
+                    prefix_args=["taskset", "-c", "0"],
+                )
+                max_overlaps = compute_max_overlapped_spans(b.path, task_count)
+                self.assertEqual(max_overlaps, task_count)
+
     def _test_MAKEFLAGS_value(
         self, ninja_args: T.List[str] = [], prefix_args: T.List[str] = []
     ):

src/manifest_parser.cc

@@ -35,6 +35,10 @@ ManifestParser::ManifestParser(State* state, FileReader* file_reader,
   env_ = &state->bindings_;
 }
 
+std::string ManifestParser::LookupVariable(const std::string& varname) {
+  return env_->LookupVariable(varname);
+}
+
 bool ManifestParser::Parse(const string& filename, const string& input,
                            string* err) {
   lexer_.Start(filename, input);

src/manifest_parser.h

@@ -48,7 +48,12 @@ struct ManifestParser : public Parser {
     return Parse("input", input, err);
   }
 
-private:
+  /// Retrieve the expanded value of a top-level variable from the
+  /// manifest. Returns an empty string if the variable is not defined.
+  /// Must be called only after a successful Load() call.
+  std::string LookupVariable(const std::string& varname);
+
+ private:
   /// Parse a file, given its contents as a string.
   bool Parse(const std::string& filename, const std::string& input,
              std::string* err);

src/manifest_parser_test.cc

@@ -1136,3 +1136,28 @@ TEST_F(ParserTest, DyndepRuleInput) {
   EXPECT_TRUE(edge->dyndep_->dyndep_pending());
   EXPECT_EQ(edge->dyndep_->path(), "in");
 }
+
+struct ManifestParserTest : public testing::Test {
+  ManifestParserTest() : parser(&state, &fs_) {}
+
+  void AssertParse(const char* input) {
+    string err;
+    EXPECT_TRUE(parser.ParseTest(input, &err));
+    ASSERT_EQ("", err);
+    VerifyGraph(state);
+  }
+
+  State state;
+  VirtualFileSystem fs_;
+  ManifestParser parser;
+};
+
+TEST_F(ManifestParserTest, LookupVariable) {
+  ASSERT_NO_FATAL_FAILURE(
+      AssertParse("foo = World\n"
+                  "bar = Hello $foo\n"));
+
+  ASSERT_EQ(parser.LookupVariable("foo"), "World");
+  ASSERT_EQ(parser.LookupVariable("bar"), "Hello World");
+  ASSERT_EQ(parser.LookupVariable("zoo"), "");
+}

src/ninja.cc

@@ -1965,6 +1965,16 @@ NORETURN void real_main(int argc, char** argv) {
       exit(1);
     }
 
+    // If enable_jobserver_pool is set to 1, enable jobserver pool mode as
+    // if --jobserver-pool had been passed on the command line. Note that
+    // any other value is ignored (thus 0 does not disable the flag if it is
+    // used).
+    std::string enable_jobserver_pool =
+        parser.LookupVariable("enable_jobserver_pool");
+    if (enable_jobserver_pool == "1") {
+      const_cast<BuildConfig&>(ninja.config_).jobserver_pool = true;
+    }
+
     ninja.ParsePreviousElapsedTimes();
 
     ExitStatus result = ninja.RunBuild(argc, argv, status);