Allow running builds sequentially (#14)

This adds a new config option `SEQUENTIAL` for building the revisions sequentially instead of calling the build steps for all revisions in parallel. This helps for repositories with a massive amount of revisions which would suffer memory overflows without this option.

* sphinx_polyversion/driver.py (srun): Implement the new build mode.
* sphinx_polyversion/driver.py (run): Add `sequential` parameter that controls whether `arun` or `srun` is called.

* sphinx_polyversion/main.py (get_parser): Add `--sequential` cmd option.
* sphinx_polyversion/api.py (apply_overrides): Implement new `--sequential` cmd option.
* tests/test_api.py (test_apply_overrides): Update test

* docs/poly.py : Add `SEQUENTIAL` option.

* docs/sphinx/guide/getting-started.rst : Configure new option and add missing link.

* README.md: Update example `poly.py` config file to include the new `SEQUENTIAL` option.

---------

Co-authored-by: real-yfprojects <[email protected]>

Author: vpratz

README.md

@@ -108,8 +108,12 @@ MOCK_DATA = {
     ],
     "current": GitRef("local", "", "", GitRefType.BRANCH, datetime.fromtimestamp(6)),
 }
+#: Whether to build using only local files and mock data
 MOCK = False
 
+#: Whether to run the builds in sequence or in parallel
+SEQUENTIAL = False
+
 # Load overrides read from commandline to global scope
 apply_overrides(globals())
 # Determine repository root directory
@@ -131,7 +135,7 @@ DefaultDriver(
     template_dir=root / src / "templates",
     static_dir=root / src / "static",
     mock=MOCK_DATA,
-).run(MOCK)
+).run(MOCK, SEQUENTIAL)
 ```
 
 Build your docs by running
@@ -143,7 +147,7 @@ $ sphinx-polyversion docs/poly.py
 ### Commandline Options
 
 ```
-usage: sphinx-polyversion [-h] [-o [OVERRIDE [OVERRIDE ...]]] [-v] [-l] conf [out]
+usage: sphinx-polyversion [-h] [-o [OVERRIDE [OVERRIDE...]]] [-v] [-l] [--sequential] conf [out]
 
 Build multiple versions of your sphinx docs and merge them into one site.
 
@@ -157,6 +161,7 @@ optional arguments:
                         Override config options. Pass them as `key=value` pairs.
   -v, --verbosity       Increase output verbosity (decreases minimum log level). The default log level is ERROR.
   -l, --local, --mock   Build the local version of your docs.
+  --sequential          Build the revisions sequentially.
 ```
 
 ### How To Build Versions Differently

docs/poly.py

@@ -37,8 +37,13 @@
     ],
     "current": GitRef("local", "", "", GitRefType.TAG, datetime.fromtimestamp(6)),
 }
+
+#: Whether to build using only local files and mock data
 MOCK = False
 
+#: Whether to run the builds in sequence or in parallel
+SEQUENTIAL = False
+
 
 #: Data passed to templates
 def data(driver, rev, env):
@@ -84,4 +89,4 @@ def root_data(driver):
     data_factory=data,
     root_data_factory=root_data,
     mock=MOCK_DATA,
-).run(MOCK)
+).run(MOCK, SEQUENTIAL)

docs/sphinx/guide/getting-started.rst

@@ -45,9 +45,13 @@ you define and how you name them.
 
 .. warning::
 
-    There are to naming specifications for config variables:
+    To be able to override configuration options using the `sphix-polyversion`
+    command, you have to use the following naming specifications for config
+    variables:
     The output directory must always be called :code:`OUTPUT_DIR`.
-    And you have to pass :code:`MOCK` to :code:`DefaultDriver.run`.
+    The flag to use the local version and mock data must be called :code:`MOCK`.
+    The flag to use sequential builds must be called :code:`SEQUENTIAL`.
+    Finally, you have to pass :code:`MOCK` and :code:`SEQUENTIAL` to :code:`DefaultDriver.run`.
 
 .. TODO link reference
 
@@ -68,10 +72,8 @@ you define and how you name them.
 Defining the options as variables at the beginning not only makes
 the configuration file easier to understand but also allows those variables to
 be overridden from the commandline before being used to build the documentation.
-This is a major feature of `sphinx-polyversion` which will be explained
-further down this guide.
-
-.. TODO: link overrides section
+This is a major feature of `sphinx-polyversion` which will be explained in this
+section and :ref:`further down this guide<Overriding config options>`.
 
 .. code-block:: py
     :caption: `docs/poly.py` - imports and config variables
@@ -116,6 +118,11 @@ further down this guide.
         "current": GitRef("local", "", "", GitRefType.BRANCH, datetime.fromtimestamp(6)),
     }
 
+    #: Whether to build using only local files and mock data
+    MOCK = False
+
+    #: Whether to run the builds in sequence or in parallel
+    SEQUENTIAL = False
 
 Next you add the code handling the overrides read from the commandline.
 This is straightforward since `sphinx-polyversion` provides the function :code:`apply_overrides` that
@@ -177,7 +184,7 @@ in its revision.
         template_dir=root / src / "templates",
         static_dir=root / src / "static",
         mock=MOCK_DATA,
-    ).run(MOCK)
+    ).run(MOCK, SEQUENTIAL)
 
 Using versioning data in :code:`conf.py`
 ----------------------------------------
@@ -314,13 +321,15 @@ this flag all other versions are not build.
     :prog: sphinx_polyversion
     :nodescription:
 
+.. _Overriding config options:
 
 Overriding config options
 -------------------------
 
 You can override the defaults set in `poly.py` by specifying values on the
 commandline. Specifying an output location will override :code:`OUTPUT_DIR` while
 specifying :code:`--local` will set :code:`MOCK` to :code:`True`.
+Specifying :code:`--sequential` will set :code:`SEQUENTIAL` to :code:`True`.
 All other variables can be overidden through the :code:`-o` flag. You can
 override the arguments passed to `sphinx-build` by entering the following:
 

sphinx_polyversion/api.py

@@ -82,6 +82,9 @@ def apply_overrides(namespace: dict[str, Any]) -> dict[str, Any]:
     overrides.setdefault("MOCK", False)
     if args.local:
         overrides["MOCK"] = True
+    overrides.setdefault("SEQUENTIAL", False)
+    if args.sequential:
+        overrides["SEQUENTIAL"] = True
     namespace.update(overrides)
     return overrides
 

sphinx_polyversion/driver.py

@@ -361,10 +361,29 @@ async def arun(self) -> None:
         await asyncio.gather(*(self.build_revision(rev) for rev in self.targets))
         await self.build_root()
 
-    def run(self, mock: bool = False) -> None:
-        """Build all revisions or build from local files."""
+    async def srun(self) -> None:
+        """Build all revisions (sequential)."""
+        await self.init()
+        for rev in self.targets:
+            await self.build_revision(rev)
+        await self.build_root()
+
+    def run(self, mock: bool = False, sequential: bool = False) -> None:
+        """
+        Build all revisions or build from local files.
+
+        Parameters
+        ----------
+        mock : bool, optional, default: False
+            Whether to build all revisions (False) or from local files (True)
+        sequential : bool, optional, default: False
+            Whether to build revisions in parallel (False) or sequential (True)
+
+        """
         if mock:
             asyncio.run(self.build_local())
+        elif sequential:
+            asyncio.run(self.srun())
         else:
             asyncio.run(self.arun())
 

sphinx_polyversion/main.py

@@ -80,6 +80,11 @@ def get_parser(expect_config: bool = True) -> argparse.ArgumentParser:
         action="store_true",
         help="Build the local version of your docs.",
     )
+    parser.add_argument(
+        "--sequential",
+        action="store_true",
+        help="Build the revisions sequentially.",
+    )
     return parser
 
 

tests/test_api.py

@@ -31,4 +31,4 @@ def test_apply_overrides():
     namespace = {}
     apply_overrides(namespace)
 
-    assert namespace == {"a": "1", "b": "2", "MOCK": False}
+    assert namespace == {"a": "1", "b": "2", "MOCK": False, "SEQUENTIAL": False}