basic incremental worker

Author: rickeylev

sphinxdocs/private/sphinx.bzl

@@ -272,19 +272,22 @@ def _run_sphinx(ctx, format, source_path, inputs, output_prefix, use_persistent_
 
     args.add("--show-traceback")  # Full tracebacks on error
     run_args.append("--show-traceback")
-    args.add("--builder", format)
-    run_args.extend(("--builder", format))
+    args.add(format, format = "--builder=%s")
+    run_args.append("--builder={}".format(format))
 
-    if ctx.attr._quiet_flag[BuildSettingInfo].value:
-        # Not added to run_args because run_args is for debugging
-        args.add("--quiet")  # Suppress stdout informational text
+    ##if ctx.attr._quiet_flag[BuildSettingInfo].value:
+    ##    # Not added to run_args because run_args is for debugging
+    ##    args.add("--quiet")  # Suppress stdout informational text
 
     # Build in parallel, if possible
     # Don't add to run_args: parallel building breaks interactive debugging
-    args.add("--jobs", "auto")
+    args.add("--jobs=auto")
 
     if use_persistent_workers:
-        args.add("--doctree-dir", paths.join(output_dir.path, ".doctrees"))
+        # Sphinx normally uses `.doctrees`, but we use underscore so it isn't
+        # hidden by default
+        args.add(paths.join(output_dir.path + "_doctrees"), format = "--doctree-dir=%s")
+
     else:
         args.add("--fresh-env")  # Don't try to use cache files. Bazel can't make use of them.
         run_args.append("--fresh-env")
@@ -312,6 +315,9 @@ def _run_sphinx(ctx, format, source_path, inputs, output_prefix, use_persistent_
 
     execution_requirements = {}
     if use_persistent_workers:
+        args.add("-v")
+        args.add("-v")
+        args.add("-v")
         args.use_param_file("@%s", use_always = True)
         args.set_param_file_format("multiline")
         execution_requirements["supports-workers"] = "1"

sphinxdocs/private/sphinx_build.py

@@ -23,17 +23,27 @@
 # parser.add_argument('srcdir')
 # parser.add_argument('outdir')
 parser.add_argument("--persistent_worker", action="store_true")
-parser.add_argument("--doctree-dir")
+##parser.add_argument("--doctree-dir")
 
+logger = logging.getLogger('sphinxdocs-build')
 
 class Worker:
 
     def __init__(self, instream: "typing.TextIO", outstream: "typing.TextIO"):
         self._instream = instream
         self._outstream = outstream
-        self._logger = logging.getLogger("worker")
-        logging.basicConfig(filename='echo.log', encoding='utf-8', level=logging.DEBUG)
-        self._logger.info("starting worker")
+        # Annoying. Sphinx resets its loging config as part of main()
+        # and the Sphinx() app setup/invocation. So any logging we try
+        # to setup here to get info out of sphinx is meaningless.
+        # -v -v -v will output more logging, but to stderr/stdout, and thus
+        # bazel's worker log file, due to sphinx's logging re-configuration.
+        # one-liner to get most recent worker log:
+        # find $workerLogDir -type f -printf '%T@ %p\n' | sort -n | tail -1 | awk '{print $2}'
+        logging.basicConfig(
+            ##filename='/tmp/sphinx-builder.log', encoding='utf-8',
+            level=logging.DEBUG
+        )
+        logger.info("starting worker")
         self._current = {}
         self._previous = {}
         self._cache = {}
@@ -45,13 +55,14 @@ def run(self) -> None:
                 try:
                     request = self._get_next_request()
                     if request is None:
-                        self._logger.info("Empty request: exiting")
+                        logger.info("Empty request: exiting")
                         break
                     response = self._process_request(request)
+                    logger.info("response:%s", response)
                     if response:
                         self._send_response(response)
                 except Exception:
-                    self._logger.exception("Unhandled error: request=%s", request)
+                    logger.exception("Unhandled error: request=%s", request)
                     output = (
                         f"Unhandled error:\nRequest: {request}\n"
                         + traceback.format_exc()
@@ -65,7 +76,7 @@ def run(self) -> None:
                         }
                     )
         finally:
-            self._logger.info("Worker shutting down")
+            logger.info("Worker shutting down")
 
     def _get_next_request(self) -> "object | None":
         line = self._instream.readline()
@@ -81,13 +92,14 @@ def inputs(self):
 
     def _update_digest(self, request):
         args, unknown = parser.parse_known_args(request["arguments"])
-        # Make room for the new build's data. 
+        # Make room for the new build's data.
         self._previous = self._current
         # Rearrange the new data into a dict to make comparisons easier.
         self._current = {}
         for page in request["inputs"]:
             path = page["path"]
             self._current[path] = page["digest"]
+            logger.info("path mtime: %s", pathlib.Path(path).stat().st_mtime)
         # Compare the content hashes to determine what pages have changed.
         changed_paths = []
         for path in self._current:
@@ -104,20 +116,21 @@ def _update_digest(self, request):
         # Normalize the paths into docnames
         digest = []
         for path in changed_paths:
+            logger.info("Changed: %s", path)
             if not path.endswith(".rst"):
                 continue
             srcdir = self.args[0]
             docname = path.replace(srcdir + "/", "")
             docname = docname.replace(".rst", "")
             digest.append(docname)
         args, unknown = parser.parse_known_args(self.args)
-        # Save the digest.
-        doctree_dir = Path(args.doctree_dir)
-        # On a fresh build, _restore_cache() does nothing, so this dir won't exist yet.
-        if not doctree_dir.is_dir():
-            doctree_dir.mkdir(parents=True)
-        with open(doctree_dir / Path("digest.json"), "w") as f:
-            json.dump(digest, f, indent=2)
+        ### Save the digest.
+        ##doctree_dir = Path(args.doctree_dir)
+        ### On a fresh build, _restore_cache() does nothing, so this dir won't exist yet.
+        ##if not doctree_dir.is_dir():
+        ##    doctree_dir.mkdir(parents=True)
+        ##with open(doctree_dir / Path("digest.json"), "w") as f:
+        ##    json.dump(digest, f, indent=2)
 
     def _restore_cache(self):
         for filepath in self._cache:
@@ -138,13 +151,20 @@ def _update_cache(self):
                     self._cache[str(filepath)] = f.read()
 
     def _process_request(self, request: "WorkRequest") -> "WorkResponse | None":
+        logger.info("request:%s", json.dumps(request, sort_keys=True, indent=2))
         if request.get("cancel"):
             return None
         self.args = request["arguments"]
-        self._restore_cache()
-        self._update_digest(request)
-        main(self.args)
-        self._update_cache()
+        ##self._restore_cache()
+        ##self._update_digest(request)
+        logger.info("main: %s", self.args)
+        orig_stdout = sys.stdout
+        sys.stdout = sys.stderr
+        try:
+            main(self.args)
+        finally:
+            sys.stdout = orig_stdout
+        ##self._update_cache()
         response = {
             "requestId": request.get("requestId", 0),
             "exitCode": 0,

sphinxdocs/tests/sphinx_docs/conf.py

@@ -5,11 +5,24 @@
 
 # -- Project info
 
-project = "Sphinx Docs Test"
+project = "Sphinx Docs Test xx"
 
 extensions = [
     "myst_parser",
 ]
 myst_enable_extensions = [
     "colon_fence",
 ]
+
+import logging
+logger = logging.getLogger('conf')
+
+def on_env_get_outdated(*args, **kwargs):
+    logger.info("env-get-outdated args: %s", args)
+    logger.info("env-get-outdated kwargs: %s", kwargs)
+    return []
+
+
+def setup(app):
+
+    app.connect('env-get-outdated', on_env_get_outdated)

sphinxdocs/tests/sphinx_docs/doc1.md

@@ -0,0 +1,4 @@
+# doc1
+
+hello doc 1
+x

sphinxdocs/tests/sphinx_docs/doc2.md

@@ -0,0 +1,5 @@
+# doc 2
+
+
+hello doc 3
+x