Add throttling to file system polling and reduce CPU usage on default setting, remove watchman references (#83)

* reduce poll cpu usage
* Add poll throttle
* Fix references to watchman
* Increment version

Author: JamesHutchison

.ptyme_track/JamesHutchison

@@ -39,7 +39,7 @@
             "program": "${workspaceFolder}/metatests/metatest_runner.py",
             "justMyCode": false,
             "args": [
-                "--use-watchman"
+                "--use-os-events"
             ]
             // "args": [
             //     "--do-not-reset-daemon"

.vscode/launch.json

@@ -104,11 +104,9 @@ This can easily be done in VS Code with the following launch profile:
         },
 ```
 
-The daemon can be configured to use either watchman or polling to detect file changes.
-In my experience, watchman, running on docker on your system, frequently stops detecting changes without warning.
-The default behavior is to use polling, which is slower and has higher CPU usage, but is more likely to work.
-You can improve the behavior by limiting what files are watched via the watch globs. When using watchman,
-a file is opened for every file watched, so you will exhaust your open file limit if you watch too many.
+The daemon can be configured to use either file system polling or OS-based polling.
+The polling behavior is used by default and has higher compatibility. For example, if you're using
+Docker for Windows with WSL2, you're going to have a bad time with inotify.
 
 If the daemon is already running and you run pytest with `--daemon`, then the old one will be stopped
 and a new one will be started. Note that `pytest --daemon` is NOT how you run tests. It is only used to start
@@ -134,10 +132,15 @@ Then enable automatically starting the daemon in your settings:
 ```
 
 ## Arguments and Env Variables
-- `PYTEST_DAEMON_USE_WATCHMAN`
-    - Use watchman to check for file changes (recommended if your system supports it)
+- `PYTEST_DAEMON_USE_OS_EVENTS`
+    - Instead of polling the file system, use OS events such as inotify to check for file changes (recommended if your system supports it)
     - Default: `False`
-    - Command line: `--daemon-use-watchman`
+    - Command line: `--daemon-use-os-events`
+- `PYTEST_DAEMON_POLL_THROTTLE`
+    - A multipler for how aggressive the daemon does file system polling. This is not used if OS events are used.
+    - 2.0 = twice as slow, less CPU usage
+    - Default: `1.0`
+    - Command line: `--daemon-poll-throttle`
 - `PYTEST_DAEMON_PORT`
     - The port the daemon listens on.
     - Default: `4852`.
@@ -205,7 +208,7 @@ the given module will not be executed.
         addopts = "-p pytest_asyncio.plugin -p megamock.plugins.pytest -p pytest_hot_reloading.plugin"
     ```
 - Run out of a Github Codespace or similar dedicated external environment
-- Prefer watchman, if your system works well with it. It uses less CPU and can pick up changes faster. Enable it with the environment variable `PYTEST_DAEMON_USE_WATCHMAN=1`. It is only disabled by default for maximum compatibility.
+- Prefer using OS events, if your system works well with it. It uses less CPU and can pick up changes faster. Enable it with the environment variable `PYTEST_DAEMON_USE_OS_EVENTS=1`. It is only disabled by default for maximum compatibility.
 
 ## Known Issues
 - This is alpha, although it's getting closer to where it can be called beta

README.md

@@ -14,13 +14,13 @@ class MetaTestRunner:
     def __init__(
         self,
         do_not_reset_daemon: bool,
-        use_watchman: bool,
+        use_os_events: bool,
         change_delay: float,
         retries: int,
         temp_dir: Path,
     ) -> None:
         self.do_not_reset_daemon = do_not_reset_daemon
-        self.use_watchman = use_watchman
+        self.use_os_events = use_os_events
         self.change_delay = change_delay
         self.retries = retries
         self.temp_dir = temp_dir
@@ -40,15 +40,15 @@ def run_test(
         test_name: str,
         *file_mod_funcs: Callable,
         expect_fail: bool = False,
-        use_watchman: bool = False,
+        use_os_events: bool = False,
         retries: int = 0,
         test_file: str = "test_fixture_changes.py",
     ):
         for retry_num in range(retries + 1):
             self.make_fresh_copy()
             os.chdir(self.temp_dir)
             if system(
-                f"pytest -p pytest_hot_reloading.plugin --daemon-start-if-needed {'--daemon-use-watchman' if use_watchman else ''} "
+                f"pytest -p pytest_hot_reloading.plugin --daemon-start-if-needed {'--daemon-use-os-events' if use_os_events else ''} "
                 f"--daemon-watch-globs '{self.temp_dir}/*.py' "
                 f"{self.temp_dir}/test_fixture_changes.py::test_always_ran"
             ):
@@ -266,8 +266,8 @@ def modify_staticmethod_return_value(self) -> None:
     def main(self) -> None:
         if not self.do_not_reset_daemon:
             system("pytest --stop-daemon")
-        if self.use_watchman:
-            self.run_test("test_always_ran", use_watchman=True)
+        if self.use_os_events:
+            self.run_test("test_always_ran", use_os_events=True)
         self.run_test(
             "test_adding_fixture",
             self.add_fixture,
@@ -354,7 +354,7 @@ def main(self) -> None:
 if __name__ == "__main__":
     argparser = argparse.ArgumentParser()
     argparser.add_argument("--do-not-reset-daemon", action="store_true")
-    argparser.add_argument("--use-watchman", action="store_true")
+    argparser.add_argument("--use-os-events", action="store_true")
     argparser.add_argument("--change-delay", default=0.01, type=float)
     argparser.add_argument("--retry", default=0, type=int)
     argparser.add_argument("--temp-dir", default="/tmp/_metatests")
@@ -365,7 +365,7 @@ def main(self) -> None:
         temp_dir.mkdir()
     runner = MetaTestRunner(
         args.do_not_reset_daemon,
-        args.use_watchman,
+        args.use_os_events,
         args.change_delay,
         args.retry,
         Path(temp_dir),

metatests/metatest_runner.py

@@ -9,7 +9,7 @@ line-length = 98
 
 [tool.poetry]
 name = "pytest-hot-reloading"
-version = "0.1.0-alpha.15"
+version = "0.1.0-alpha.16"
 description = ""
 authors = ["James Hutchison <jamesghutchison@proton.me>"]
 readme = "README.md"

pyproject.toml

@@ -15,7 +15,7 @@ class PytestClient:
     _pytest_name: str
     _will_start_daemon_if_needed: bool
     _do_not_autowatch_fixtures: bool
-    _use_watchman: bool
+    _use_os_events: bool
 
     def __init__(
         self,
@@ -24,7 +24,8 @@ def __init__(
         pytest_name: str = "pytest",
         start_daemon_if_needed: bool = False,
         do_not_autowatch_fixtures: bool = False,
-        use_watchman: bool = False,
+        use_os_events: bool = False,
+        poll_throttle: float = 1.0,
         additional_args: Sequence[str] = [],
     ) -> None:
         self._socket = None
@@ -33,8 +34,9 @@ def __init__(
         self._pytest_name = pytest_name
         self._will_start_daemon_if_needed = start_daemon_if_needed
         self._do_not_autowatch_fixtures = do_not_autowatch_fixtures
-        self._use_watchman = use_watchman
+        self._use_os_events = use_os_events
         self._additional_args = additional_args
+        self._poll_throttle = poll_throttle
 
     def _get_server(self) -> xmlrpc.client.ServerProxy:
         server_url = f"http://{self._daemon_host}:{self._daemon_port}"
@@ -113,6 +115,7 @@ def _start_daemon(self) -> None:
             port=self._daemon_port,
             pytest_name=self._pytest_name,
             do_not_autowatch_fixtures=self._do_not_autowatch_fixtures,
-            use_watchman=self._use_watchman,
+            use_os_events=self._use_os_events,
             additional_args=self._additional_args,
+            poll_throttle=self._poll_throttle,
         )

pytest_hot_reloading/client.py

@@ -47,7 +47,8 @@ def start(
         watch_globs: str | None = None,
         ignore_watch_globs: str | None = None,
         do_not_autowatch_fixtures: bool | None = None,
-        use_watchman: bool | None = None,
+        use_os_events: bool | None = None,
+        poll_throttle: float | None = None,
         additional_args: Sequence[str] | None = None,
     ) -> None:
         # start the daemon such that it will not close when the parent process closes
@@ -66,8 +67,10 @@ def start(
                 args += ["--daemon-ignore-watch-globs", ignore_watch_globs]
             if do_not_autowatch_fixtures:
                 args += ["--daemon-do-not-autowatch-fixtures"]
-            if use_watchman:
-                args += ["--daemon-use-watchman"]
+            if use_os_events:
+                args += ["--daemon-use-os-events"]
+            if poll_throttle:
+                args += ["--daemon-poll-throttle", str(poll_throttle)]
             subprocess.Popen(
                 args + list(additional_args or []),
                 env=os.environ,

pytest_hot_reloading/daemon.py

@@ -6,6 +6,7 @@
 import inspect
 import os
 import sys
+import time
 from enum import Enum
 from pathlib import Path
 from typing import TYPE_CHECKING, Callable, Optional
@@ -32,7 +33,8 @@ class EnvVariables(str, Enum):
     PYTEST_DAEMON_START_IF_NEEDED = "PYTEST_DAEMON_START_IF_NEEDED"
     PYTEST_DAEMON_DISABLE = "PYTEST_DAEMON_DISABLE"
     PYTEST_DAEMON_DO_NOT_AUTOWATCH_FIXTURES = "PYTEST_DAEMON_DO_NOT_AUTOWATCH_FIXTURES"
-    PYTEST_DAEMON_USE_WATCHMAN = "PYTEST_DAEMON_USE_WATCHMAN"
+    PYTEST_DAEMON_USE_OS_EVENTS = "PYTEST_DAEMON_USE_OS_EVENTS"
+    PYTEST_DAEMON_POLL_THROTTLE = "PYTEST_DAEMON_POLL_THROTTLE"
 
 
 def pytest_addoption(parser) -> None:
@@ -110,18 +112,26 @@ def pytest_addoption(parser) -> None:
         ),
     )
     group.addoption(
-        "--daemon-use-watchman",
+        "--daemon-use-os-events",
         action="store_true",
         default=(
-            os.getenv(EnvVariables.PYTEST_DAEMON_USE_WATCHMAN, "False").lower() in ("true", "1")
+            os.getenv(EnvVariables.PYTEST_DAEMON_USE_OS_EVENTS, "False").lower() in ("true", "1")
         ),
         help=(
-            "Use watchman instead of polling. "
+            "Use OS events such as inotify instead of polling. "
             "This reduces CPU usage, takes up open file handles, and improves responsiveness. "
             "Some systems cannot reliably use this."
         ),
     )
 
+    group.addoption(
+        "--daemon-poll-throttle",
+        default=(os.getenv(EnvVariables.PYTEST_DAEMON_POLL_THROTTLE, "1")),
+        help=(
+            "The throttle for polling, as a float multiplier. Higher numbers are slower but tax the CPU less."
+        ),
+    )
+
 
 # list of pytest hooks
 # https://docs.pytest.org/en/stable/reference.html#_pytest.hookspec.pytest_addhooks
@@ -301,15 +311,37 @@ def setup_jurigged(config: Config):
     monkey_patch_jurigged_function_definition()
     monkeypatch_group_definition()
     if not config.option.daemon_do_not_autowatch_fixtures:
-        monkeypatch_fixture_marker(config.option.daemon_use_watchman)
+        monkeypatch_fixture_marker(config.option.daemon_use_os_events)
     else:
         print("Not autowatching fixtures")
 
     pattern = _get_pattern_filters(config)
-    # TODO: intelligently use poll versus watchman (https://github.com/JamesHutchison/pytest-hot-reloading/issues/16)
+    # TODO: intelligently use poll (https://github.com/JamesHutchison/pytest-hot-reloading/issues/16)
+
+    poll_throttle = float(config.option.daemon_poll_throttle)
+
+    from watchdog.observers.polling import PollingObserverVFS
+
+    class NewPollingObserverVFS(PollingObserverVFS):
+        def __init__(self, stat, listdir, polling_interval=2) -> None:
+            def lagged_listdir(*args, **kwargs):
+                time.sleep(0.02 * poll_throttle)  # give CPU a break!
+                return listdir(*args, **kwargs)
+
+            super().__init__(stat, lagged_listdir, polling_interval * poll_throttle)
+
+    jurigged.live.PollingObserverVFS = NewPollingObserverVFS
+
+    poll: bool | float
+
+    if config.option.daemon_use_os_events:
+        poll = False
+    else:
+        poll = 2  # seconds
+
     jurigged.watch(
         pattern=pattern,
-        poll=(not config.option.daemon_use_watchman),
+        poll=poll,
     )
 
 
@@ -323,7 +355,7 @@ def watch_file(path: Path | str) -> None:
 seen_files: set[str] = set()
 
 
-def monkeypatch_fixture_marker(use_watchman: bool):
+def monkeypatch_fixture_marker(use_os_events: bool):
     import pytest
     from _pytest import fixtures
 
@@ -389,7 +421,8 @@ def _plugin_logic(config: Config) -> int:
             pytest_name=pytest_name,
             start_daemon_if_needed=config.option.daemon_start_if_needed,  # --daemon-start-if-needed
             do_not_autowatch_fixtures=config.option.daemon_do_not_autowatch_fixtures,  # --daemon-do-not-autowatch-fixtures
-            use_watchman=config.option.daemon_use_watchman,  # --daemon-use-watchman
+            use_os_events=config.option.daemon_use_os_events,  # --daemon-use-os-events
+            poll_throttle=config.option.daemon_poll_throttle,  # --daemon-poll-throttle
             additional_args=config.invocation_params.args,
         )