Restore logging colors, refactor metatests, fix error (#72)

Refactors and fixes for the metatests:
- Uses a consistent temporary directory, allowing for the daemon to detect changes
- VS Code config uses watchman
- Daemon VS Code config watches `/tmp/_metatests`
- Removed code that wrote a `.gitignore` file as it is obsolete
- Added `file_changes.py` which contains non-test changes
- Added another test file, `test_file_changes.py`
- Added additional test cases including one that brings out the bug that was fixed
- Fixed async tests being skipped. They for sure worked before but seemed to have broke when a temporary directory was used and pytest was no longer using the same config.

Refactors and fixes for the client / daemon:
- The default jurigged logging is used again. Before a placeholder was used to avoid duplicated output, but it no longer appears to be needed. The behavior seems correct without any changes.
- Reduced retries for waiting on the daemon to come up, from 1000 to 100. The 1000 value seemed excessive
- Pytest arguments are passed from the client to the server when the server is automatically started. Note that includes tests to run, but the nature of the daemon results in it having no effect. This was needed because the server wasn't using the plugin and possibly other plugins or relevant plugin arguments that were provided to the client.
- The server now returns an explicit stacktrace back to the client rather than using the built-in error handling of the XML RPC server. The XML RPC server was only giving the error and the error message, which can be insufficient.
- The server no longer restores the current working directory. In the case of using temporary directories, the cwd would get deleted resulting in a cryptic file not found error after the tests successfully ran. Storing and restoring the cwd before and after test runs didn't seem to provide value.
- Fixed bug where the hot reloading was attempting to access a global function that didn't exist, if the updated function was actually the method or static method of a class. This also resulted in a refactor of how function signatures are checked. It now pulls `self.node` rather than attempting to pull the old function from the global scope. The signatures are pulled from the AST, and then just the names of the arguments are copied for the purposes of detecting signature changes.
- Added comments so its easier to trace args to where they get plugged in

Notes:
- Resolves #71

Author: JamesHutchison

.devcontainer/postStartBackground.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
 poetry run pytest --daemon &
+poetry run dmypy start
 
 ptyme-track --standalone

.ptyme_track/JamesHutchison

@@ -13,7 +13,7 @@
             "args": [
                 "--daemon",
                 "--daemon-watch-globs",
-                "./pytest_hot_reloading/daemon*.py:./pytest_hot_reloading/plugin.py",
+                "./pytest_hot_reloading/daemon*.py:./pytest_hot_reloading/plugin.py:/tmp/_metatests/*.py",
                 "--daemon-ignore-watch-globs",
                 "./.venv/*" // this is the default value
             ]
@@ -38,6 +38,9 @@
             "request": "launch",
             "program": "${workspaceFolder}/metatests/metatest_runner.py",
             "justMyCode": false,
+            "args": [
+                "--use-watchman"
+            ]
             // "args": [
             //     "--do-not-reset-daemon"
             // ]

.vscode/launch.json

@@ -1,6 +1,6 @@
 import argparse
+import os
 import shutil
-import tempfile
 import time
 from os import system
 from pathlib import Path
@@ -27,36 +27,37 @@ def __init__(
         self.modified_conftest_file = self.temp_dir / "conftest.py"
         self.modified_test_file = self.temp_dir / "test_fixture_changes.py"
         self.modified_used_by_conftest_file = self.temp_dir / "used_by_conftest.py"
+        self.modified_code_file = self.temp_dir / "file_changes.py"
 
     def make_fresh_copy(self):
         # delete the directory contents if it is not empty
         if self.temp_dir.exists():
             shutil.rmtree(self.temp_dir)
         shutil.copytree(TEMPLATE_DIR, self.temp_dir)
-        with (self.temp_dir / ".gitignore").open("w") as gitignore:
-            gitignore.write("*")
 
     def run_test(
         self,
         test_name: str,
         *file_mod_funcs: Callable,
         expect_fail: bool = False,
         use_watchman: bool = False,
-        change_delay: float = 0.01,
         retries: int = 0,
+        test_file: str = "test_fixture_changes.py",
     ):
         for retry_num in range(retries + 1):
             self.make_fresh_copy()
+            os.chdir(self.temp_dir)
             if system(
                 f"pytest -p pytest_hot_reloading.plugin --daemon-start-if-needed {'--daemon-use-watchman' if use_watchman else ''} "
+                f"--daemon-watch-globs '{self.temp_dir}/*.py' "
                 f"{self.temp_dir}/test_fixture_changes.py::test_always_ran"
             ):
                 raise Exception("Failed to prep daemon")
             for func in file_mod_funcs:
                 func()
-            time.sleep(change_delay + retry_num * 0.25)
+            time.sleep(self.change_delay + retry_num * 0.25)
             try:
-                if system(f"pytest {self.temp_dir}/test_fixture_changes.py::{test_name}"):
+                if system(f"pytest {self.temp_dir}/{test_file}::{test_name}"):
                     if not expect_fail:
                         raise Exception(f"Failed to run test {test_name}")
                 elif expect_fail:
@@ -231,6 +232,37 @@ def remove_autouse_fixture_outside_of_conftest(self) -> None:
         with self.modified_used_by_conftest_file.open("w") as f:
             f.writelines(new_lines)
 
+    def modify_function_return_value(self) -> None:
+        print(self.modified_code_file)
+        # modify the function in file_changes.py
+        with self.modified_code_file.open() as f:
+            lines = f.readlines()
+
+        # write new version of file_changes.py
+        with self.modified_code_file.open("w") as f:
+            for line in lines:
+                f.write(line.replace('return "foo"', 'return "foo modified"'))
+
+    def modify_method_return_value(self) -> None:
+        # modify the method in file_changes.py
+        with self.modified_code_file.open() as f:
+            lines = f.readlines()
+
+        # write new version of file_changes.py
+        with self.modified_code_file.open("w") as f:
+            for line in lines:
+                f.write(line.replace('return "bar"', 'return "bar modified"'))
+
+    def modify_staticmethod_return_value(self) -> None:
+        # modify the method in file_changes.py
+        with self.modified_code_file.open() as f:
+            lines = f.readlines()
+
+        # write new version of file_changes.py
+        with self.modified_code_file.open("w") as f:
+            for line in lines:
+                f.write(line.replace('return "moo"', 'return "moo modified"'))
+
     def main(self) -> None:
         if not self.do_not_reset_daemon:
             system("pytest --stop-daemon")
@@ -265,6 +297,11 @@ def main(self) -> None:
             self.rename_fixture,
             self.rename_use_of_fixture,
         )
+        self.run_test(
+            "TestClass::test_method_fixture_change",
+            self.rename_fixture,
+            self.rename_use_of_fixture,
+        )
         self.run_test(
             "test_renaming_should_fail",
             self.rename_fixture,
@@ -297,6 +334,21 @@ def main(self) -> None:
             "test_autouse_fixture_outside_of_conftest_is_removed",
             self.remove_autouse_fixture_outside_of_conftest,
         )
+        self.run_test(
+            "test_file_function_change",
+            self.modify_function_return_value,
+            test_file="test_file_changes.py",
+        )
+        self.run_test(
+            "test_class_method_change",
+            self.modify_method_return_value,
+            test_file="test_file_changes.py",
+        )
+        self.run_test(
+            "test_staticmethod_change",
+            self.modify_staticmethod_return_value,
+            test_file="test_file_changes.py",
+        )
 
 
 if __name__ == "__main__":
@@ -305,14 +357,17 @@ def main(self) -> None:
     argparser.add_argument("--use-watchman", action="store_true")
     argparser.add_argument("--change-delay", default=0.01, type=float)
     argparser.add_argument("--retry", default=0, type=int)
+    argparser.add_argument("--temp-dir", default="/tmp/_metatests")
     args = argparser.parse_args()
 
-    with tempfile.TemporaryDirectory() as temp_dir:
-        runner = MetaTestRunner(
-            args.do_not_reset_daemon,
-            args.use_watchman,
-            args.change_delay,
-            args.retry,
-            Path(temp_dir),
-        )
-        runner.main()
+    temp_dir = Path(args.temp_dir)
+    if not temp_dir.exists():
+        temp_dir.mkdir()
+    runner = MetaTestRunner(
+        args.do_not_reset_daemon,
+        args.use_watchman,
+        args.change_delay,
+        args.retry,
+        Path(temp_dir),
+    )
+    runner.main()

metatests/metatest_runner.py

@@ -0,0 +1,11 @@
+def some_func():
+    return "foo"
+
+
+class SomeClass:
+    def some_method(self):
+        return "bar"
+
+    @staticmethod
+    def some_static_method():
+        return "moo"

metatests/template/file_changes.py

@@ -0,0 +1,2 @@
+[pytest]
+asyncio_mode: auto

metatests/template/pytest.ini

@@ -0,0 +1,28 @@
+from .file_changes import SomeClass, some_func
+
+
+def test_file_function_change():
+    """
+    This test uses a function that has the return value changed.
+
+    Should pass
+    """
+    assert some_func() == "foo modified"
+
+
+def test_class_method_change():
+    """
+    This test uses a method that has the return value changed.
+
+    Should pass
+    """
+    assert SomeClass().some_method() == "bar modified"
+
+
+def test_staticmethod_change():
+    """
+    This test uses a static method that has the return value changed.
+
+    Should pass
+    """
+    assert SomeClass.some_static_method() == "moo modified"

metatests/template/test_file_changes.py

@@ -108,3 +108,12 @@ def test_autouse_fixture_outside_of_conftest_is_removed(fixture_outside_of_conft
     Should pass
     """
     assert fixture_outside_of_conftest == "modified by autouse value"
+
+
+class TestClass:
+    def test_method_fixture_change(self, renamed_fixture):
+        """
+        This test uses a fixture that is renamed
+
+        Should pass
+        """

metatests/template/test_fixture_changes.py

@@ -5,7 +5,7 @@
 import time
 import xmlrpc.client
 from pathlib import Path
-from typing import cast
+from typing import Sequence, cast
 
 
 class PytestClient:
@@ -25,6 +25,7 @@ def __init__(
         start_daemon_if_needed: bool = False,
         do_not_autowatch_fixtures: bool = False,
         use_watchman: bool = False,
+        additional_args: Sequence[str] = [],
     ) -> None:
         self._socket = None
         self._daemon_host = daemon_host
@@ -33,6 +34,7 @@ def __init__(
         self._will_start_daemon_if_needed = start_daemon_if_needed
         self._do_not_autowatch_fixtures = do_not_autowatch_fixtures
         self._use_watchman = use_watchman
+        self._additional_args = additional_args
 
     def _get_server(self) -> xmlrpc.client.ServerProxy:
         server_url = f"http://{self._daemon_host}:{self._daemon_port}"
@@ -112,4 +114,5 @@ def _start_daemon(self) -> None:
             pytest_name=self._pytest_name,
             do_not_autowatch_fixtures=self._do_not_autowatch_fixtures,
             use_watchman=self._use_watchman,
+            additional_args=self._additional_args,
         )