reworked file iteration and discovery

Author: jsl12

appdaemon/app_management.py

@@ -8,6 +8,7 @@
 import pstats
 import subprocess
 import sys
+import threading
 import traceback
 from collections import OrderedDict
 from collections.abc import AsyncGenerator, Iterable
@@ -558,10 +559,10 @@ async def terminate_sequence(self, name: str) -> bool:
 
         return True
 
-    async def read_all(self, config_files: Iterable[Path] = None) -> AllAppConfig:
+    async def read_all(self, config_files: Iterable[Path]) -> AllAppConfig:
         config_files = config_files or self.dependency_manager.config_files
 
-        async def config_model_factory() -> AsyncGenerator[AllAppConfig, None, None]:
+        async def config_model_factory() -> AsyncGenerator[AllAppConfig, None]:
             """Creates a generator that sets the config_path of app configs"""
             for path in config_files:
                 @ade.wrap_async(self.error, self.AD.app_dir, "Reading user apps")
@@ -608,7 +609,8 @@ def update(d1: dict, d2: dict) -> dict:
 
     async def check_app_config_files(self, update_actions: UpdateActions):
         """Updates self.mtimes_config and self.app_config"""
-        files = await self.get_app_config_files()
+        # get_files_in_other_thread = utils.executor_decorator(self.get_app_config_files)
+        files = await self.get_app_config_files_async()
         self.dependency_manager.app_deps.update(files)
 
         # If there were config file changes
@@ -686,6 +688,7 @@ def read_config_file(self, file: Path) -> AllAppConfig:
 
         This function is primarily used by the create/edit/remove app methods that write yaml files.
         """
+        assert threading.current_thread().name.startswith("ThreadPool")
         raw_cfg = utils.read_config_file(file, app_config=True)
         if not bool(raw_cfg):
             self.logger.warning(
@@ -849,7 +852,7 @@ def _process_import_paths(self):
             case 'default' | 'expert' | None:
                 # Get unique set of the absolute paths of all the subdirectories containing python files
                 python_file_parents = set(
-                    f.parent.resolve() for f in Path(self.AD.app_dir).rglob("*.py")
+                    f.parent.resolve() for f in self.get_python_files()
                 )
 
                 # Filter out any that have __init__.py files in them
@@ -907,43 +910,65 @@ async def _init_dep_manager(self):
         async def safe_dep_create(self: "AppManagement"):
             try:
                 self.dependency_manager = DependencyManager(
-                    python_files=await self.get_python_files(),
-                    config_files=await self.get_app_config_files()
+                    python_files=await self.get_python_files_async(),
+                    config_files=await self.get_app_config_files_async()
                 )
                 self.config_filecheck.mtimes = {}
                 self.python_filecheck.mtimes = {}
             except ValidationError as e:
-                raise ade.BadAppConfigFile("Error creating dependency manager") from e
+                raise ade.DependencyManagerError("Failed to create dependency manager") from e
             except ade.AppDaemonException as e:
                 raise e
 
         await safe_dep_create(self)
 
-    @utils.executor_decorator
-    def get_python_files(self) -> Iterable[Path]:
-        """Iterates through ``*.py`` in the app directory. Excludes directory names defined in exclude_dirs and with a "." character. Also excludes files that aren't readable."""
+    def get_python_files(self) -> set[Path]:
+        """Get a set of valid Python files in the app directory.
+
+        Valid files are ones that are readable, not inside an excluded directory, and not starting with a "." character.
+        """
+        assert threading.current_thread().name.startswith("ThreadPool")
         return set(
-            f
-            for f in self.AD.app_dir.resolve().rglob("*.py")
-            if f.parent.name not in self.AD.exclude_dirs  # apply exclude_dirs
-            and "." not in f.parent.name  # also excludes *.egg-info folders
-            and os.access(f, os.R_OK)  # skip unreadable files
+            utils.recursive_get_files(
+                base=self.AD.app_dir.resolve(),
+                suffix=".py",
+                exclude=set(self.AD.exclude_dirs),
+            )
         )
 
     @utils.executor_decorator
-    def get_app_config_files(self) -> Iterable[Path]:
-        """Iterates through config files in the config directory. Excludes directory names defined in exclude_dirs and files with a "." character. Also excludes files that aren't readable."""
+    def get_python_files_async(self) -> set[Path]:
+        """Get a set of valid app config files in the app directory.
+
+        Valid files are ones that are readable, not inside an excluded directory, and not starting with a "." character.
+        """
+        return self.get_python_files()
+
+    def get_app_config_files(self) -> set[Path]:
+        """Get a set of valid app fonfig files in the app directory.
+
+        Valid files are ones that are readable, not inside an excluded directory, and not starting with a "." character.
+        """
+        assert threading.current_thread().name.startswith("ThreadPool")
         return set(
-            f
-            for f in self.AD.app_dir.resolve().rglob(f"*{self.ext}")
-            if f.parent.name not in self.AD.exclude_dirs  # apply exclude_dirs
-            and "." not in f.stem
-            and os.access(f, os.R_OK)  # skip unreadable files
+            utils.recursive_get_files(
+                base=self.AD.app_dir.resolve(),
+                suffix=self.ext,
+                exclude=set(self.AD.exclude_dirs),
+            )
         )
 
+    @utils.executor_decorator
+    def get_app_config_files_async(self) -> set[Path]:
+        """Get a set of valid app config files in the app directory.
+
+        Valid files are ones that are readable, not inside an excluded directory, and not starting with a "." character.
+        """
+        return self.get_app_config_files()
+
     async def check_app_python_files(self, update_actions: UpdateActions):
         """Checks the python files in the app directory. Part of self.check_app_updates sequence"""
-        files = await self.get_python_files()
+        files = await self.get_python_files_async()
         self.dependency_manager.update_python_files(files)
 
         # We only need to init the modules necessary for the new apps, not reloaded ones

appdaemon/exceptions.py

@@ -385,6 +385,14 @@ def __str__(self):
         return f"Class '{self.class_name}' takes the wrong number of arguments. Check the inheritance"
 
 
+@dataclass
+class DependencyManagerError(AppDaemonException):
+    msg: str
+
+    def __str__(self) -> str:
+        return self.msg
+
+
 @dataclass
 class AppDependencyError(AppDaemonException):
     app_name: str

appdaemon/threads.py

@@ -171,7 +171,7 @@ async def create_initial_threads(self):
         else:
             # Force a config check here so we have an accurate activate app count
             self.AD.app_management.logger.debug("Reading app config files to determine how many threads to make")
-            cfg_paths = await self.AD.app_management.get_app_config_files()
+            cfg_paths = await self.AD.app_management.get_app_config_files_async()
             if not cfg_paths:
                 self.logger.warning(f"No apps found in {self.AD.app_dir}. This is probably a mistake")
                 self.total_threads = 10

appdaemon/utils.py

@@ -16,7 +16,7 @@
 import threading
 import time
 import traceback
-from collections.abc import Awaitable, Iterable
+from collections.abc import Awaitable, Generator, Iterable
 from datetime import timedelta, tzinfo
 from functools import wraps
 from logging import Logger
@@ -1120,3 +1120,23 @@ def deprecation_warnings(model: BaseModel, logger: Logger):
                         deprecation_warnings(val, logger)
             case BaseModel():
                 deprecation_warnings(attr, logger)
+
+
+def recursive_get_files(base: Path, suffix: str, exclude: set[str] | None = None) -> Generator[Path, None, None]:
+    """Recursively generate file paths.
+
+    Args:
+        base (Path): The base directory to start searching from.
+        suffix (str): The file extension to filter by.
+        exclude (set[str]): A set of directory names to exclude from the search.
+
+    Yields:
+        Path objects to files that have the matching extension and are readable.
+    """
+    for item in base.iterdir():
+        if item.name.startswith(".") or (exclude is None or item.name in exclude):
+            continue
+        elif item.is_file() and item.suffix == suffix and os.access(item, os.R_OK):
+            yield item
+        elif item.is_dir() and os.access(item, os.R_OK):
+            yield from recursive_get_files(item, suffix, exclude)