From 69a36a79dfd65fdd267ad1136bf4e7d546a7d242 Mon Sep 17 00:00:00 2001 From: John Lancaster <32917998+jsl12@users.noreply.github.com> Date: Mon, 9 Jun 2025 20:06:10 -0500 Subject: [PATCH] reworked file iteration and discovery --- appdaemon/app_management.py | 71 +++++++++++++++++++++++++------------ appdaemon/exceptions.py | 8 +++++ appdaemon/threads.py | 2 +- appdaemon/utils.py | 22 +++++++++++- 4 files changed, 78 insertions(+), 25 deletions(-) diff --git a/appdaemon/app_management.py b/appdaemon/app_management.py index ffc7b23fb..fbdbf3b2d 100644 --- a/appdaemon/app_management.py +++ b/appdaemon/app_management.py @@ -8,6 +8,7 @@ import pstats import subprocess import sys +import threading import traceback from collections import OrderedDict from collections.abc import AsyncGenerator, Iterable @@ -558,10 +559,10 @@ async def terminate_sequence(self, name: str) -> bool: return True - async def read_all(self, config_files: Iterable[Path] = None) -> AllAppConfig: + async def read_all(self, config_files: Iterable[Path]) -> AllAppConfig: config_files = config_files or self.dependency_manager.config_files - async def config_model_factory() -> AsyncGenerator[AllAppConfig, None, None]: + async def config_model_factory() -> AsyncGenerator[AllAppConfig, None]: """Creates a generator that sets the config_path of app configs""" for path in config_files: @ade.wrap_async(self.error, self.AD.app_dir, "Reading user apps") @@ -608,7 +609,8 @@ def update(d1: dict, d2: dict) -> dict: async def check_app_config_files(self, update_actions: UpdateActions): """Updates self.mtimes_config and self.app_config""" - files = await self.get_app_config_files() + # get_files_in_other_thread = utils.executor_decorator(self.get_app_config_files) + files = await self.get_app_config_files_async() self.dependency_manager.app_deps.update(files) # If there were config file changes @@ -686,6 +688,7 @@ def read_config_file(self, file: Path) -> AllAppConfig: This function is primarily used by the create/edit/remove app methods that write yaml files. """ + assert threading.current_thread().name.startswith("ThreadPool") raw_cfg = utils.read_config_file(file, app_config=True) if not bool(raw_cfg): self.logger.warning( @@ -849,7 +852,7 @@ def _process_import_paths(self): case 'default' | 'expert' | None: # Get unique set of the absolute paths of all the subdirectories containing python files python_file_parents = set( - f.parent.resolve() for f in Path(self.AD.app_dir).rglob("*.py") + f.parent.resolve() for f in self.get_python_files() ) # Filter out any that have __init__.py files in them @@ -907,43 +910,65 @@ async def _init_dep_manager(self): async def safe_dep_create(self: "AppManagement"): try: self.dependency_manager = DependencyManager( - python_files=await self.get_python_files(), - config_files=await self.get_app_config_files() + python_files=await self.get_python_files_async(), + config_files=await self.get_app_config_files_async() ) self.config_filecheck.mtimes = {} self.python_filecheck.mtimes = {} except ValidationError as e: - raise ade.BadAppConfigFile("Error creating dependency manager") from e + raise ade.DependencyManagerError("Failed to create dependency manager") from e except ade.AppDaemonException as e: raise e await safe_dep_create(self) - @utils.executor_decorator - def get_python_files(self) -> Iterable[Path]: - """Iterates through ``*.py`` in the app directory. Excludes directory names defined in exclude_dirs and with a "." character. Also excludes files that aren't readable.""" + def get_python_files(self) -> set[Path]: + """Get a set of valid Python files in the app directory. + + Valid files are ones that are readable, not inside an excluded directory, and not starting with a "." character. + """ + assert threading.current_thread().name.startswith("ThreadPool") return set( - f - for f in self.AD.app_dir.resolve().rglob("*.py") - if f.parent.name not in self.AD.exclude_dirs # apply exclude_dirs - and "." not in f.parent.name # also excludes *.egg-info folders - and os.access(f, os.R_OK) # skip unreadable files + utils.recursive_get_files( + base=self.AD.app_dir.resolve(), + suffix=".py", + exclude=set(self.AD.exclude_dirs), + ) ) @utils.executor_decorator - def get_app_config_files(self) -> Iterable[Path]: - """Iterates through config files in the config directory. Excludes directory names defined in exclude_dirs and files with a "." character. Also excludes files that aren't readable.""" + def get_python_files_async(self) -> set[Path]: + """Get a set of valid app config files in the app directory. + + Valid files are ones that are readable, not inside an excluded directory, and not starting with a "." character. + """ + return self.get_python_files() + + def get_app_config_files(self) -> set[Path]: + """Get a set of valid app fonfig files in the app directory. + + Valid files are ones that are readable, not inside an excluded directory, and not starting with a "." character. + """ + assert threading.current_thread().name.startswith("ThreadPool") return set( - f - for f in self.AD.app_dir.resolve().rglob(f"*{self.ext}") - if f.parent.name not in self.AD.exclude_dirs # apply exclude_dirs - and "." not in f.stem - and os.access(f, os.R_OK) # skip unreadable files + utils.recursive_get_files( + base=self.AD.app_dir.resolve(), + suffix=self.ext, + exclude=set(self.AD.exclude_dirs), + ) ) + @utils.executor_decorator + def get_app_config_files_async(self) -> set[Path]: + """Get a set of valid app config files in the app directory. + + Valid files are ones that are readable, not inside an excluded directory, and not starting with a "." character. + """ + return self.get_app_config_files() + async def check_app_python_files(self, update_actions: UpdateActions): """Checks the python files in the app directory. Part of self.check_app_updates sequence""" - files = await self.get_python_files() + files = await self.get_python_files_async() self.dependency_manager.update_python_files(files) # We only need to init the modules necessary for the new apps, not reloaded ones diff --git a/appdaemon/exceptions.py b/appdaemon/exceptions.py index ca4dbb812..6a243610d 100644 --- a/appdaemon/exceptions.py +++ b/appdaemon/exceptions.py @@ -385,6 +385,14 @@ def __str__(self): return f"Class '{self.class_name}' takes the wrong number of arguments. Check the inheritance" +@dataclass +class DependencyManagerError(AppDaemonException): + msg: str + + def __str__(self) -> str: + return self.msg + + @dataclass class AppDependencyError(AppDaemonException): app_name: str diff --git a/appdaemon/threads.py b/appdaemon/threads.py index 8f6b6fd7f..cc40070f3 100644 --- a/appdaemon/threads.py +++ b/appdaemon/threads.py @@ -171,7 +171,7 @@ async def create_initial_threads(self): else: # Force a config check here so we have an accurate activate app count self.AD.app_management.logger.debug("Reading app config files to determine how many threads to make") - cfg_paths = await self.AD.app_management.get_app_config_files() + cfg_paths = await self.AD.app_management.get_app_config_files_async() if not cfg_paths: self.logger.warning(f"No apps found in {self.AD.app_dir}. This is probably a mistake") self.total_threads = 10 diff --git a/appdaemon/utils.py b/appdaemon/utils.py index bcf7cb32c..ab414242a 100644 --- a/appdaemon/utils.py +++ b/appdaemon/utils.py @@ -16,7 +16,7 @@ import threading import time import traceback -from collections.abc import Awaitable, Iterable +from collections.abc import Awaitable, Generator, Iterable from datetime import timedelta, tzinfo from functools import wraps from logging import Logger @@ -1120,3 +1120,23 @@ def deprecation_warnings(model: BaseModel, logger: Logger): deprecation_warnings(val, logger) case BaseModel(): deprecation_warnings(attr, logger) + + +def recursive_get_files(base: Path, suffix: str, exclude: set[str] | None = None) -> Generator[Path, None, None]: + """Recursively generate file paths. + + Args: + base (Path): The base directory to start searching from. + suffix (str): The file extension to filter by. + exclude (set[str]): A set of directory names to exclude from the search. + + Yields: + Path objects to files that have the matching extension and are readable. + """ + for item in base.iterdir(): + if item.name.startswith(".") or (exclude is None or item.name in exclude): + continue + elif item.is_file() and item.suffix == suffix and os.access(item, os.R_OK): + yield item + elif item.is_dir() and os.access(item, os.R_OK): + yield from recursive_get_files(item, suffix, exclude)