diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index b5f17d314..ccb81729e 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -44,7 +44,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v5 diff --git a/.python-version b/.python-version new file mode 100644 index 000000000..24ee5b1be --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/appdaemon/adapi.py b/appdaemon/adapi.py index da4fcedf5..3bb0f26e1 100644 --- a/appdaemon/adapi.py +++ b/appdaemon/adapi.py @@ -587,99 +587,115 @@ async def get_pin_thread(self) -> int: # Namespace # - def set_namespace(self, namespace: str, writeback: str = "safe", persist: bool = True) -> None: - """Set the current namespace of the app + def set_namespace( + self, + namespace: str, + writeback: Literal["safe", "hybrid"] | utils.ADWritebackType = "safe", + persist: bool = True, + ) -> None: + """Set the current namespace of the app. + + This will create a new namespace if it doesn't already exist. By default, this will be a persistent namespace + with ``safe`` writeback, which means that all state changes will be stored to disk as they happen. - See the `namespace documentation `__ for more information. + See the :py:ref:`app_namespaces` for more information. Args: namespace (str): Name of the new namespace writeback (str, optional): The writeback to be used if a new namespace gets created. Will be ``safe`` by default. persist (bool, optional): Whether to make the namespace persistent if a new one is created. Defaults to - ``True``. + `True`. Returns: None. Examples: - >>> self.set_namespace("hass1") + Create a namespace that buffers state changes in memory and periodically writes them to disk. + >>> self.set_namespace("on_disk", writeback="hybrid", persist=True) + + Create an in-memory namespace that won't survive AppDaemon restarts. + + >>> self.set_namespace("in_memory", persist=False) """ - # Keeping namespace get/set functions for legacy compatibility if not self.namespace_exists(namespace): - self.add_namespace(namespace=namespace, writeback=writeback, persist=persist) + self.add_namespace( + namespace=namespace, + writeback=utils.ADWritebackType(writeback), + persist=persist + ) self.namespace = namespace def get_namespace(self) -> str: """Get the app's current namespace. - See the `namespace documentation `__ for more information. + See :py:ref:`app_namespaces` for more information. """ # Keeping namespace get/set functions for legacy compatibility return self.namespace @utils.sync_decorator async def namespace_exists(self, namespace: str) -> bool: - """Check the existence of a namespace in AppDaemon. + """Check for the existence of a namespace. - See the `namespace documentation `__ for more information. + See :py:ref:`app_namespaces` for more information. Args: - namespace (str): The namespace to be checked. + namespace (str): The namespace to check for. Returns: - bool: ``True`` if the namespace exists, otherwise ``False``. - - Examples: - Check if the namespace ``storage`` exists within AD - - >>> if self.namespace_exists("storage"): - >>> #do something like create it - + bool: `True` if the namespace exists, otherwise `False`. """ return self.AD.state.namespace_exists(namespace) @utils.sync_decorator - async def add_namespace(self, namespace: str, writeback: str = "safe", persist: bool = True) -> str | None: - """Add a user-defined namespace, which has a database file associated with it. - - When AppDaemon restarts these entities will be loaded into the namespace with all their previous states. This - can be used as a basic form of non-volatile storage of entity data. Depending on the configuration of the - namespace, this function can be setup to constantly be running automatically - or only when AD shutdown. + async def add_namespace( + self, + namespace: str, + writeback: Literal["safe", "hybrid"] | utils.ADWritebackType = "safe", + persist: bool = True, + ) -> str | None: + """Add a user-defined namespace. - See the `namespace documentation `__ for more information. + See the :py:ref:`app_namespaces` for more information. Args: namespace (str): The name of the new namespace to create - writeback (optional): The writeback to be used. Will be ``safe`` by default + writeback (optional): The writeback to be used. Defaults to ``safe``, which writes every state change to + disk. This can be problematic for namespaces that have a lot of state changes. `Safe` in this case + refers data loss, rather than performance. The other option is ``hybrid``, which buffers state changes. persist (bool, optional): Whether to make the namespace persistent. Persistent namespaces are stored in a - database file and are reloaded when AppDaemon restarts. Defaults to ``True`` + database file and are reloaded when AppDaemon restarts. Defaults to `True`. Returns: - The file path to the newly created namespace. Will be ``None`` if not persistent + The file path to the newly created namespace. Will be ``None`` if not persistent. Examples: - Add a new namespace called `storage`. + Create a namespace that buffers state changes in memory and periodically writes them to disk. + + >>> self.add_namespace("on_disk", writeback="hybrid", persist=True) - >>> self.add_namespace("storage") + Create an in-memory namespace that won't survive AppDaemon restarts. + >>> self.add_namespace("in_memory", persist=False) """ - new_namespace = await self.AD.state.add_namespace(namespace, writeback, persist, self.name) - match new_namespace: - case Path() | str(): - new_namespace = str(new_namespace) - self.AD.state.app_added_namespaces.add(new_namespace) - return new_namespace - case _: - self.logger.warning("Namespace %s already exists or was not created", namespace) + match await self.AD.state.add_namespace( + namespace, + utils.ADWritebackType(writeback), + persist, + self.name + ): + case Path() as ns_path: + return str(ns_path) + case False | None: + return None @utils.sync_decorator async def remove_namespace(self, namespace: str) -> dict[str, Any] | None: """Remove a user-defined namespace, which has a database file associated with it. - See the `namespace documentation `__ for more information. + See :py:ref:`app_namespaces` for more information. Args: namespace (str): The namespace to be removed, which must not be the current namespace. @@ -709,32 +725,22 @@ async def list_namespaces(self) -> list[str]: return self.AD.state.list_namespaces() @utils.sync_decorator - async def save_namespace(self, namespace: str | None = None) -> None: - """Saves entities created in user-defined namespaces into a file. + async def save_namespace(self, namespace: str | None = None) -> bool: + """Saves the given state namespace to its corresponding file. - This way, when AD restarts these entities will be reloaded into AD with its - previous states within the namespace. This can be used as a basic form of - non-volatile storage of entity data. Depending on the configuration of the - namespace, this function can be setup to constantly be running automatically - or only when AD shutdown. This function also allows for users to manually - execute the command as when needed. + This is only relevant for persistent namespaces, which if not set to ``safe`` buffers changes in memory and only + periodically writes them to disk. This function manually forces a write of all the changes since the last save + to disk. See the :py:ref:`app_namespaces` docs section for more information. Args: - namespace (str, optional): Namespace to use for the call. See the section on - `namespaces `__ for a detailed description. - In most cases it is safe to ignore this parameter. + namespace (str, optional): Namespace to save. If not specified, the current app namespace will be used. Returns: - None. - - Examples: - Save all entities of the default namespace. - - >>> self.save_namespace() + bool: `True` if the namespace was saved successfully, `False` otherwise. """ namespace = namespace if namespace is not None else self.namespace - await self.AD.state.save_namespace(namespace) + return await self.AD.state.save_namespace(namespace) # # Utility diff --git a/appdaemon/appdaemon.py b/appdaemon/appdaemon.py index 9fa340550..623c23ecd 100755 --- a/appdaemon/appdaemon.py +++ b/appdaemon/appdaemon.py @@ -376,6 +376,7 @@ def start(self) -> None: self.thread_async.start() self.sched.start() self.utility.start() + self.state.start() if self.apps_enabled: self.app_management.start() diff --git a/appdaemon/models/config/misc.py b/appdaemon/models/config/misc.py index f6ac2ec1f..00833c509 100644 --- a/appdaemon/models/config/misc.py +++ b/appdaemon/models/config/misc.py @@ -1,10 +1,14 @@ import json -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from typing import Any, Literal from pydantic import BaseModel, Field, model_validator +from appdaemon.utils import ADWritebackType + +from .common import ParsedTimedelta + LEVELS = Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] @@ -31,8 +35,9 @@ class FilterConfig(BaseModel): class NamespaceConfig(BaseModel): - writeback: Literal["safe", "hybrid"] | None = None + writeback: ADWritebackType | None = None persist: bool = Field(default=False, alias="persistent") + save_interval: ParsedTimedelta = Field(default=timedelta(seconds=1)) @model_validator(mode="before") @classmethod @@ -40,14 +45,14 @@ def validate_persistence(cls, values: Any): """Sets persistence to True if writeback is set to safe or hybrid.""" match values: case {"writeback": wb} if wb is not None: - values["persistent"] = True + values["persist"] = True case _ if getattr(values, "writeback", None) is not None: - values.persistent = True + values.persist = True return values @model_validator(mode="after") def validate_writeback(self): """Makes the writeback safe by default if persist is set to True.""" if self.persist and self.writeback is None: - self.writeback = "safe" + self.writeback = ADWritebackType.safe return self diff --git a/appdaemon/state.py b/appdaemon/state.py index 3e9b74f84..851ac267c 100644 --- a/appdaemon/state.py +++ b/appdaemon/state.py @@ -1,16 +1,19 @@ +import asyncio +import sys import threading import traceback import uuid from collections.abc import Mapping from copy import copy, deepcopy from datetime import timedelta +from enum import Enum from logging import Logger from pathlib import Path -from typing import (TYPE_CHECKING, Any, Awaitable, List, Optional, - Protocol, Set, Union, overload) +from typing import TYPE_CHECKING, Any, Awaitable, List, Literal, Optional, Protocol, Set, overload from . import exceptions as ade from . import utils +from .utils import ADWritebackType if TYPE_CHECKING: from .adbase import ADBase @@ -25,7 +28,15 @@ class AsyncStateCallback(Protocol): def __call__(self, entity: str, attribute: str, old: Any, new: Any, **kwargs: Any) -> Awaitable[None]: ... -StateCallbackType = Union[StateCallback, AsyncStateCallback] +StateCallbackType = StateCallback | AsyncStateCallback + + +class StateServices(str, Enum): + SET = "set" + ADD_ENTITY = "add_entity" + REMOVE_ENTITY = "remove_entity" + ADD_NAMESPACE = "add_namespace" + REMOVE_NAMESPACE = "remove_namespace" class State: @@ -71,31 +82,55 @@ def __init__(self, ad: "AppDaemon"): def namespace_path(self) -> Path: return self.AD.config_dir / "namespaces" - # @property - # def namespace_db_path(self) -> Path: - # return self.namespace_path / + def start(self) -> None: + self.AD.loop.create_task(self.periodic_save(1.0), name="periodic save of hybrid namespaces") def stop(self) -> None: - self.save_all_namespaces() + self.close_namespaces() def namespace_db_path(self, namespace: str) -> Path: - return self.namespace_path / f"{namespace}" + path = (self.namespace_path / f"{namespace}") + if sys.version_info.minor < 13: + path = path.with_suffix("") + else: + path = path.with_suffix(".db") + return path + + def namespace_exists(self, namespace: str) -> bool: + return namespace in self.state async def add_namespace( self, namespace: str, - writeback: str, + writeback: ADWritebackType, persist: bool, name: str | None = None, - ) -> Path | bool | None: # fmt: skip - """Used to Add Namespaces from Apps""" + ) -> Path | Literal[False] | None: # fmt: skip + """Add a state namespace. + + Fires a ``__AD_NAMESPACE_ADDED`` event in the ``admin`` namespace if it's actually added. + + Args: + namespace (str): Name of the namespace to add. + writeback (Literal["safe", "hybrid"]): Writeback strategy for the namespace. + persist (bool): If ``True``, the namespace will be persistent by saving it to a file. + name (str, optional): Name of the app adding the namespace. + + Returns: + Path | Literal[False] | None: The path to the namespace database file if added successfully, False if it + already exists, or None if the namespace isn't persistent. + """ if self.namespace_exists(namespace): - self.logger.warning("Namespace %s already exists. Cannot process add_namespace from %s", namespace, name) + self.logger.warning("App '%s' tried to add a namespace that already exists: %s", name, namespace) return False if persist: nspath_file = await self.add_persistent_namespace(namespace, writeback) + # This will happen if the namespace already exists + if nspath_file is None: + # Warning message will be logged by self.add_persistent_namespace + return False else: nspath_file = None self.state[namespace] = {} @@ -113,72 +148,91 @@ async def add_namespace( return nspath_file - def namespace_exists(self, namespace: str) -> bool: - return namespace in self.state - - async def remove_namespace(self, namespace: str) -> dict[str, Any] | None: - """Used to Remove Namespaces from Apps + async def remove_namespace(self, namespace: str) -> utils.PersistentDict | dict[str, Any] | None: + """Remove a state namespace. Must not be configured by the appdaemon.yaml file, and must have been added by an + app. Fires an ``__AD_NAMESPACE_REMOVED`` event in the ``admin`` namespace if it's actually removed. """ - if self.state.pop(namespace, False): - nspath_file = await self.remove_persistent_namespace(namespace) - self.app_added_namespaces.remove(namespace) + if namespace in self.AD.config.namespaces: + self.logger.warning("Cannot delete namespace '%s', because it's configured by file.", namespace) + return + elif namespace not in self.app_added_namespaces: + self.logger.warning("Cannot delete namespace '%s', because it wasn't made by an app.", namespace) + return - self.logger.warning("Namespace %s, has ben removed", namespace) + match state := self.state.pop(namespace, False): + case utils.PersistentDict(): + nspath_file = await self.remove_persistent_namespace(namespace, state) + case dict(): + nspath_file = None + case False | _: + self.logger.warning("Cannot delete namespace '%s', because it doesn't exist", namespace) + return - data = { + self.app_added_namespaces.remove(namespace) + await self.AD.events.process_event( + "admin", { "event_type": "__AD_NAMESPACE_REMOVED", "data": {"namespace": namespace, "database_filename": nspath_file}, - } - - await self.AD.events.process_event("admin", data) - - elif namespace in self.state: - self.logger.warning("Cannot delete namespace %s, as not an app defined namespace", namespace) - - else: - self.logger.warning("Namespace %s doesn't exists", namespace) - - # @utils.warning_decorator(error_text='Unexpected error in add_persistent_namespace') - async def add_persistent_namespace(self, namespace: str, writeback: str) -> Path | None: - """Used to add a database file for a created namespace. + }, + ) + return state - Needs to be an async method to make sure it gets run from the event loop in the - main thread. Otherwise, the DbfilenameShelf can get messed up because it's not - thread-safe. In some systems, it'll complain about being accessed from multiple - threads.""" + async def add_persistent_namespace( + self, + namespace: str, + writeback: ADWritebackType = ADWritebackType.safe, + ) -> Path | None: + """Add a namespace that's stored in a persistent file. + + This needs to be an async method to make sure it gets run from the event loop in the main thread. Otherwise, the + :py:class:`~shelve.DbfilenameShelf` can get messed up because it's not thread-safe. In some systems, it'll + complain about being accessed from multiple threads, depending on what database driver is used in the + background. + """ match self.state.get(namespace): case utils.PersistentDict(): self.logger.info(f"Persistent namespace '{namespace}' already initialized") return ns_db_path = self.namespace_db_path(namespace) - safe = writeback == "safe" + wb = ADWritebackType(writeback) + self.logger.debug( + "Creating persistent namespace '%s' at %s with writeback_strategy=%s", + namespace, + ns_db_path.name, + wb.name + ) # fmt: skip try: - self.state[namespace] = utils.PersistentDict(ns_db_path, safe) + self.state[namespace] = utils.PersistentDict(ns_db_path, writeback_type=wb) except Exception as exc: raise ade.PersistentNamespaceFailed(namespace, ns_db_path) from exc - current_thread = threading.current_thread().name - self.logger.info(f"Persistent namespace '{namespace}' initialized from {current_thread}") - return ns_db_path + else: + current_thread = threading.current_thread().name + self.logger.info("Persistent namespace '%s' initialized from %s", namespace, current_thread) + return ns_db_path - @utils.executor_decorator - def remove_persistent_namespace(self, namespace: str) -> Path | None: + async def remove_persistent_namespace(self, namespace: str, state: utils.PersistentDict) -> Path | None: """Used to remove the file for a created namespace""" - try: - ns_db_path = self.namespace_db_path(namespace) - if ns_db_path.exists(): - ns_db_path.unlink() - return ns_db_path + state.close() except Exception: self.logger.warning("-" * 60) - self.logger.warning("Unexpected error in namespace removal") + self.logger.warning("Unexpected error closing namespace '%s':", namespace) self.logger.warning("-" * 60) self.logger.warning(traceback.format_exc()) self.logger.warning("-" * 60) + else: + for ns_file in state.filepath.parent.iterdir(): + if ns_file.is_file() and ns_file.stem == state.filepath.stem: + try: + await asyncio.to_thread(ns_file.unlink) + self.logger.debug("Removed persistent namespace file '%s'", ns_file.name) + except Exception as e: + self.logger.error('Error removing namespace file %s: %s', ns_file.name, e) + continue def list_namespaces(self) -> List[str]: return list(self.state.keys()) @@ -669,48 +723,52 @@ async def add_to_attr(self, name: str, namespace: str, entity_id: str, attr, i): state["attributes"][attr] = copy(state["attributes"][attr]) + i await self.set_state(name, namespace, entity_id, attributes=state["attributes"]) - async def state_services(self, namespace, domain, service, kwargs): - self.logger.debug("state_services: %s, %s, %s, %s", namespace, domain, service, kwargs) - if service in ["add_entity", "remove_entity", "set"]: - if "entity_id" not in kwargs: - self.logger.warning("Entity not specified in %s service call: %s", service, kwargs) - return - - else: - entity_id = kwargs["entity_id"] - del kwargs["entity_id"] + def register_state_services(self, namespace: str) -> None: + """Register the set of state services for the given namespace.""" + for service_name in StateServices: + self.AD.services.register_service(namespace, "state", service_name, self.AD.state._state_service) - elif service in ["add_namespace", "remove_namespace"]: - if "namespace" not in kwargs: - self.logger.warning("Namespace not specified in %s service call: %s", service, kwargs) + async def _state_service( + self, + namespace: str, + domain: str, + service: str, + *, + entity_id: str | None = None, + persist: bool = False, + writeback: Literal["safe", "hybrid"] = "safe", + **kwargs: Any + ) -> Any | None: + self.logger.debug("state_services: %s, %s, %s, %s", namespace, domain, service, kwargs) + match StateServices(service): + case StateServices.SET | StateServices.ADD_ENTITY | StateServices.REMOVE_ENTITY: # fmt: skip + if entity_id is None: + self.logger.warning("Entity not specified in %s service call: %s", service, kwargs) + return + match service: + case StateServices.SET: + return await self.set_state(domain, namespace, entity_id, **kwargs) + case StateServices.REMOVE_ENTITY: + return await self.remove_entity(namespace, entity_id) + case StateServices.ADD_ENTITY: + state = kwargs.get("state") + attributes = kwargs.get("attributes") + return await self.add_entity(namespace, entity_id, state, attributes) + case StateServices.ADD_NAMESPACE | StateServices.REMOVE_NAMESPACE: + if namespace is None: + self.logger.warning("Namespace not specified in %s service call: %s", service, kwargs) + return + match service: + case StateServices.ADD_NAMESPACE: + assert isinstance(persist, bool), "persist must be a boolean" + assert writeback in ("safe", "hybrid"), "writeback must be 'safe' or 'hybrid'" + return await self.add_namespace(namespace, writeback, persist, kwargs.get("name")) + case StateServices.REMOVE_NAMESPACE: + return await self.remove_namespace(namespace) + case _: + self.logger.warning("Unknown service in state service call: %s", kwargs) return - else: - namespace = kwargs["namespace"] - del kwargs["namespace"] - - if service == "set": - await self.set_state(domain, namespace, entity_id, **kwargs) - - elif service == "remove_entity": - await self.remove_entity(namespace, entity_id) - - elif service == "add_entity": - state = kwargs.get("state") - attributes = kwargs.get("attributes") - await self.add_entity(namespace, entity_id, state, attributes) - - elif service == "add_namespace": - writeback = kwargs.get("writeback") - persist = kwargs.get("persist") - await self.add_namespace(namespace, writeback, persist, kwargs.get("name")) - - elif service == "remove_namespace": - await self.remove_namespace(namespace) - - else: - self.logger.warning("Unknown service in state service call: %s", kwargs) - @overload async def set_state( self, @@ -821,7 +879,7 @@ async def set_namespace_state(self, namespace: str, state: dict[str, Any], persi self.state[namespace].update(state) else: # first in case it had been created before, it should be deleted - await self.remove_persistent_namespace(namespace) + await self.remove_namespace(namespace) self.state[namespace] = state def update_namespace_state(self, namespace: str | list[str], state: dict): @@ -838,28 +896,57 @@ def update_namespace_state(self, namespace: str | list[str], state: dict): else: self.state[namespace].update(state) - async def save_namespace(self, namespace: str) -> None: + async def save_namespace(self, namespace: str) -> bool: match self.state.get(namespace): - case utils.PersistentDict() as state: - self.logger.debug("Saving namespace: %s", namespace) - state.sync() + case None: + self.logger.warning("Namespace: %s does not exist", namespace) + return False + case utils.PersistentDict() as ns: + self.logger.debug("Saving persistent namespace: %s", namespace) + try: + # This could take a while if there's been a lot of changes since the last save, so run it in a separate + # thread to avoid blocking the async event loop + await asyncio.to_thread(ns.sync) + except Exception: + self.logger.warning("Unexpected error saving namespace: %s", namespace) + return False + else: + return True case _: self.logger.warning("Namespace: %s cannot be saved", namespace) + return False - def save_all_namespaces(self) -> None: - self.logger.debug("Saving all namespaces") + def close_namespaces(self) -> None: + """Close all the persistent namespaces, which includes saving them.""" + self.logger.debug("Closing all namespaces") for ns, state in self.state.items(): - match state: - case utils.PersistentDict() as state: - self.logger.debug("Saving namespace: %s", ns) - state.sync() - self.logger.debug("Closing namespace: %s", ns) - state.close() + try: + match state: + case utils.PersistentDict(): + self.logger.info("Closing persistent namespace: %s", ns) + state.close() + except Exception: + self.logger.error("Unexpected error saving namespace: %s", ns) + self.logger.error(traceback.format_exc()) + + async def periodic_save(self, interval: str | int | float | timedelta) -> None: + """Periodically save all namespaces that are persistent with writeback_type 'hybrid'""" + interval = utils.parse_timedelta(interval).total_seconds() + while not self.AD.stopping: + self.save_hybrid_namespaces() + await self.AD.utility.sleep(interval, timeout_ok=True) def save_hybrid_namespaces(self) -> None: - for ns_name, cfg in self.AD.namespaces.items(): - if cfg.writeback == "hybrid" and isinstance((ns := self.state.get(ns_name)), utils.PersistentDict): - ns.sync() + """Save all the persistent namespaces with the hybrid writeback type""" + for ns_name, ns_state in self.state.items(): + try: + match ns_state: + case utils.PersistentDict(writeback_type=ADWritebackType.hybrid) as persistent_state: + self.logger.debug("Saving hybrid persistent namespace: %s", ns_name) + persistent_state.sync() + except Exception: + self.logger.error("Unexpected error saving hybrid namespace: %s", ns_name) + self.logger.error(traceback.format_exc()) # # Utilities diff --git a/appdaemon/utility_loop.py b/appdaemon/utility_loop.py index af253db8b..34d492acd 100644 --- a/appdaemon/utility_loop.py +++ b/appdaemon/utility_loop.py @@ -119,13 +119,9 @@ async def _register_services(self): """Register core AppDaemon services for state management, events, sequences, and admin functions.""" # Register state services for ns in self.AD.state.list_namespaces(): - # only default, rules or it belongs to a local plugin. Don't allow for admin/appdaemon/global namespaces - if ns in ["default", "rules"] or ns in self.AD.plugins.plugin_objs or ns in self.AD.namespaces: - self.AD.services.register_service(ns, "state", "add_namespace", self.AD.state.state_services) - self.AD.services.register_service(ns, "state", "add_entity", self.AD.state.state_services) - self.AD.services.register_service(ns, "state", "set", self.AD.state.state_services) - self.AD.services.register_service(ns, "state", "remove_namespace", self.AD.state.state_services) - self.AD.services.register_service(ns, "state", "remove_entity", self.AD.state.state_services) + if ns in ("admin", "appdaemon", "global"): + continue # Don't allow admin/appdaemon/global namespaces + self.AD.state.register_state_services(ns) # Register fire_event services self.AD.services.register_service(ns, "event", "fire", self.AD.events.event_services) @@ -210,7 +206,7 @@ async def loop(self): await self.AD.threading.check_overdue_and_dead_threads() # Save any hybrid namespaces - self.AD.state.save_hybrid_namespaces() + # self.AD.state.save_hybrid_namespaces() # Run utility for each plugin self.AD.plugins.run_plugin_utility() diff --git a/appdaemon/utils.py b/appdaemon/utils.py index 720f440ff..9e3552295 100644 --- a/appdaemon/utils.py +++ b/appdaemon/utils.py @@ -18,6 +18,7 @@ import traceback from collections.abc import Awaitable, Generator, Iterable, Mapping, Sequence from datetime import datetime, time, timedelta, tzinfo +from enum import Enum from functools import wraps from logging import Logger from pathlib import Path @@ -112,18 +113,66 @@ def format_tuple(self, value, indent): return "(%s)" % (",".join(items) + self.lfchar + self.htchar * indent) +class ADWritebackType(str, Enum): + """Represents different strategies for AppDaemon to write persistent namespaces to disk. + + See :py:meth:`shelve.open` for more details about writeback modes for the underlying shelf object and + `user Defined Namespaces `_ + """ + safe = "safe" + """The namespace is written to disk every time a change is made so will be up to date even if a crash happens. The + downside is that there is a possible performance impact for systems with slower disks, or that set state on many + UDNS at a time.""" + hybrid = "hybrid" + """A compromise setting in which the namespaces are saved periodically (once each time around the utility loop, + usually once every second- with this setting a maximum of 1 second of data will be lost if AppDaemon crashes.""" + + class PersistentDict(shelve.DbfilenameShelf): """ - Dict-like object that uses a Shelf to persist its contents. + Dict-like object that uses a shelf to persist its contents. + + A “shelf” is a persistent, dictionary-like object. The difference with “dbm” databases is that the values (not the + keys!) in a shelf can be essentially arbitrary Python objects — anything that the pickle module can handle. This + includes most class instances, recursive data types, and objects containing lots of shared sub-objects. The keys are + ordinary strings. """ - def __init__(self, filename: str | Path, safe: bool, **kwargs): - filename = Path(filename).resolve().as_posix() - # writeback=True allows for mutating objects in place, like with a dict. - super().__init__(filename, writeback=True) - self.safe = safe + writeback_type: ADWritebackType + safe: bool + rlock: threading.RLock + filepath: Path + + def __init__(self, filename: str | Path, writeback_type: ADWritebackType = ADWritebackType.safe) -> None: + match writeback_type: + case ADWritebackType.safe: + # This is the default condition for shelf objects, which saves all assignments to the dict to disk. + writeback = False + case ADWritebackType.hybrid: + # From the Python docs: + # If the optional writeback parameter is set to True, all entries accessed are also cached in memory, + # and written back on sync() and close(); this can make it handier to mutate mutable entries in the + # persistent dictionary, but, if many entries are accessed, it can consume vast amounts of memory for + # the cache, and it can make the close operation very slow since all accessed entries are written back + # (there is no way to determine which accessed entries are mutable, nor which ones were actually mutated). + writeback = True + + filepath = Path(filename).resolve() + if sys.version_info.minor < 13: + filepath = filepath.with_suffix("") + else: + filepath = filepath.with_suffix(".db") + + super().__init__(str(filepath), writeback=writeback) + self.writeback_type = writeback_type + self.safe = writeback_type == ADWritebackType.safe self.rlock = threading.RLock() - self.update(new=kwargs) + self.filepath = filepath + # print(f'PersistentDict using writeback mode: {self.writeback_type}, writeback={writeback}') + + @property + def is_safe(self) -> bool: + return self.writeback_type == ADWritebackType.safe def __contains__(self, key): with self.rlock: @@ -165,13 +214,12 @@ def sync(self): with self.rlock: super().sync() - def update(self, new: dict, *args, save=True, **kwargs): + def update(self, new: dict[str, Any], save: bool = False) -> None: with self.rlock: - for key, value in dict(*args, **new, **kwargs).items(): - # use super().__setitem__() to prevent multiple save() calls - super().__setitem__(key, value) - if self.safe and save: - self.sync() + for key, val in new.items(): + super().__setitem__(key, val) + if self.is_safe or save: + self.sync() class AttrDict(dict): diff --git a/docs/AD_API_REFERENCE.rst b/docs/AD_API_REFERENCE.rst index ee46656b8..f55901ae9 100644 --- a/docs/AD_API_REFERENCE.rst +++ b/docs/AD_API_REFERENCE.rst @@ -381,7 +381,9 @@ Namespace .. automethod:: appdaemon.adapi.ADAPI.set_namespace .. automethod:: appdaemon.adapi.ADAPI.get_namespace +.. automethod:: appdaemon.adapi.ADAPI.add_namespace .. automethod:: appdaemon.adapi.ADAPI.list_namespaces +.. automethod:: appdaemon.adapi.ADAPI.namespace_exists .. automethod:: appdaemon.adapi.ADAPI.save_namespace diff --git a/docs/APPGUIDE.rst b/docs/APPGUIDE.rst index 5f408f3c0..5d67a8850 100644 --- a/docs/APPGUIDE.rst +++ b/docs/APPGUIDE.rst @@ -2852,106 +2852,193 @@ The ``type`` parameter defines which of the plugins are used, and the parameters As you can see, the parameters for both hass instances are similar, and it supports all the parameters described in the installation section of the docs - here I am just using a subset. +.. _namespaces: + Namespaces ---------- -A critical piece of this is the concept of ``namespaces``. Each plugin has an optional ``namespace`` directive. If you have more than 1 plugin of any type, their state is separated into namespaces, and you need to name those namespaces using the ``namespace`` parameter. If you don't supply a namespace, the namespace defaults to ``default`` and this is the default for all areas of AppDaemon meaning that if you only have one plugin you don't need to worry about namespace at all. +Namespaces primarily organize AppDaemon's internal state and event handling. The default namespace is ``default``, and +if you are only using a single plugin, you don't need to worry about namespaces at all because everything will happen in +the ``default`` namespace. -In the case above, the first instance had no namespace so its namespace will be called ``default``. The second hass namespace will be ``hass2`` and so on. +Plugin Namespaces +~~~~~~~~~~~~~~~~~ -These namespaces can be accessed separately by the various API calls to keep things separate, but individual Apps can switch between namespaces at will as well as monitor all namespaces in certain calls like ``listen_state()`` or ``listen_event()`` by setting the namespace to ``global``. +If only using a single plugin, this will default to ``default``, and no further action is required. -Use of Namespaces in Apps -~~~~~~~~~~~~~~~~~~~~~~~~~ +However, if using multiple plugins, each plugin needs its own namespace to keep their states and events separate. Only +one of them can use the ``default`` namespace, so all the others need to have a namespace specified in their +configuration. For example, if using 2 instances of the :py:ref:`hass_plugin`, one of them needs to have +a namespace other than ``default`` specified. -Each App maintains a current namespace at all times. At initialization, this is set to ``default``. This means that if you only have a single plugin, you don't need to worry about namespaces at all as everything will just work. +.. caution:: + Use caution when using plugin namespaces for other things because plugins can overwrite anything in their namespace + state at any time, for instance when they connect or restart. -There are 2 ways to work with namespaces in apps. The first is to make a call to ``set_namespace()`` whenever you want to change namespaces. For instance, if in the configuration above, you wanted a particular App to work entirely with the ``HASS2`` plugin instance, all you would need to do is put the following code at the top of your ``initialize()`` function: +.. code-block:: yaml + :caption: Example plugin config for Hass plugin and an MQTT plugin + + appdaemon: + ... # other config here + plugins: + main_hass: # this plugin will have the `default` namespace + type: hass + ... # other config here + zigbee2mqtt: # this plugin will have the `mqtt` namespace + type: mqtt + namespace: mqtt + ... # other config here -.. code:: python +.. code-block:: yaml + :caption: Example plugin config for 2 instances of the Hass plugin + + appdaemon: + ... # other config here + plugins: + main_hass: # this instance will have the `default` namespace + type: hass + ... # other config here + other_hass: # this instance will have the `hass2` namespace + type: hass + namespace: hass2 + ... # other config here + +.. _app_namespaces: + +App Namespaces +~~~~~~~~~~~~~~ - self.set_namespace("hass2") +Apps all start in the ``default`` namespace, but they can change their namespace at any time using +:py:meth:`~appdaemon.adapi.ADAPI.set_namespace`. Doing so changes the namespace for subsequent calls to methods like +:py:meth:`~appdaemon.adapi.ADAPI.listen_event` or :py:meth:`~appdaemon.adapi.ADAPI.listen_state`, among many others. +Namespaces can also be specified on a per-call basis for most API calls using the ``namespace`` parameter. The namespace +``global`` is a special value that will make these calls apply to all namespaces. + +.. code-block:: python + :caption: Continued example with 2 instances of the Hass plugin -Note that you should use the value of the namespace parameter, not the name of the plugin section. From that point on, all state changes, events, service calls, etc. will apply to the ``HASS2`` instance and the ``HASS1`` and ``DUMMY`` instances will be ignored. This is convenient for the case in which you don't need to switch between namespaces. + from appdaemon.plugins.hass import Hass -In addition, most of the API calls allow you to optionally supply a namespace for them to operate under. This will override the namespace set by ``set_namespace()`` for that call only. -For example: + class MyApp(Hass): + def initialize(self): + self.set_namespace("hass2") + # The app will now operate on the plugin in the `hass2` namespace by default -.. code:: python + # The app has been set to the `hass2` namespace so this will get the entity from the other_hass plugin + state = self.get_state("light.light1") - self.set_namespace("hass2") - # Get the entity value from the HASS2 plugin - # Since the HASS2 plugin is configured with a namespace of "hass2" - state = self.get_state("light.light1") + # Get the entity value from the main_hass plugin since it uses the default namespace of `default` + state = self.get_state("light.light1", namespace="default") - # Get the entity value from the HASS1 plugin - # Since the HASS1 plugin is configured with a namespace of "default" - state = self.get_state("light.light1", namespace="default") + # The app is still using the `hass2` namespace -In this way it is possible to use a single App to work with multiple namespaces easily and quickly. -A Note on Callbacks +Callback Namespaces ~~~~~~~~~~~~~~~~~~~ -One important thing to note, when working with namespaces is that callbacks will honor the namespace they were created with. So if for instance, you create a ``listen_state()`` callback with a namespace of ``default`` then later change the namespace to ``hass1``, that callback will continue to listen to the ``default`` namespace. +The namespace for a callback is the namespace of the app that at the time the callback is registered, but that can +be overridden by passing the ``namespace`` argument to the registration method. Callbacks are only effective for the +events or state changes in the namespace they are created in, with the exception of the namespace ``global`` which +causes callbacks to listen in all namespaces. -For instance: +For example, these are 3 ways to register state callbacks in multiple namespaces: -.. code:: python - - self.set_namespace("default") - self.listen_state(callback) - self.set_namespace("hass2") - self.listen_state(callback) - self.set_namespace("dummy1") +.. code-block:: python -This will leave us with 2 callbacks, one listening for state changes in ``default`` and one for state changes in ``hass2``, regardless of the final value of the namespace. + from appdaemon.adapi import ADAPI -Similarly: -.. code:: python + class MyApp(ADAPI): + def initialize(self): + self.register_callbacks() + # self.register_callbacks2() + # self.register_callbacks3() + + def register_callbacks(self): + for ns in ("default", "hass2"): + self.set_namespace(ns) + self.listen_state(self.state_callback, "light.light1") + self.set_namespace("default") + + def register_callbacks2(self): + for ns in ("default", "hass2"): + self.listen_state(self.state_callback, "light.light1", namespace=ns) + + def register_callbacks3(self): + self.listen_state(self.state_callback, "light.light1", namespace='global') + + def state_callback(self, entity, attribute, old, new, **kwargs) -> None: + self.log(f"State change for {entity}: {new}") + return + +register_callbacks + Uses :py:meth:`~appdaemon.adapi.ADAPI.set_namespace` to change the namespace of the app before registering each + callback, which causes the callback to be registered once for each namespace. +register_callbacks2 + Uses the ``namespace`` parameter of :py:meth:`~appdaemon.adapi.ADAPI.listen_state` to register the callback in + each namespace. +register_callbacks3 + Uses the global namespace to register a single callback that will listen to for state changes in all namespaces. + +User-Defined (Persistent) Namespaces +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Users can define custom namespaces, which is recommended for custom entities to avoid clashing with anything in the +namespaces used/managed by the plugins. These user-defined namespaces are guaranteed to not be changed by plugins, and +can additionally be made persistent across AppDaemon restarts, using a thread-safe version of +:py:class:`~shelve.DbfilenameShelf` to save their state to disk. These namespaces are available to all apps the same way +that plugin namespaces are. + +There are 2 `writeback` modes for persistent namespaces, ``safe`` and ``hybrid``. + +``safe`` + The namespace state is written to disk every time a change is made so will be up to date even if a crash happens. The + downside is that there is a possible performance impact for systems with slower disks, or that set many states. +``hybrid`` + The namespaces state is only saved periodically, and state changes between writes are cached in memory. This can + greatly improve performance in systems with many states changes. + +Defined in Configuration +^^^^^^^^^^^^^^^^^^^^^^^^ - self.set_namespace("dummy2") - self.listen_state(callback, namespace="default") - self.listen_state(callback, namespace="hass2") - self.set_namespace("dummy1") +One way users can define namespaces is in the ``namespaces`` section of the ``appdaemon.yaml`` configuration file. -This code fragment will achieve the same result as above since the namespace is being overridden, and will -keep the same value for that callback regardless of what the namespace is set to. +For example, here we are defining 3 new namespaces, ``my_namespace1``, ``other_namespace`` and ``temp_namespace``. The +first 2 are written to disk so that they survive restarts, and the last one is not persistent and will only exist in +memory. -User Defined Namespaces -~~~~~~~~~~~~~~~~~~~~~~~ +.. code-block:: yaml -Each plugin has it's own unique namespace as described above, and they are pretty much in control of those -namespaces. It is possible to set a state in a plugin managed namespace which can be used as a temporary -variable or even as a way of signalling other apps using ``listen_state()`` however this is not recommended: + appdaemon: + ... # other config here + namespaces: + my_namespace: + writeback: safe + other_namespace: + writeback: hybrid + temp_namespace: + persist: false # this namespace will only be in memory -- Plugin managed namespaces may be overwritten at any time by the plugin -- They will likely be overwritten when the plugin restarts even if AppDaemon does not -- They will not survive a restart of AppDaemon because it is regarded as the job of the plugin to reconstruct it's state and it knows nothing about any additional variables you have added. Although this technique can still be useful, for example, to add sensors to Home Assistant, a better alternative for Apps to use are User Defined Namespaces. +Defined by Apps +^^^^^^^^^^^^^^^ +Another way users can create namespaces is by calling :py:meth:`~appdaemon.adapi.ADAPI.add_namespace` or +:py:meth:`~appdaemon.adapi.ADAPI.set_namespace` from within an app. -A User Defined Namespace is a new area of storage for entities that is not managed by a plugin. UDMs are guaranteed -not to be changed by any plugin and are available to all apps just the same as a plugin-based namespace. UDMs also -survive AppDaemon restarts and crashes, creating durable storage for saving the information and communicating with -other apps via ``listen_state()`` and ``set_state()``. +.. code-block:: python + :caption: Example of creating a new persistent namespace from an app -They are configured in the ``appdaemon.yaml`` file as follows: + from appdaemon.adapi import ADAPI -.. code:: yaml - namespaces: - my_namespace: - # writeback is safe or hybrid - writeback: safe - my_namespace2: - writeback: hybrid + class MyApp(ADAPI): + def initialize(self): + # A new persistent namespace called `storage` will be added if it doesn't already exist + self.set_namespace("storage", writeback="hybrid") -Here we are defining 3 new namespaces - you can have as many as you want. Their names are ``my_namespace1``, ``my_namespace2`` and ``my_namespace3``. UDMs are written to disk so that they survive restarts, and this can be done in 3 different ways, set by the writeback parameter for each UDM. They are: + # Do stuff in the new namespace... -- ``safe`` - the namespace is written to disk every time a change is made so will be up to date even if a crash happens. The downside is that there is a possible performance impact for systems with slower disks, or that set state on many UDMs at a time. -- ``hybrid`` - a compromise setting in which the namespaces are saved periodically (once each time around the utility loop, usually once every second- with this setting a maximum of 1 second of data will be lost if AppDaemon crashes. Using Multiple APIs From One App -------------------------------- diff --git a/docs/DEV.rst b/docs/DEV.rst index d108b71a5..aea8f7526 100644 --- a/docs/DEV.rst +++ b/docs/DEV.rst @@ -12,26 +12,26 @@ Python ^^^^^^ `uv `_ - An extremely fast Python package and project manager, written in Rust. + An extremely fast Python package and project manager, written in Rust. `ruff `_ - An extremely fast Python linter and code formatter, written in Rust. + An extremely fast Python linter and code formatter, written in Rust. `pytest `_ - The pytest framework makes it easy to write small, readable tests, and can scale to support complex functional testing for applications and libraries. + The pytest framework makes it easy to write small, readable tests, and can scale to support complex functional testing for applications and libraries. `pre-commit `_ - A framework for managing and maintaining multi-language pre-commit hooks. Once enabled, these run things like the linter on every commit. + A framework for managing and maintaining multi-language pre-commit hooks. Once enabled, these run things like the linter on every commit. `sphinx `_ for `readthedocs `_ - Sphinx is a powerful documentation generator that has many features for writing technical documentation. Sphinx is written in Python, and supports documentation written in reStructuredText and Markdown. + Sphinx is a powerful documentation generator that has many features for writing technical documentation. Sphinx is written in Python, and supports documentation written in reStructuredText and Markdown. `VSCode `_ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The AppDaemon repo itself contains some configuration specifically for VSCode, which makes some routine tasks easier. `Python `_ - The Python extension makes Visual Studio Code an excellent Python editor, works on any operating system, and is usable with a variety of Python interpreters. + The Python extension makes Visual Studio Code an excellent Python editor, works on any operating system, and is usable with a variety of Python interpreters. `Python testing `_ - The Python extension builds on the built-in testing features in VS Code and provides test discovery, test coverage, and running and debugging tests for Python's built-in unittest framework and pytest. + The Python extension builds on the built-in testing features in VS Code and provides test discovery, test coverage, and running and debugging tests for Python's built-in unittest framework and pytest. `Ruff Extension `_ - A Visual Studio Code extension for Ruff, an extremely fast Python linter and code formatter, written in Rust. Available on the Visual Studio Marketplace. + A Visual Studio Code extension for Ruff, an extremely fast Python linter and code formatter, written in Rust. Available on the Visual Studio Marketplace. Dev Setup --------- @@ -82,7 +82,7 @@ The extra ``doc`` is optional, but needed to work on the documentation. .. code-block:: console :caption: Create documentation environment - $ uv sync --extra doc + $ uv sync --group doc Pre-Commit Hooks ^^^^^^^^^^^^^^^^ @@ -182,7 +182,7 @@ Assistance with the docs is always welcome, whether its fixing typos and incorre :caption: Run sphinx-autobuild $ uv run \ - --extra doc \ + --group doc \ sphinx-autobuild \ --show-traceback --fresh-env \ --host 0.0.0.0 --port 9999 \ diff --git a/docs/HASS_API_REFERENCE.rst b/docs/HASS_API_REFERENCE.rst index ac34fa63c..28e6f543b 100644 --- a/docs/HASS_API_REFERENCE.rst +++ b/docs/HASS_API_REFERENCE.rst @@ -4,6 +4,8 @@ Hass Plugin/API About ----- +.. _hass_plugin: + Hass Plugin ~~~~~~~~~~~ diff --git a/docs/TESTING.rst b/docs/TESTING.rst index 4fa4f9d80..254ce269a 100644 --- a/docs/TESTING.rst +++ b/docs/TESTING.rst @@ -7,7 +7,7 @@ AppDaemon uses `pytest `_ and `pytest-asynci .. literalinclude:: ../pyproject.toml :language: toml - :lines: 94-104 + :lines: 88-96 :caption: Pytest configuration options - Pytest-asyncio manages creating the event loop, which is normally handled by :py:class:`~appdaemon.__main__.ADMain`. diff --git a/pyproject.toml b/pyproject.toml index 448db2254..d38376a2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ authors = [ ] dynamic = ["version"] # The version is computed dynamically. See `tool.setuptools.dynamic` section below. license-files = [ "LICENSE.md" ] -requires-python = ">=3.10,<3.13" +requires-python = ">=3.10,<3.14" keywords=[ "appdaemon", "home", "automation" ] dependencies = [ 'importlib-metadata; python_version<"3.8"', diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 843b6312a..4a8bc59cb 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -7,7 +7,8 @@ readonly REPO_DIR=$(cd $(dirname $(dirname $(readlink -f "${BASH_SOURCE[0]}"))) rm -rf ${REPO_DIR}/build ${REPO_DIR}/dist if command -v uv >/dev/null 2>&1; then - uv sync -U --all-extras + uv lock + uv sync --inexact echo -n "Building wheel..." uv build --wheel --refresh -q echo "done." diff --git a/scripts/multiplatform-docker-build.sh b/scripts/multiplatform-docker-build.sh index 0cf723d26..0b66ca1bd 100755 --- a/scripts/multiplatform-docker-build.sh +++ b/scripts/multiplatform-docker-build.sh @@ -13,7 +13,7 @@ PLATFORMS="linux/arm64/v8,linux/amd64,linux/arm/v7,linux/arm/v6" IMAGE_TAG="acockburn/appdaemon:local-dev" BUILD_DIR="${REPO_DIR}/build" DOCKER_BUILD_DIR="${BUILD_DIR}/docker" -DOCKERFILE="${REPO_DIR}/Dockerfile.uv" +DOCKERFILE="${REPO_DIR}/Dockerfile" ARCHIVE_NAME="appdaemon-docker.tar" ARCHIVE_PATH="${DOCKER_BUILD_DIR}/${ARCHIVE_NAME}" @@ -117,7 +117,7 @@ docker buildx build \ --platform "$PLATFORMS" \ --tag "$IMAGE_TAG" \ --output "type=oci,dest=${ARCHIVE_PATH}" \ - -f Dockerfile \ + -f ${DOCKERFILE} \ ${REPO_DIR} print_success "Multi-platform images built and exported" diff --git a/tests/conf/apps/apps.yaml b/tests/conf/apps/apps.yaml index 582b7edc1..7156f6155 100644 --- a/tests/conf/apps/apps.yaml +++ b/tests/conf/apps/apps.yaml @@ -28,3 +28,7 @@ test_run_daily: basic_namespace_app: module: namespace_app class: BasicNamespaceTester + +hybrid_namespace_app: + module: namespace_app + class: HybridWritebackTester diff --git a/tests/conf/apps/namespace_app.py b/tests/conf/apps/namespace_app.py index 20a2f32cd..7f9136147 100644 --- a/tests/conf/apps/namespace_app.py +++ b/tests/conf/apps/namespace_app.py @@ -1,3 +1,4 @@ +import functools from datetime import timedelta from typing import Any @@ -10,15 +11,14 @@ class BasicNamespaceTester(ADAPI): def initialize(self) -> None: self.set_namespace(self.custom_test_namespace) self.logger.info('Current namespaces: %s', sorted(self.current_namespaces)) - - self.show_entities() + self.run_in(self.show_entities, 0) exists = self.test_entity.exists() self.logger.info(f"Entity exists: {exists}") if not exists: self.add_entity("sensor.test", state="initial", attributes={"friendly_name": "Test Sensor"}) - self.show_entities() + self.run_in(self.show_entities, 0) non_existence = "sensor.other_entity" self.logger.info("Setting %s in default namespace", non_existence) @@ -28,6 +28,9 @@ def initialize(self) -> None: self.test_entity.listen_state(self.handle_state) self.log(f"Initialized {self.name}") + self.set_namespace('default') + self.remove_namespace(self.custom_test_namespace) + @property def current_namespaces(self) -> set[str]: return set(self.AD.state.state.keys()) @@ -44,11 +47,10 @@ def start_delay(self) -> timedelta: def test_entity(self) -> Entity: return self.get_entity("sensor.test", check_existence=False) - def show_entities(self, *args, **kwargs): + async def show_entities(self, *args, **kwargs) -> None: ns = self.AD.state.state.get(self.custom_test_namespace, {}) entities = sorted(ns.keys()) self.log('Test entities: %s', entities) - return entities def start_test(self, *args, **kwargs: Any) -> None: match kwargs: @@ -62,3 +64,38 @@ def handle_state(self, entity: str, attribute: str, old: Any, new: Any, **kwargs full_state = self.test_entity.get_state('all') self.log(f"Full state: {full_state}") + + def terminate(self) -> None: + self.set_namespace('default') + self.remove_namespace(self.custom_test_namespace) + + +class HybridWritebackTester(ADAPI): + def initialize(self) -> None: + self.AD.logging.get_child("_state") + self.set_namespace(self.custom_test_namespace, writeback="hybrid", persist=True) + self.logger.info("Initialized %s in namespace '%s'", self.name, self.custom_test_namespace) + + self.run_in(self.rapid_changes, self.start_delay) + + @property + def custom_test_namespace(self) -> str: + return self.args.get("custom_namespace", "test_namespace") + + @property + def start_delay(self) -> timedelta: + return timedelta(seconds=self.args.get("start_delay", 1.0)) + + @property + def test_n(self) -> int: + return self.args.get("test_n", 10) + + async def rapid_changes(self, *args, **kwargs) -> None: + for i in range(self.test_n): + func = functools.partial(self.set_state, "sensor.hybrid_test", state=f"change_{i}") + delay = i * 0.05 + self.AD.loop.call_later(delay, func) + + def terminate(self) -> None: + self.set_namespace('default') + self.remove_namespace(self.custom_test_namespace) diff --git a/tests/conftest.py b/tests/conftest.py index 289c030a2..2a5938f52 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -91,6 +91,7 @@ def ad_cfg() -> AppDaemonConfig: # loglevel="INFO", module_debug={ "_app_management": "DEBUG", + "_state": "DEBUG", # "_events": "DEBUG", # "_scheduler": "DEBUG", "_utility": "DEBUG", diff --git a/tests/functional/test_namespaces.py b/tests/functional/test_namespaces.py index d18314f95..e98b9147e 100644 --- a/tests/functional/test_namespaces.py +++ b/tests/functional/test_namespaces.py @@ -3,18 +3,47 @@ import uuid import pytest +from appdaemon.utils import PersistentDict from .utils import AsyncTempTest logger = logging.getLogger("AppDaemon._test") +pytestmark = [ + pytest.mark.ci, + pytest.mark.functional, +] + + +def assert_current_namespace(caplog: pytest.LogCaptureFixture, expected_namespace: str) -> None: + """Assert that the expected namespace is in the current namespaces log.""" + for record in caplog.records: + match record: + case logging.LogRecord(levelno=logging.INFO, msg=str(msg), args=[list(namespaces), *_]): + if msg == 'Current namespaces: %s': + assert expected_namespace in namespaces, f'Expected {expected_namespace} in current namespaces' + return + assert False, f"Did not find log record for current namespaces including {expected_namespace}" + + +def assert_non_existence_warning(caplog: pytest.LogCaptureFixture, correct: int = 1) -> None: + """Assert that a warning about non-existence of an entity in a namespace was logged.""" + non_existence_warnings = [ + r for r in caplog.records + if r.levelno >= logging.WARNING and + r.msg == "Entity %s not found in namespace %s" + ] + assert len(non_existence_warnings) == correct, f"Expected {correct} warning(s) about non-existence should be logged" + + @pytest.mark.asyncio(loop_scope="session") async def test_simple_namespaces(run_app_for_time: AsyncTempTest) -> None: """Test simple namespace functionality.""" test_val = str(uuid.uuid4()) + test_ns = "test_namespace" app_kwargs = { - "custom_namespace": "test_namespace", + "custom_namespace": test_ns, 'start_delay': 0.1, "test_val": test_val, } @@ -25,9 +54,48 @@ async def test_simple_namespaces(run_app_for_time: AsyncTempTest) -> None: # was created in the correct namespace and the state change was detected. assert test_val in caplog.text - non_existence_warnings = [ - r for r in caplog.records - if r.levelno >= logging.WARNING and - r.msg == "Entity %s not found in namespace %s" + # The current namespaces should include the custom test namespace + assert_current_namespace(caplog, test_ns) + + # There should only be one warning about non-existence of the entity in the custom namespace + assert_non_existence_warning(caplog) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_hybrid_writeback(run_app_for_time: AsyncTempTest) -> None: + """Test hybrid namespace functionality. + + The general idea is to create a namespace with hybrid writeback and ensure that it saves correctly. + """ + test_val = str(uuid.uuid4()) + test_ns = "hybrid_test_ns" + app_kwargs = { + "custom_namespace": test_ns, + "test_val": test_val, + "start_delay": 0.5, + "test_n": 10**3, + } + async with run_app_for_time("hybrid_namespace_app", 2.2, **app_kwargs) as (ad, caplog): + match ad.state.state.get(test_ns): + case PersistentDict() as state: + def get_files(): + return list(state.filepath.parent.glob(f"{test_ns}*")) + files = get_files() + assert len(files) > 0, f'Namespace files for {test_ns} should exist, but it does not.' + case _: + assert False, f"Expected a PersistentDict for namespace '{test_ns}'" + assert f"Persistent namespace '{test_ns}' initialized from MainThread" in caplog.text + + saves = [ + record + for record in caplog.records + if record.levelno == logging.DEBUG and + record.msg == "Saving hybrid persistent namespace: %s" ] - assert len(non_existence_warnings) == 1, "Only one warning about non-existence should be logged" + assert len(saves) == 2, "Expected exactly two saves of hybrid persistent namespace" + + files = get_files() + namespace_files = [f.name for f in state.filepath.parent.iterdir() if f.is_file()] + assert not namespace_files, f"Namespace files for {test_ns} should not exist after test, but they do: {namespace_files}" + + assert "dbm.sqlite3.error" not in caplog.text