Merge pull request #2446 from AppDaemon/ns-fix

Namespace fix

Author: acockburn

.gitignore

@@ -46,6 +46,7 @@ nosetests.xml
 coverage.xml
 *,cover
 .hypothesis/
+tests/conf/namespaces
 
 # Translations
 *.mo

appdaemon/adapi.py

@@ -765,9 +765,9 @@ def _check_entity(self, namespace: str, entity_id: str | None) -> None:
         """Ensures that the entity exists in the given namespace"""
         if entity_id is not None and "." in entity_id and not self.AD.state.entity_exists(namespace, entity_id):
             if namespace == "default":
-                self.logger.warning(f"Entity {entity_id} not found in the default namespace")
+                self.logger.warning("Entity %s not found in the default namespace", entity_id)
             else:
-                self.logger.warning(f"Entity {entity_id} not found in namespace {namespace}")
+                self.logger.warning("Entity %s not found in namespace %s", entity_id, namespace)
 
     @staticmethod
     def get_ad_version() -> str:
@@ -817,7 +817,7 @@ async def add_entity(
             >>> self.add_entity('mqtt.living_room_temperature', namespace='mqtt')
 
         """
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         return await self.AD.state.add_entity(namespace, entity_id, state, attributes)
 
     @utils.sync_decorator
@@ -850,7 +850,7 @@ async def entity_exists(self, entity_id: str, namespace: str | None = None) -> b
             >>> if self.entity_exists("mqtt.security_settings", namespace = "mqtt"):
             >>>    #do something
         """
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         return self.AD.state.entity_exists(namespace, entity_id)
 
     @utils.sync_decorator
@@ -877,7 +877,7 @@ async def split_entity(self, entity_id: str, namespace: str | None = None) -> li
             >>>     #do something specific to scenes
 
         """
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         self._check_entity(namespace, entity_id)
         return entity_id.split(".")
 
@@ -907,7 +907,7 @@ async def remove_entity(self, entity_id: str, namespace: str | None = None) -> N
             >>> self.remove_entity('mqtt.living_room_temperature', namespace = 'mqtt')
 
         """
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         await self.AD.state.remove_entity(namespace, entity_id)
 
     @staticmethod
@@ -952,7 +952,7 @@ async def get_plugin_config(self, namespace: str | None = None) -> Any:
             My current position is 50.8333(Lat), 4.3333(Long)
 
         """
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         return self.AD.plugins.get_plugin_meta(namespace)
 
     @utils.sync_decorator
@@ -976,7 +976,7 @@ async def friendly_name(self, entity_id: str, namespace: str | None = None) -> s
             device_tracker.andrew (Andrew Tracker) is on.
 
         """
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         self._check_entity(namespace, entity_id)
         return await self.get_state(
             entity_id=entity_id,
@@ -1584,7 +1584,7 @@ async def listen_state(
         """
         kwargs = dict(new=new, old=old, duration=duration, attribute=attribute, **kwargs)
         kwargs = {k: v for k, v in kwargs.items() if v is not None}
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
 
         # pre-fill some arguments here
         add_callback = functools.partial(
@@ -1724,9 +1724,10 @@ async def get_state(
         if kwargs:
             self.logger.warning(f"Extra kwargs passed to get_state, will be ignored: {kwargs}")
 
+        namespace = namespace if namespace is not None else self.namespace
         return await self.AD.state.get_state(
             name=self.name,
-            namespace=namespace or self.namespace,
+            namespace=namespace,
             entity_id=entity_id,
             attribute=attribute,
             default=default,
@@ -1783,7 +1784,7 @@ async def set_state(
             >>> self.set_state("light.office_1", state="off", namespace="hass")
 
         """
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         if check_existence:
             self._check_entity(namespace, entity_id)
         return await self.AD.state.set_state(
@@ -1846,7 +1847,7 @@ def register_service(self, service: str, cb: Callable, namespace: str | None = N
         self._check_service(service)
         self.logger.debug("register_service: %s, %s", service, kwargs)
 
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         try:
             domain, service = service.split("/", 2)
         except ValueError as e:
@@ -1886,7 +1887,7 @@ def deregister_service(self, service: str, namespace: str | None = None) -> bool
             >>> self.deregister_service("myservices/service1")
 
         """
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         self.logger.debug("deregister_service: %s, %s", service, namespace)
         self._check_service(service)
         return self.AD.services.deregister_service(namespace, *service.split("/"), name=self.name)
@@ -1996,7 +1997,7 @@ async def call_service(
         """
         self.logger.debug("call_service: %s, %s", service, data)
         self._check_service(service)
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
 
         # Check the entity_id if it exists
         if eid := data.get("entity_id"):
@@ -2054,7 +2055,7 @@ async def run_sequence(self, sequence: str | list[dict[str, dict[str, str]]], na
                 ])
 
         """
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         self.logger.debug("Calling run_sequence() for %s from %s", sequence, self.name)
 
         try:
@@ -2197,11 +2198,12 @@ async def listen_event(
         """
         self.logger.debug(f"Calling listen_event() for {self.name} for {event}: {kwargs}")
 
+        namespace = namespace if namespace is not None else self.namespace
         # pre-fill some arguments here
         add_callback = functools.partial(
             self.AD.events.add_event_callback,
             name=self.name,
-            namespace=namespace or self.namespace,
+            namespace=namespace,
             cb=callback,
             timeout=timeout,
             oneshot=oneshot,
@@ -2311,7 +2313,7 @@ async def fire_event(
             # Convert to float if it's not None
             timeout = utils.parse_timedelta(timeout).total_seconds() if timeout is not None else timeout
             kwargs["timeout"] = timeout
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         await self.AD.events.fire_event(namespace, event, **kwargs)
 
     #
@@ -3743,7 +3745,7 @@ async def sleep(delay: float, result: T = None) -> T:
     #
 
     def get_entity(self, entity: str, namespace: str | None = None, check_existence: bool = True) -> Entity:
-        namespace = namespace or self.namespace
+        namespace = namespace if namespace is not None else self.namespace
         if check_existence:
             self._check_entity(namespace, entity)
         return Entity(self, namespace, entity)

appdaemon/state.py

@@ -1,6 +1,7 @@
 import threading
 import traceback
 import uuid
+from collections.abc import Mapping
 from copy import copy, deepcopy
 from datetime import timedelta
 from logging import Logger
@@ -78,7 +79,7 @@ def stop(self) -> None:
         self.save_all_namespaces()
 
     def namespace_db_path(self, namespace: str) -> Path:
-        return self.namespace_path / f"{namespace}.db"
+        return self.namespace_path / f"{namespace}"
 
     async def add_namespace(
         self,
@@ -159,7 +160,7 @@ async def add_persistent_namespace(self, namespace: str, writeback: str) -> Path
             self.state[namespace] = utils.PersistentDict(ns_db_path, safe)
         except Exception as exc:
             raise ade.PersistentNamespaceFailed(namespace, ns_db_path) from exc
-        current_thread = threading.current_thread().getName()
+        current_thread = threading.current_thread().name
         self.logger.info(f"Persistent namespace '{namespace}' initialized from {current_thread}")
         return ns_db_path
 
@@ -482,10 +483,8 @@ async def process_state_callbacks(self, namespace, state):
 
     def entity_exists(self, namespace: str, entity: str) -> bool:
         match self.state.get(namespace):
-            case dict(ns_state):
-                match ns_state.get(entity):
-                    case dict():
-                        return True
+            case Mapping() as ns_state:
+                return entity in ns_state
         return False
 
     def get_entity(self, namespace: Optional[str] = None, entity_id: Optional[str] = None, name: Optional[str] = None):
@@ -854,6 +853,8 @@ def save_all_namespaces(self) -> None:
                 case utils.PersistentDict() as state:
                     self.logger.debug("Saving namespace: %s", ns)
                     state.sync()
+                    self.logger.debug("Closing namespace: %s", ns)
+                    state.close()
 
     def save_hybrid_namespaces(self) -> None:
         for ns_name, cfg in self.AD.namespaces.items():

docs/HISTORY.md

@@ -4,13 +4,14 @@
 
 **Features**
 
-- Added the ``ws_max_msg_size`` config option to the Hass plugin
+- Added some basic test for persistent namespaces
 
 **Fixes**
 
-- Better error handling for receiving huge websocket messages in the Hass plugin
 - Fix for sunrise and sunset with offsets - contributed by [ekutner](https://github.com/ekutner)
 - Fix for random MQTT disconnects  - contributed by [Xsandor](https://github.com/Xsandor)
+- Fix for persistent namespaces in Python 3.12
+- Better error handling for receiving huge websocket messages in the Hass plugin
 
 **Breaking Changes**
 

tests/conf/apps/apps.yaml

@@ -24,3 +24,7 @@ test_run_daily:
   module: scheduler_test_app
   class: TestSchedulerRunDaily
   time: "00:00:05"
+
+basic_namespace_app:
+  module: namespace_app
+  class: BasicNamespaceTester

tests/conf/apps/namespace_app.py

@@ -0,0 +1,64 @@
+from datetime import timedelta
+from typing import Any
+
+from appdaemon.adapi import ADAPI, Entity
+
+
+class BasicNamespaceTester(ADAPI):
+    handle: str | None
+
+    def initialize(self) -> None:
+        self.set_namespace(self.custom_test_namespace)
+        self.logger.info('Current namespaces: %s', sorted(self.current_namespaces))
+
+        self.show_entities()
+
+        exists = self.test_entity.exists()
+        self.logger.info(f"Entity exists: {exists}")
+        if not exists:
+            self.add_entity("sensor.test", state="initial", attributes={"friendly_name": "Test Sensor"})
+
+        self.show_entities()
+
+        non_existence = "sensor.other_entity"
+        self.logger.info("Setting %s in default namespace", non_existence)
+        self.set_state(non_existence, state="other", attributes={"friendly_name": "Other Entity"})
+
+        self.run_in(self.start_test, self.start_delay)
+        self.test_entity.listen_state(self.handle_state)
+        self.log(f"Initialized {self.name}")
+
+    @property
+    def current_namespaces(self) -> set[str]:
+        return set(self.AD.state.state.keys())
+
+    @property
+    def custom_test_namespace(self) -> str:
+        return self.args.get("custom_namespace", "test_namespace")
+
+    @property
+    def start_delay(self) -> timedelta:
+        return timedelta(seconds=self.args.get("start_delay", 1.0))
+
+    @property
+    def test_entity(self) -> Entity:
+        return self.get_entity("sensor.test", check_existence=False)
+
+    def show_entities(self, *args, **kwargs):
+        ns = self.AD.state.state.get(self.custom_test_namespace, {})
+        entities = sorted(ns.keys())
+        self.log('Test entities: %s', entities)
+        return entities
+
+    def start_test(self, *args, **kwargs: Any) -> None:
+        match kwargs:
+            case {"__thread_id": str(thread_id)}:
+                self.log(f"Change called from thread {thread_id}")
+        self.test_entity.set_state("changed")
+
+    def handle_state(self, entity: str, attribute: str, old: Any, new: Any, **kwargs: Any) -> None:
+        self.log(f"State changed for {entity}: {attribute} = {old} -> {new}")
+        self.log(f"Test val: {self.args.get('test_val')}")
+
+        full_state = self.test_entity.get_state('all')
+        self.log(f"Full state: {full_state}")

tests/conftest.py

@@ -17,21 +17,7 @@
 
 
 @pytest_asyncio.fixture(scope="function")
-async def ad(running_loop: asyncio.BaseEventLoop, ad_cfg: AppDaemonConfig, logging_obj: Logging) -> AsyncGenerator[AppDaemon]:
-    """Pytest fixture that provides a full AppDaemon instance for tests.
-
-    General steps:
-      - Create the top-level AppDaemon object.
-      - Set the log levels of the main logs to DEBUG.
-      - Process the import paths.
-      - Set up the dependency manager with the app directory.
-        - Reads all the config files in the app directory.
-      - Disables apps for the duration of the fixture.
-      - Starts/stops the AppDaemon instance.
-    """
-    # logger.info(f"Passed loop: {hex(id(running_loop))}")
-    assert running_loop == asyncio.get_running_loop(), "The running loop should match the one passed in"
-
+async def ad_obj(running_loop: asyncio.BaseEventLoop, ad_cfg: AppDaemonConfig, logging_obj: Logging) -> AsyncGenerator[AppDaemon]:
     ad = AppDaemon(
         logging=logging_obj,
         loop=running_loop,
@@ -45,6 +31,26 @@ async def ad(running_loop: asyncio.BaseEventLoop, ad_cfg: AppDaemonConfig, loggi
         logger_.setLevel("DEBUG")
 
     await ad.app_management._process_import_paths()
+    ad.app_management.dependency_manager = DependencyManager(python_files=list(), config_files=list())
+    yield ad
+
+
+@pytest_asyncio.fixture(scope="function")
+async def ad(ad_obj: AppDaemon, running_loop: asyncio.BaseEventLoop) -> AsyncGenerator[AppDaemon]:
+    """Pytest fixture that provides a full AppDaemon instance for tests.
+
+    General steps:
+      - Create the top-level AppDaemon object.
+      - Set the log levels of the main logs to DEBUG.
+      - Process the import paths.
+      - Set up the dependency manager with the app directory.
+        - Reads all the config files in the app directory.
+      - Disables apps for the duration of the fixture.
+      - Starts/stops the AppDaemon instance.
+    """
+    # logger.info(f"Passed loop: {hex(id(running_loop))}")
+    assert running_loop == asyncio.get_running_loop(), "The running loop should match the one passed in"
+    ad = ad_obj
     config_files = list(recursive_get_files(base=ad.app_dir, suffix=ad.config.ext))
     ad.app_management.dependency_manager = DependencyManager(python_files=list(), config_files=config_files)
 
@@ -89,7 +95,7 @@ def ad_cfg() -> AppDaemonConfig:
                 # "_scheduler": "DEBUG",
                 "_utility": "DEBUG",
             },
-            namespaces={"test": {}},
+            # namespaces={"test_namespace": {"writeback": "hybrid", "persist": False}},
         )
     )