Merge pull request #69 from meshcore-dev/2.1.7

2.1.7 - Additional improvements to request tracking and retries

Author: awolden

custom_components/meshcore/config_flow.py

@@ -34,6 +34,7 @@
     CONF_REPEATER_UPDATE_INTERVAL,
     CONF_REPEATER_TELEMETRY_ENABLED,
     DEFAULT_REPEATER_UPDATE_INTERVAL,
+    MIN_UPDATE_INTERVAL,
     CONF_TRACKED_CLIENTS,
     CONF_CLIENT_NAME,
     CONF_CLIENT_UPDATE_INTERVAL,
@@ -449,7 +450,7 @@ def _show_add_repeater_form(self, repeater_dict, errors=None, user_input=None):
                 vol.Required(CONF_REPEATER_NAME): vol.In(repeater_dict.keys()),
                 vol.Optional(CONF_REPEATER_PASSWORD, default=default_password): str,
                 vol.Optional(CONF_REPEATER_TELEMETRY_ENABLED, default=default_telemetry): bool,
-                vol.Optional(CONF_REPEATER_UPDATE_INTERVAL, default=default_interval): vol.All(cv.positive_int, vol.Range(min=300, max=3600)),
+                vol.Optional(CONF_REPEATER_UPDATE_INTERVAL, default=default_interval): vol.All(cv.positive_int, vol.Range(min=MIN_UPDATE_INTERVAL)),
             }),
             errors=errors,
         )
@@ -600,7 +601,7 @@ async def async_step_add_client(self, user_input=None):
                 step_id="add_client",
                 data_schema=vol.Schema({
                     vol.Required(CONF_CLIENT_NAME): vol.In(client_dict.keys()),
-                    vol.Optional(CONF_CLIENT_UPDATE_INTERVAL, default=DEFAULT_CLIENT_UPDATE_INTERVAL): vol.All(cv.positive_int, vol.Range(min=600, max=7200)),
+                    vol.Optional(CONF_CLIENT_UPDATE_INTERVAL, default=DEFAULT_CLIENT_UPDATE_INTERVAL): vol.All(cv.positive_int, vol.Range(min=MIN_UPDATE_INTERVAL)),
                 }),
                 errors=errors,
             )
@@ -622,7 +623,7 @@ async def async_step_add_client(self, user_input=None):
                 step_id="add_client",
                 data_schema=vol.Schema({
                     vol.Required(CONF_CLIENT_NAME): vol.In(client_dict.keys()),
-                    vol.Optional(CONF_CLIENT_UPDATE_INTERVAL, default=DEFAULT_CLIENT_UPDATE_INTERVAL): vol.All(cv.positive_int, vol.Range(min=600, max=7200)),
+                    vol.Optional(CONF_CLIENT_UPDATE_INTERVAL, default=DEFAULT_CLIENT_UPDATE_INTERVAL): vol.All(cv.positive_int, vol.Range(min=MIN_UPDATE_INTERVAL)),
                 }),
                 errors=errors,
             )
@@ -816,7 +817,7 @@ async def async_step_edit_repeater(self, user_input=None):
             data_schema=vol.Schema({
                 vol.Optional(CONF_REPEATER_PASSWORD, default=repeater.get("password", "")): str,
                 vol.Optional(CONF_REPEATER_TELEMETRY_ENABLED, default=repeater.get("telemetry_enabled", False)): bool,
-                vol.Optional(CONF_REPEATER_UPDATE_INTERVAL, default=repeater.get("update_interval", DEFAULT_REPEATER_UPDATE_INTERVAL)): vol.All(cv.positive_int, vol.Range(min=300, max=3600)),
+                vol.Optional(CONF_REPEATER_UPDATE_INTERVAL, default=repeater.get("update_interval", DEFAULT_REPEATER_UPDATE_INTERVAL)): vol.All(cv.positive_int, vol.Range(min=MIN_UPDATE_INTERVAL)),
             }),
             description_placeholders={
                 "device_name": repeater.get("name", "Unknown")
@@ -852,7 +853,7 @@ async def async_step_edit_client(self, user_input=None):
         return self.async_show_form(
             step_id="edit_client",
             data_schema=vol.Schema({
-                vol.Optional(CONF_CLIENT_UPDATE_INTERVAL, default=client.get("update_interval", DEFAULT_CLIENT_UPDATE_INTERVAL)): vol.All(cv.positive_int, vol.Range(min=600, max=7200)),
+                vol.Optional(CONF_CLIENT_UPDATE_INTERVAL, default=client.get("update_interval", DEFAULT_CLIENT_UPDATE_INTERVAL)): vol.All(cv.positive_int, vol.Range(min=MIN_UPDATE_INTERVAL)),
             }),
             description_placeholders={
                 "device_name": client.get("name", "Unknown")

custom_components/meshcore/const.py

@@ -56,6 +56,7 @@
 CONF_REPEATER_UPDATE_INTERVAL: Final = "update_interval"
 CONF_REPEATER_TELEMETRY_ENABLED: Final = "telemetry_enabled"
 DEFAULT_REPEATER_UPDATE_INTERVAL: Final = 900  # 15 minutes in seconds
+MIN_UPDATE_INTERVAL: Final = 300  # 5 minutes minimum
 MAX_REPEATER_FAILURES_BEFORE_LOGIN: Final = 5  # After this many failures, try login
 
 # Client tracking constants
@@ -76,6 +77,8 @@
 # Backoff constants for repeater failures
 REPEATER_BACKOFF_BASE: Final = 2  # Base multiplier for exponential backoff
 REPEATER_BACKOFF_MAX_MULTIPLIER: Final = 120  # Maximum backoff multiplier (10 minutes when * 5 seconds)
+MAX_FAILURES_BEFORE_PATH_RESET: Final = 3  # Reset path after this many failures
+MAX_RETRY_ATTEMPTS: Final = 5  # Maximum retry attempts within refresh window
 
 
 # Generic battery voltage to percentage lookup table

custom_components/meshcore/coordinator.py

@@ -30,6 +30,8 @@
     MAX_REPEATER_FAILURES_BEFORE_LOGIN,
     REPEATER_BACKOFF_BASE,
     REPEATER_BACKOFF_MAX_MULTIPLIER,
+    MAX_FAILURES_BEFORE_PATH_RESET,
+    MAX_RETRY_ATTEMPTS,
     CONF_CONTACT_REFRESH_INTERVAL,
     DEFAULT_CONTACT_REFRESH_INTERVAL,
     CONF_REPEATER_TELEMETRY_ENABLED,
@@ -128,6 +130,34 @@ def __init__(
 
         if not hasattr(self, "last_update_success_time"):
             self.last_update_success_time = self._current_time()
+            
+        # Initialize reliability stats tracking
+        self._reliability_stats = {}
+    
+    def _increment_success(self, pubkey_prefix: str) -> None:
+        """Increment success counter for a node."""
+        stats_key = f"{pubkey_prefix}_request_successes"
+        self._reliability_stats[stats_key] = self._reliability_stats.get(stats_key, 0) + 1
+        
+    def _increment_failure(self, pubkey_prefix: str) -> None:
+        """Increment failure counter for a node."""
+        stats_key = f"{pubkey_prefix}_request_failures"
+        self._reliability_stats[stats_key] = self._reliability_stats.get(stats_key, 0) + 1
+    
+    async def _reset_node_path(self, contact, node_name: str) -> bool:
+        """Reset routing path for a node and return success status."""
+        try:
+            result = await self.api.mesh_core.commands.reset_path(contact)
+            if result and result.type != EventType.ERROR:
+                self.logger.info(f"Successfully reset path for {node_name}")
+                return True
+            else:
+                error_msg = result.payload if result and result.type == EventType.ERROR else "no response or unexpected result"
+                self.logger.warning(f"Failed to reset path for {node_name}: {error_msg}")
+                return False
+        except Exception as ex:
+            self.logger.warning(f"Exception resetting path for {node_name}: {ex}")
+            return False
 
     def update_telemetry_settings(self, config_entry: ConfigEntry) -> None:
         """Update telemetry settings from config entry."""
@@ -190,15 +220,19 @@ async def _update_repeater(self, repeater_config):
                         repeater_config.get(CONF_REPEATER_PASSWORD, "")
                     )
 
-                    if login_result.type == EventType.ERROR:
-                        self.logger.error(f"Login to repeater {repeater_name} failed: {login_result.payload}")
-                    else:
+                    if login_result and login_result.type != None and login_result.type != EventType.ERROR:
                         self.logger.info(f"Successfully logged in to repeater {repeater_name}")
+                        self._increment_success(pubkey_prefix)
                         # Track login time for telemetry refresh
                         self._repeater_login_times[pubkey_prefix] = self._current_time()
+                    else:
+                        error_msg = login_result.payload if login_result and login_result.type == EventType.ERROR else "timeout or no response"
+                        self.logger.error(f"Login to repeater {repeater_name} failed: {error_msg}")
+                        self._increment_failure(pubkey_prefix)
 
                 except Exception as ex:
                     self.logger.error(f"Exception during login to repeater {repeater_name}: {ex}")
+                    self._increment_failure(pubkey_prefix)
 
                 # Reset failures after login attempt regardless of outcome
                 # This prevents repeated login attempts if they keep failing
@@ -219,27 +253,26 @@ async def _update_repeater(self, repeater_config):
                 # Increment failure count and apply backoff
                 new_failure_count = failure_count + 1
                 self._repeater_consecutive_failures[pubkey_prefix] = new_failure_count
+                self._increment_failure(pubkey_prefix)
 
-                # Reset path after 5 failures if there's an established path
-                if new_failure_count == 5 and contact and contact.get("out_path_len", 0) != -1:
-                    try:
-                        await self.api.mesh_core.commands.reset_path(pubkey_prefix)
-                        self.logger.info(f"Reset path for repeater {repeater_name} after 5 failures")
-                    except Exception as ex:
-                        self.logger.warning(f"Failed to reset path for repeater {repeater_name}: {ex}")
+                # Reset path after configured failures if there's an established path
+                if new_failure_count == MAX_FAILURES_BEFORE_PATH_RESET and contact and contact.get("out_path_len", -1) >= 0:
+                    await self._reset_node_path(contact, repeater_name)
 
                 update_interval = repeater_config.get(CONF_REPEATER_UPDATE_INTERVAL, DEFAULT_REPEATER_UPDATE_INTERVAL)
                 self._apply_repeater_backoff(pubkey_prefix, new_failure_count, update_interval)
             elif result.payload.get('uptime', 0) == 0:
                 self.logger.warn(f"Malformed status response from repeater {repeater_name}: {result.payload}")
                 new_failure_count = failure_count + 1
                 self._repeater_consecutive_failures[pubkey_prefix] = new_failure_count
+                self._increment_failure(pubkey_prefix)
                 update_interval = repeater_config.get(CONF_REPEATER_UPDATE_INTERVAL, DEFAULT_REPEATER_UPDATE_INTERVAL)
                 self._apply_repeater_backoff(pubkey_prefix, new_failure_count, update_interval)
             else:
                 self.logger.debug(f"Successfully updated repeater {repeater_name}")
                 # Reset failure count on success
                 self._repeater_consecutive_failures[pubkey_prefix] = 0
+                self._increment_success(pubkey_prefix)
 
                 # Trigger state updates for any entities listening for this repeater
                 self.async_set_updated_data(self.data)
@@ -254,6 +287,7 @@ async def _update_repeater(self, repeater_config):
             # Increment failure count and apply backoff
             new_failure_count = self._repeater_consecutive_failures.get(pubkey_prefix, 0) + 1
             self._repeater_consecutive_failures[pubkey_prefix] = new_failure_count
+            self._increment_failure(pubkey_prefix)
             update_interval = repeater_config.get(CONF_REPEATER_UPDATE_INTERVAL, DEFAULT_REPEATER_UPDATE_INTERVAL)
             self._apply_repeater_backoff(pubkey_prefix, new_failure_count, update_interval)
         finally:
@@ -264,13 +298,21 @@ async def _update_repeater(self, repeater_config):
     def _apply_backoff(self, pubkey_prefix: str, failure_count: int, update_interval: int, update_type: str = "repeater") -> None:
         """Apply exponential backoff delay for failed updates.
         
+        Uses dynamic base interval to ensure max 5 retries within the refresh window.
+        Resets failure count after MAX_RETRY_ATTEMPTS to start fresh.
+        
         Args:
             pubkey_prefix: The node's public key prefix
             failure_count: Number of consecutive failures
             update_interval: The configured update interval to cap the backoff at
             update_type: Type of update ("repeater" or "telemetry")
         """
-        backoff_delay = min(REPEATER_BACKOFF_BASE ** failure_count, update_interval)
+        # Calculate base interval to fit 5 retries within refresh window
+        # Sum of geometric series: base * (2^5 - 1) / (2 - 1) = base * 31
+        # We want this to be roughly half the refresh interval for safety
+        base_interval = max(1, update_interval // (31 * 2))
+        
+        backoff_delay = min(base_interval * (REPEATER_BACKOFF_BASE ** failure_count), update_interval)
         next_update_time = self._current_time() + backoff_delay
 
         if update_type == "telemetry":
@@ -280,6 +322,7 @@ def _apply_backoff(self, pubkey_prefix: str, failure_count: int, update_interval
 
         self.logger.debug(f"Applied backoff for {update_type} {pubkey_prefix}: "
                          f"failure_count={failure_count}, "
+                         f"base_interval={base_interval}s, "
                          f"delay={backoff_delay}s, "
                          f"interval_cap={update_interval}s")
 
@@ -313,6 +356,7 @@ async def _update_node_telemetry(self, contact, pubkey_prefix: str, node_name: s
                 self.logger.debug(f"Telemetry response received from {node_name}: {telemetry_result}")
                 # Reset failure count on success
                 self._telemetry_consecutive_failures[pubkey_prefix] = 0
+                self._increment_success(pubkey_prefix)
                 # Schedule next telemetry update
                 next_telemetry_time = self._current_time() + update_interval
                 self._next_telemetry_update_times[pubkey_prefix] = next_telemetry_time
@@ -321,14 +365,11 @@ async def _update_node_telemetry(self, contact, pubkey_prefix: str, node_name: s
                 # Increment failure count and apply backoff
                 new_failure_count = failure_count + 1
                 self._telemetry_consecutive_failures[pubkey_prefix] = new_failure_count
+                self._increment_failure(pubkey_prefix)
 
-                # Reset path after 5 failures if there's an established path
-                if new_failure_count == 5 and contact and contact.get("out_path_len", 0) != -1:
-                    try:
-                        await self.api.mesh_core.commands.reset_path(pubkey_prefix)
-                        self.logger.info(f"Reset path for node {node_name} after 5 telemetry failures")
-                    except Exception as ex:
-                        self.logger.warning(f"Failed to reset path for node {node_name}: {ex}")
+                # Reset path after configured failures if there's an established path
+                if new_failure_count == MAX_FAILURES_BEFORE_PATH_RESET and contact and contact.get("out_path_len", -1) >= 0:
+                    await self._reset_node_path(contact, node_name)
 
                 self._apply_backoff(pubkey_prefix, new_failure_count, update_interval, "telemetry")
 
@@ -337,12 +378,13 @@ async def _update_node_telemetry(self, contact, pubkey_prefix: str, node_name: s
             # Increment failure count and apply backoff
             new_failure_count = failure_count + 1
             self._telemetry_consecutive_failures[pubkey_prefix] = new_failure_count
+            self._increment_failure(pubkey_prefix)
 
-            # Reset path after 5 failures if there's an established path
-            if new_failure_count == 5 and contact and contact.get("out_path_len", 0) != -1:
+            # Reset path after configured failures if there's an established path
+            if new_failure_count == MAX_FAILURES_BEFORE_PATH_RESET and contact and contact.get("out_path_len", -1) != -1:
                 try:
                     await self.api.mesh_core.commands.reset_path(pubkey_prefix)
-                    self.logger.info(f"Reset path for node {node_name} after 5 telemetry failures")
+                    self.logger.info(f"Reset path for node {node_name} after {MAX_FAILURES_BEFORE_PATH_RESET} telemetry failures")
                 except Exception as reset_ex:
                     self.logger.warning(f"Failed to reset path for node {node_name}: {reset_ex}")
 
@@ -479,7 +521,7 @@ async def _async_update_data(self) -> None:
             pubkey_prefix = repeater_config.get("pubkey_prefix")
             repeater_name = repeater_config.get("name")
 
-            # Clean up completed or failed tasks
+            # Clean c completed or failed tasks
             if pubkey_prefix in self._active_repeater_tasks:
                 task = self._active_repeater_tasks[pubkey_prefix]
                 if task.done():