Filter low outliers on battery change (#3082)

* WIP

* Add outlier filter and config flow

* MyPy

* Work on outlier filtering

* Fix battery low state for outliers

* Remove unnecessary log

* Tidy up unused code

* Lint

* Outlier defaults

* Wording

* Change to 80% outlier

* Only filter outliers if lower

* Rename outlier

* Fix low outlier filter

* Return earlier

* Work on increased outlier

Author: andrew-codechimp

custom_components/battery_notes/config_flow.py

@@ -33,6 +33,7 @@
     CONF_BATTERY_QUANTITY,
     CONF_BATTERY_TYPE,
     CONF_DEVICE_NAME,
+    CONF_FILTER_OUTLIERS,
     CONF_MANUFACTURER,
     CONF_MODEL,
     CONF_MODEL_ID,
@@ -356,6 +357,7 @@ async def async_step_battery(
             self.data[CONF_BATTERY_LOW_TEMPLATE] = user_input.get(
                 CONF_BATTERY_LOW_TEMPLATE, None
             )
+            self.data[CONF_FILTER_OUTLIERS] = user_input.get(CONF_FILTER_OUTLIERS, False)
 
             source_entity_id = self.data.get(CONF_SOURCE_ENTITY_ID, None)
             device_id = self.data.get(CONF_DEVICE_ID, None)
@@ -433,6 +435,9 @@ async def async_step_battery(
                     vol.Optional(
                         CONF_BATTERY_LOW_TEMPLATE
                     ): selector.TemplateSelector(),
+                    vol.Optional(
+                        CONF_FILTER_OUTLIERS,
+                        default=False): selector.BooleanSelector(),
                 }
             ),
             errors=errors,
@@ -452,6 +457,7 @@ def __init__(self) -> None:
         self.battery_type: str
         self.battery_quantity: int
         self.battery_low_template: str
+        self.filter_outliers: bool
 
     async def async_step_init(
         self,
@@ -467,6 +473,9 @@ async def async_step_init(
         self.battery_low_template = str(
             self.current_config.get(CONF_BATTERY_LOW_TEMPLATE) or ""
         )
+        self.filter_outliers = bool(
+            self.current_config.get(CONF_FILTER_OUTLIERS) or False
+        )
 
         if self.source_device_id:
             device_registry = dr.async_get(self.hass)
@@ -595,6 +604,7 @@ def build_options_schema(self) -> vol.Schema:
                     ),
                 ),
                 vol.Optional(CONF_BATTERY_LOW_TEMPLATE): selector.TemplateSelector(),
+                vol.Optional(CONF_FILTER_OUTLIERS): selector.BooleanSelector(),
             }
         )
 

custom_components/battery_notes/const.py

@@ -51,6 +51,7 @@
 CONF_HIDE_BATTERY = "hide_battery"
 CONF_ROUND_BATTERY = "round_battery"
 CONF_BATTERY_LOW_TEMPLATE = "battery_low_template"
+CONF_FILTER_OUTLIERS = "filter_outliers"
 
 DATA_CONFIGURED_ENTITIES = "configured_entities"
 DATA_DISCOVERED_ENTITIES = "discovered_entities"
@@ -91,6 +92,9 @@
 ATTR_PREVIOUS_BATTERY_LEVEL = "previous_battery_level"
 ATTR_BATTERY_THRESHOLD_REMINDER = "reminder"
 
+WINDOW_SIZE_UNIT_NUMBER_EVENTS = 1
+WINDOW_SIZE_UNIT_TIME = 2
+
 SERVICE_BATTERY_REPLACED_SCHEMA = vol.Schema(
     {
         vol.Optional(ATTR_DEVICE_ID): cv.string,

custom_components/battery_notes/coordinator.py

@@ -41,6 +41,7 @@
     LAST_REPORTED,
     LAST_REPORTED_LEVEL,
 )
+from .filters import LowOutlierFilter
 from .store import BatteryNotesStorage
 
 _LOGGER = logging.getLogger(__name__)
@@ -68,13 +69,15 @@ class BatteryNotesCoordinator(DataUpdateCoordinator[None]):
     _battery_low_binary_state: bool = False
     _previous_battery_low_binary_state: bool | None = None
     _source_entity_name: str | None = None
+    _outlier_filter: LowOutlierFilter | None = None
 
     def __init__(
         self,
         hass,
         store: BatteryNotesStorage,
         wrapped_battery: RegistryEntry | None,
         wrapped_battery_low: RegistryEntry | None,
+        filter_outliers: bool,
     ):
         """Initialize."""
         self.store = store
@@ -88,6 +91,10 @@ def __init__(
 
         super().__init__(hass, _LOGGER, name=DOMAIN)
 
+        if filter_outliers:
+            self._outlier_filter = LowOutlierFilter(window_size=3, radius=80)
+            _LOGGER.debug("Outlier filter enabled")
+
     @property
     def source_entity_name(self):
         """Get the current name of the source_entity_id."""
@@ -222,8 +229,21 @@ def current_battery_level(self):
     @current_battery_level.setter
     def current_battery_level(self, value):
         """Set the current battery level and fire events if valid."""
-        self._current_battery_level = value
 
+        if self._outlier_filter:
+            if value not in [STATE_UNAVAILABLE, STATE_UNKNOWN]:
+                self._outlier_filter.filter_state(float(value))
+
+                _LOGGER.debug(
+                    "Checking outlier (%s=%s) -> %s",
+                    self.device_id or self.source_entity_id or "",
+                    value,
+                    "skip" if self._outlier_filter.skip_processing else self._outlier_filter.filter_state(value),
+                )
+                if self._outlier_filter.skip_processing:
+                    return
+
+        self._current_battery_level = value
         if (
             self._previous_battery_level is not None
             and self.battery_low_template is None

custom_components/battery_notes/device.py

@@ -25,6 +25,7 @@
     CONF_BATTERY_QUANTITY,
     CONF_BATTERY_TYPE,
     CONF_DEFAULT_BATTERY_LOW_THRESHOLD,
+    CONF_FILTER_OUTLIERS,
     CONF_SOURCE_ENTITY_ID,
     DATA,
     DATA_STORE,
@@ -214,7 +215,8 @@ async def async_setup(self) -> bool:
 
         self.store = self.hass.data[DOMAIN][DATA_STORE]
         self.coordinator = BatteryNotesCoordinator(
-            self.hass, self.store, self.wrapped_battery, self.wrapped_battery_low
+            self.hass, self.store, self.wrapped_battery, self.wrapped_battery_low,
+            cast(bool, self.config.data.get(CONF_FILTER_OUTLIERS, False))
         )
 
         assert(self.device_name)

custom_components/battery_notes/filters.py

@@ -0,0 +1,146 @@
+"""Filters for battery_notes."""
+
+import logging
+import statistics
+from collections import Counter, deque
+from datetime import datetime, timedelta
+from numbers import Number
+from typing import cast
+
+from .const import WINDOW_SIZE_UNIT_NUMBER_EVENTS, WINDOW_SIZE_UNIT_TIME
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class FilterState:
+    """State abstraction for filter usage."""
+
+    state: str | float | int
+
+    def __init__(self, state: str | float | int) -> None:
+        """Initialize with HA State object."""
+        self.timestamp = datetime.utcnow()
+        try:
+            self.state = float(state)
+        except ValueError:
+            self.state = state
+
+    def __str__(self) -> str:
+        """Return state as the string representation of FilterState."""
+        return str(self.state)
+
+    def __repr__(self) -> str:
+        """Return timestamp and state as the representation of FilterState."""
+        return f"{self.timestamp} : {self.state}"
+
+
+class Filter():
+    """Base filter class."""
+
+    def __init__(
+        self,
+        window_size: int | timedelta,
+    ) -> None:
+        """Initialize common attributes.
+
+        :param window_size: size of the sliding window that holds previous values
+        :param entity: used for debugging only
+        """
+        if isinstance(window_size, int):
+            self.states: deque[FilterState] = deque(maxlen=window_size)
+            self.window_unit = WINDOW_SIZE_UNIT_NUMBER_EVENTS
+        else:
+            self.states = deque(maxlen=0)
+            self.window_unit = WINDOW_SIZE_UNIT_TIME
+        self._skip_processing = False
+        self._window_size = window_size
+        self._store_raw = False
+        self._only_numbers = True
+
+    @property
+    def window_size(self) -> int | timedelta:
+        """Return window size."""
+        return self._window_size
+
+    @property
+    def skip_processing(self) -> bool:
+        """Return whether the current filter_state should be skipped."""
+        return self._skip_processing
+
+    def reset(self) -> None:
+        """Reset filter."""
+        self.states.clear()
+
+    def _filter_state(self, new_state: FilterState) -> FilterState:
+        """Implement filter."""
+        raise NotImplementedError
+
+    def filter_state(self, new_state: int | float | str) -> int | float | str:
+        """Implement a common interface for filters."""
+        fstate = FilterState(new_state)
+        if not isinstance(fstate.state, Number):
+            raise ValueError(f"State <{fstate.state}> is not a Number")
+
+        filtered = self._filter_state(fstate)
+
+        if self._store_raw:
+            self.states.append(FilterState(new_state))
+        else:
+            self.states.append(filtered)
+        new_state = filtered.state
+        return new_state
+
+class LowOutlierFilter(Filter):
+    """Low Outlier filter.
+
+    Determines if new state is in a band around the median, or higher.
+    """
+
+    def __init__(
+        self,
+        window_size: int,
+        radius: float,
+    ) -> None:
+        """Initialize Filter.
+
+        :param radius: band radius
+        """
+        super().__init__(
+            window_size
+        )
+        self._radius = radius
+        self._stats_internal: Counter = Counter()
+        self._store_raw = True
+
+    def _filter_state(self, new_state: FilterState) -> FilterState:
+        """Implement the outlier filter."""
+
+        previous_state_values = [cast(float, s.state) for s in self.states]
+        new_state_value = cast(float, new_state.state)
+        self._skip_processing = False
+
+        if previous_state_values and new_state_value >= previous_state_values[-1]:
+            _LOGGER.debug(
+                    "New value higher than last previous state, allowing. %s >= %s",
+                    new_state,
+                    previous_state_values[-1]
+                )
+            return new_state
+
+        median = statistics.median(previous_state_values) if self.states else 0
+
+        if (
+            len(self.states) == self.states.maxlen
+            and abs(new_state_value - median) > self._radius
+        ):
+            self._skip_processing = True
+
+            if len(self.states) == self.states.maxlen:
+                self._stats_internal["erasures"] += 1
+                _LOGGER.debug(
+                    "Outlier nr. %s: %s",
+                    self._stats_internal["erasures"],
+                    new_state,
+                )
+
+        return new_state

custom_components/battery_notes/translations/en.json

@@ -36,11 +36,13 @@
                     "battery_type": "Battery type",
                     "battery_quantity": "Battery quantity",
                     "battery_low_threshold": "Battery low threshold",
-                    "battery_low_template": "Battery low template"
+                    "battery_low_template": "Battery low template",
+                    "filter_outliers": "Filter outliers"
                 },
                 "data_description": {
                     "battery_low_threshold": "0 will use the global default threshold",
-                    "battery_low_template": "Template to determine a battery is low, should return true if low\nOnly needed for non-standard battery levels"
+                    "battery_low_template": "Template to determine a battery is low, should return true if low\nOnly needed for non-standard battery levels",
+                    "filter_outliers": "Filter out large battery level changes, reducing falsely firing events on devices that erronously report levels occasionally"
                 }
             },
             "manual": {
@@ -65,12 +67,14 @@
                     "battery_type": "Battery type",
                     "battery_quantity": "Battery quantity",
                     "battery_low_threshold": "Battery low threshold",
-                    "battery_low_template": "Battery low template"
+                    "battery_low_template": "Battery low template",
+                    "filter_outliers": "Filter outliers"
                 },
                 "data_description": {
                     "name": "Leaving blank will take the name from the source device",
                     "battery_low_threshold": "0 will use the global default threshold",
-                    "battery_low_template": "Template to determine a battery is low, should return true if low\nOnly needed for non-standard battery levels"
+                    "battery_low_template": "Template to determine a battery is low, should return true if low\nOnly needed for non-standard battery levels",
+                    "filter_outliers": "Filter out large battery level changes, reducing falsely firing events on devices that erronously report levels occasionally"
                 }
             }
         },