Add HA encryption-key handoff endpoint and adoption key reuse

Author: bdraco

docs/API.md

@@ -528,5 +528,6 @@ For Home Assistant ESPHome integration backward compat only.
 | `GET /devices` | List devices. `configured` entries keep the flat legacy shape — `runtime_state` is flattened to top-level keys (`deployed_version` etc.), which HA's `esphome-dashboard-api` `ConfiguredDevice` reads |
 | `GET /ping` | Online-status map `{<config>.yaml: true\|false\|null}` (third-party widgets, e.g. homepage) |
 | `GET /json-config?configuration=...` | Fully-resolved config (substitutions/packages/includes/secrets) as JSON, via `esphome config`; 404 missing, 422 invalid, 503 unavailable, 403 traversal |
+| `POST /encryption-key` | HA-provisioned Noise API key handoff, body `{device_name, key, mac?}`. Splices the key into a configured device's YAML (`{"result": "updated"\|"unchanged"\|"not_writable", configurations, reason?}`) or stashes it for adoption (`{"result": "stored"}`); 400 malformed body or non-32-byte-base64 key. Only accepted on the trusted peer-guarded ingress site — the public site answers 403 |
 | `GET /compile` (WebSocket) | Compile via spawn protocol |
 | `GET /upload` (WebSocket) | Upload via spawn protocol |

esphome_device_builder/api/legacy.py

@@ -7,6 +7,7 @@
 HA uses:
 - GET /devices (list configured + importable devices)
 - GET /json-config?configuration=... (parsed YAML as JSON)
+- POST /encryption-key (HA-provisioned API key handoff; ingress site only)
 - /compile (WebSocket, spawn protocol)
 - /upload (WebSocket, spawn protocol)
 
@@ -47,6 +48,7 @@
     TERMINAL_JOB_STATUSES,
     Device,
     DeviceState,
+    ErrorCode,
     EventType,
     FirmwareJob,
     JobType,
@@ -337,6 +339,52 @@ async def _json_config_response(db: DeviceBuilder, configuration: str) -> web.Re
     return json_response(config)
 
 
+def _parse_encryption_key_payload(raw: bytes) -> tuple[str, str, str] | str:
+    """Parse the POST /encryption-key body; ``(device_name, key, mac)`` or an error message."""
+    try:
+        body = loads(raw)
+    except JSONDecodeError:
+        return "invalid JSON body"
+    if not isinstance(body, dict):
+        return "invalid JSON body"
+    device_name = body.get("device_name")
+    key = body.get("key")
+    mac = body.get("mac", "")
+    if not isinstance(device_name, str) or not device_name:
+        return "device_name is required"
+    if not isinstance(key, str) or not key:
+        return "key is required"
+    if not isinstance(mac, str):
+        return "mac must be a string"
+    return device_name, key, mac
+
+
+async def _encryption_key_response(request: web.Request) -> web.Response:
+    """
+    Land an HA-provisioned key via the devices controller.
+
+    Only accepted on the trusted, peer-guarded ingress site: the key is
+    secret material and the supervisor-authenticated ingress path is the
+    one channel where the caller is known to be Home Assistant.
+    """
+    if not (request.app.get("trusted_site") and request.app.get("peer_guarded")):
+        return json_response(
+            {"error": "encryption-key is only accepted over the Home Assistant ingress"},
+            status=403,
+        )
+    parsed = _parse_encryption_key_payload(await request.read())
+    if isinstance(parsed, str):
+        return json_response({"error": parsed}, status=400)
+    device_name, key, mac = parsed
+    db = request.app["device_builder"]
+    try:
+        result = await db.devices.set_encryption_key(name=device_name, key=key, mac=mac)
+    except CommandError as err:
+        status = 400 if err.code is ErrorCode.INVALID_ARGS else 500
+        return json_response({"error": err.message}, status=status)
+    return json_response(result)
+
+
 def create_legacy_routes() -> web.RouteTableDef:
     """Create backward-compatible REST + WS routes for HA."""
     routes = web.RouteTableDef()
@@ -396,6 +444,11 @@ async def legacy_json_config(request: web.Request) -> web.Response:
         db = request.app["device_builder"]
         return await _json_config_response(db, request.query.get("configuration", ""))
 
+    @routes.post("/encryption-key")
+    async def legacy_encryption_key(request: web.Request) -> web.Response:
+        """HA-provisioned Noise API key — splice into a configured YAML or stash for adoption."""
+        return await _encryption_key_response(request)
+
     @routes.get("/compile")
     async def legacy_compile(request: web.Request) -> web.WebSocketResponse:
         return await _handle_legacy_ws_command(request, JobType.COMPILE)

esphome_device_builder/controllers/_device_state_monitor/api_reviver.py

@@ -45,6 +45,7 @@
 from typing import TYPE_CHECKING, Any
 
 from ...helpers.cooldown import CooldownLedger
+from ...helpers.mac_addresses import normalize_mac
 from ...models import Device, DeviceState
 from . import shared
 from ._api_probe import (
@@ -53,7 +54,6 @@
     api_worker_available,
     apply_worker_info,
 )
-from .helpers import _normalize_mac
 
 if TYPE_CHECKING:
     from .controller import DeviceStateMonitor
@@ -234,8 +234,8 @@ async def _verify_and_revive(self, device: Device, ip: str, rtt: float) -> None:
             self._record_dial_failure(key)
             monitor.invalidate_persisted_ip(device.name, ip)
             return
-        mac = _normalize_mac(info.get("mac_address", ""))
-        persisted_mac = _normalize_mac(device.mac_address)
+        mac = normalize_mac(info.get("mac_address", ""))
+        persisted_mac = normalize_mac(device.mac_address)
         if mac and persisted_mac and mac != persisted_mac:
             _LOGGER.warning(
                 "Device at %s reports name %s but MAC %s != persisted %s; not claiming ONLINE",

esphome_device_builder/controllers/_device_state_monitor/controller.py

@@ -30,6 +30,7 @@
 
 from ...helpers.async_ import create_eager_task, drain_tasks, log_task_exit
 from ...helpers.ip import drop_unusable_addresses, is_unusable_address
+from ...helpers.mac_addresses import normalize_mac
 from ...helpers.subscriber_presence import SubscriberPresence
 from ...models import (
     RUNTIME_STATE_FIELD_NAMES,
@@ -44,7 +45,6 @@
 from .api_info import ApiInfoSource
 from .api_reviver import ApiReviverSource
 from .helpers import (
-    _normalize_mac,
     _pick_ipv4,
 )
 from .importable import ImportableDiscovery
@@ -96,7 +96,7 @@
 ApiEncryptionChangeCallback = Callable[[str, str], None]
 
 # mDNS ``mac`` TXT change. The value has already been normalised by
-# :func:`_normalize_mac` to ``XX:XX:XX:XX:XX:XX`` so the frontend
+# :func:`normalize_mac` to ``XX:XX:XX:XX:XX:XX`` so the frontend
 # renders it directly. Empty / non-hex skips the callback so older
 # firmware without the broadcast doesn't blank a known MAC.
 MacAddressChangeCallback = Callable[[str, str], None]
@@ -528,15 +528,15 @@ def apply_mac_address(self, name: str, mac: str) -> bool:
         """
         Record a MAC-address observation from the device's mDNS TXT.
 
-        Normalised via :func:`_normalize_mac` so the dedupe /
+        Normalised via :func:`normalize_mac` so the dedupe /
         sidecar / wire all stay canonical regardless of which case
         or separator style the firmware emits. Empty / non-hex
         inputs are dropped so a broadcast that omits the ``mac``
         TXT (older firmware) doesn't blank an already-known value.
         """
         if (forward := self._on_mac_address_change) is None:
             return False
-        normalized = _normalize_mac(mac)
+        normalized = normalize_mac(mac)
         if not normalized:
             return False
         return self._apply_observation(name, "mac_address", normalized, forward, name, normalized)

esphome_device_builder/controllers/_device_state_monitor/helpers.py

@@ -21,11 +21,6 @@
 # get ``web_port`` from the YAML (``web_server:``).
 _HTTP_SERVICE_TYPE = "_http._tcp.local."
 
-# Strip ``-`` (Windows) and ``.`` (Cisco) too so a vendored tool or
-# future firmware can't slip a non-canonical form into the dedupe
-# path or the sidecar.
-_MAC_SEPARATORS = str.maketrans("", "", ":-.")
-
 
 def device_name_from_service(service_name: str) -> str:
     """
@@ -38,24 +33,6 @@ def device_name_from_service(service_name: str) -> str:
     return service_name.split(".", maxsplit=1)[0]
 
 
-def _normalize_mac(value: str) -> str:
-    """
-    Canonicalise a broadcast MAC to ``XX:XX:XX:XX:XX:XX`` form.
-
-    Returns ``""`` when the input doesn't shape into a 48-bit hex
-    MAC — callers treat that the same as "TXT absent" and skip
-    the apply path.
-    """
-    stripped = value.translate(_MAC_SEPARATORS).upper()
-    if len(stripped) != 12:
-        return ""
-    try:
-        int(stripped, 16)
-    except ValueError:
-        return ""
-    return ":".join(stripped[i : i + 2] for i in range(0, 12, 2))
-
-
 def _http_url_from_service_info(device_name: str, info: AsyncServiceInfo) -> str:
     """
     Build ``http://<host>[:port]`` from a populated HTTP service info.

esphome_device_builder/controllers/devices/_pending_keys_store.py

@@ -0,0 +1,79 @@
+"""Name-keyed store of HA-provisioned API keys awaiting adoption."""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from ...helpers.json import JSONDecodeError, dumps_indent, loads
+from ...helpers.storage import Store
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from ...helpers.storage import ShutdownRegister
+
+_LOGGER = logging.getLogger(__name__)
+
+_STORE_FILENAME = ".device-builder-pending-keys.json"
+_SAVE_DELAY = 1.0
+
+
+def _encode(data: dict[str, dict[str, str]]) -> bytes:
+    return dumps_indent(data)
+
+
+def _decode(raw: bytes) -> dict[str, dict[str, str]]:
+    try:
+        obj = loads(raw)
+    except JSONDecodeError:
+        _LOGGER.warning("pending keys store: corrupt JSON, starting empty")
+        return {}
+    if not isinstance(obj, dict):
+        return {}
+    return {k: v for k, v in obj.items() if isinstance(k, str) and isinstance(v, dict)}
+
+
+class PendingKeysStore:
+    """RAM-canonical pending keys; writes go through a debounced ``Store``."""
+
+    def __init__(self, data_dir: Path, shutdown_register: ShutdownRegister) -> None:
+        self._state: dict[str, dict[str, str]] = {}
+        self._store: Store[dict[str, dict[str, str]]] = Store(
+            data_dir / _STORE_FILENAME,
+            encoder=_encode,
+            decoder=_decode,
+            shutdown_register=shutdown_register,
+            name="pending_keys",
+        )
+
+    async def async_load(self) -> None:
+        """Seed RAM from disk."""
+        loaded = await self._store.async_load()
+        if loaded is not None:
+            self._state = loaded
+
+    def get(self, name: str) -> dict[str, str] | None:
+        """Return a copy of *name*'s pending entry, or ``None``."""
+        entry = self._state.get(name)
+        return dict(entry) if entry is not None else None
+
+    def set(self, name: str, key: str, mac: str = "") -> None:
+        """Store or overwrite the pending key for *name*."""
+        entry = {"key": key}
+        if mac:
+            entry["mac"] = mac
+        if self._state.get(name) == entry:
+            return
+        self._state[name] = entry
+        self._store.async_delay_save(self._snapshot, delay=_SAVE_DELAY)
+
+    def pop(self, name: str) -> dict[str, str] | None:
+        """Drop and return *name*'s pending entry, or ``None``."""
+        entry = self._state.pop(name, None)
+        if entry is not None:
+            self._store.async_delay_save(self._snapshot, delay=_SAVE_DELAY)
+        return entry
+
+    def _snapshot(self) -> dict[str, dict[str, str]]:
+        return dict(self._state)

esphome_device_builder/controllers/devices/controller.py

@@ -62,6 +62,7 @@
     api_key,
     archive,
     backtrace,
+    encryption_key,
     firmware_sync,
     importable,
     logs,
@@ -79,6 +80,7 @@
     validate,
 )
 from ._metadata_store import DeviceMetadataStore
+from ._pending_keys_store import PendingKeysStore
 from ._shared_sidecar import SharedSidecarClient
 from ._state import DevicesState
 from ._yaml_search_cache import YamlSearchCache
@@ -136,6 +138,12 @@ def __init__(self, device_builder: DeviceBuilder) -> None:
             data_dir=Path(CORE.data_dir),
             shutdown_register=self._shutdown_callbacks.append,
         )
+        # HA-provisioned API keys for devices not yet adopted; consumed
+        # by ``import_device`` so adoption doesn't mint a competing key.
+        self._pending_keys = PendingKeysStore(
+            data_dir=Path(CORE.data_dir),
+            shutdown_register=self._shutdown_callbacks.append,
+        )
         # Resolved here because ``CORE.data_dir`` stats the config dir;
         # the validate path reads it from the loop thread.
         self._packages_root = Path(CORE.data_dir) / "packages"
@@ -265,6 +273,7 @@ async def start(self) -> None:
         # Seed the store (and migrate on first post-upgrade boot)
         # before the scanner runs — resolver reads off it.
         await self._metadata_store.async_load()
+        await self._pending_keys.async_load()
         await self.migrate_board_id_user_set()
         await run_in_executor(self._load_ignored_devices)
         await self._scanner.scan()
@@ -912,6 +921,10 @@ async def toggle_ignore(self, *, name: str, ignore: bool = True, **kwargs: Any)
         """Mark a discovered device as ignored / visible in the import list."""
         await importable.toggle_ignore(self, name=name, ignore=ignore)
 
+    async def set_encryption_key(self, *, name: str, key: str, mac: str = "") -> dict[str, Any]:
+        """Apply an HA-provisioned API encryption key, or stash it for adoption."""
+        return await encryption_key.set_encryption_key(self, name=name, key=key, mac=mac)
+
     # ------------------------------------------------------------------
     # API commands — per-connection streams (validate, logs)
     # ------------------------------------------------------------------