Create addon boot failed issue for repair (#5397)

* Create addon boot failed issue for repair

* MDont make new objects for contains checks

Author: mdegat01

supervisor/addons/addon.py

@@ -81,7 +81,8 @@
 from ..homeassistant.const import WSEvent, WSType
 from ..jobs.const import JobExecutionLimit
 from ..jobs.decorator import Job
-from ..resolution.const import UnhealthyReason
+from ..resolution.const import ContextType, IssueType, UnhealthyReason
+from ..resolution.data import Issue
 from ..store.addon import AddonStore
 from ..utils import check_port
 from ..utils.apparmor import adjust_profile
@@ -144,11 +145,19 @@ def __init__(self, coresys: CoreSys, slug: str):
         self._listeners: list[EventListener] = []
         self._startup_event = asyncio.Event()
         self._startup_task: asyncio.Task | None = None
+        self._boot_failed_issue = Issue(
+            IssueType.BOOT_FAIL, ContextType.ADDON, reference=self.slug
+        )
 
     def __repr__(self) -> str:
         """Return internal representation."""
         return f"<Addon: {self.slug}>"
 
+    @property
+    def boot_failed_issue(self) -> Issue:
+        """Get issue used if start on boot failed."""
+        return self._boot_failed_issue
+
     @property
     def state(self) -> AddonState:
         """Return state of the add-on."""
@@ -166,6 +175,13 @@ def state(self, new_state: AddonState) -> None:
         if new_state == AddonState.STARTED or old_state == AddonState.STARTUP:
             self._startup_event.set()
 
+        # Dismiss boot failed issue if present and we started
+        if (
+            new_state == AddonState.STARTED
+            and self.boot_failed_issue in self.sys_resolution.issues
+        ):
+            self.sys_resolution.dismiss_issue(self.boot_failed_issue)
+
         self.sys_homeassistant.websocket.send_message(
             {
                 ATTR_TYPE: WSType.SUPERVISOR_EVENT,
@@ -322,6 +338,13 @@ def boot(self, value: AddonBoot) -> None:
         """Store user boot options."""
         self.persist[ATTR_BOOT] = value
 
+        # Dismiss boot failed issue if present and boot at start disabled
+        if (
+            value == AddonBoot.MANUAL
+            and self._boot_failed_issue in self.sys_resolution.issues
+        ):
+            self.sys_resolution.dismiss_issue(self._boot_failed_issue)
+
     @property
     def auto_update(self) -> bool:
         """Return if auto update is enable."""

supervisor/addons/manager.py

@@ -7,24 +7,22 @@
 import tarfile
 from typing import Union
 
+from attr import evolve
+
 from ..const import AddonBoot, AddonStartup, AddonState
 from ..coresys import CoreSys, CoreSysAttributes
 from ..exceptions import (
-    AddonConfigurationError,
     AddonsError,
     AddonsJobError,
     AddonsNotSupportedError,
     CoreDNSError,
-    DockerAPIError,
     DockerError,
-    DockerNotFound,
     HassioError,
     HomeAssistantAPIError,
 )
 from ..jobs.decorator import Job, JobCondition
 from ..resolution.const import ContextType, IssueType, SuggestionType
 from ..store.addon import AddonStore
-from ..utils import check_exception_chain
 from ..utils.sentry import capture_exception
 from .addon import Addon
 from .const import ADDON_UPDATE_CONDITIONS
@@ -118,15 +116,14 @@ async def boot(self, stage: AddonStartup) -> None:
             try:
                 if start_task := await addon.start():
                     wait_boot.append(start_task)
-            except AddonsError as err:
-                # Check if there is an system/user issue
-                if check_exception_chain(
-                    err, (DockerAPIError, DockerNotFound, AddonConfigurationError)
-                ):
-                    addon.boot = AddonBoot.MANUAL
-                    addon.save_persist()
             except HassioError:
-                pass  # These are already handled
+                self.sys_resolution.add_issue(
+                    evolve(addon.boot_failed_issue),
+                    suggestions=[
+                        SuggestionType.EXECUTE_START,
+                        SuggestionType.DISABLE_BOOT,
+                    ],
+                )
             else:
                 continue
 
@@ -135,6 +132,19 @@ async def boot(self, stage: AddonStartup) -> None:
         # Ignore exceptions from waiting for addon startup, addon errors handled elsewhere
         await asyncio.gather(*wait_boot, return_exceptions=True)
 
+        # After waiting for startup, create an issue for boot addons that are error or unknown state
+        # Ignore stopped as single shot addons can be run at boot and this is successful exit
+        # Timeout waiting for startup is not a failure, addon is probably just slow
+        for addon in tasks:
+            if addon.state in {AddonState.ERROR, AddonState.UNKNOWN}:
+                self.sys_resolution.add_issue(
+                    evolve(addon.boot_failed_issue),
+                    suggestions=[
+                        SuggestionType.EXECUTE_START,
+                        SuggestionType.DISABLE_BOOT,
+                    ],
+                )
+
     async def shutdown(self, stage: AddonStartup) -> None:
         """Shutdown addons."""
         tasks: list[Addon] = []

supervisor/mounts/manager.py

@@ -6,6 +6,8 @@
 import logging
 from pathlib import PurePath
 
+from attr import evolve
+
 from ..const import ATTR_NAME
 from ..coresys import CoreSys, CoreSysAttributes
 from ..dbus.const import UnitActiveState
@@ -171,7 +173,7 @@ async def _mount_errors_to_issues(
                 capture_exception(errors[i])
 
             self.sys_resolution.add_issue(
-                mounts[i].failed_issue,
+                evolve(mounts[i].failed_issue),
                 suggestions=[
                     SuggestionType.EXECUTE_RELOAD,
                     SuggestionType.EXECUTE_REMOVE,

supervisor/mounts/mount.py

@@ -68,6 +68,9 @@ def __init__(self, coresys: CoreSys, data: MountData) -> None:
         self._data: MountData = data
         self._unit: SystemdUnit | None = None
         self._state: UnitActiveState | None = None
+        self._failed_issue = Issue(
+            IssueType.MOUNT_FAILED, ContextType.MOUNT, reference=self.name
+        )
 
     @classmethod
     def from_dict(cls, coresys: CoreSys, data: MountData) -> "Mount":
@@ -162,7 +165,7 @@ def local_where(self) -> Path | None:
     @property
     def failed_issue(self) -> Issue:
         """Get issue used if this mount has failed."""
-        return Issue(IssueType.MOUNT_FAILED, ContextType.MOUNT, reference=self.name)
+        return self._failed_issue
 
     async def is_mounted(self) -> bool:
         """Return true if successfully mounted and available."""

supervisor/resolution/const.py

@@ -71,6 +71,7 @@ class UnhealthyReason(StrEnum):
 class IssueType(StrEnum):
     """Issue type."""
 
+    BOOT_FAIL = "boot_fail"
     CORRUPT_DOCKER = "corrupt_docker"
     CORRUPT_REPOSITORY = "corrupt_repository"
     CORRUPT_FILESYSTEM = "corrupt_filesystem"
@@ -103,13 +104,15 @@ class SuggestionType(StrEnum):
     ADOPT_DATA_DISK = "adopt_data_disk"
     CLEAR_FULL_BACKUP = "clear_full_backup"
     CREATE_FULL_BACKUP = "create_full_backup"
+    DISABLE_BOOT = "disable_boot"
     EXECUTE_INTEGRITY = "execute_integrity"
     EXECUTE_REBOOT = "execute_reboot"
     EXECUTE_REBUILD = "execute_rebuild"
     EXECUTE_RELOAD = "execute_reload"
     EXECUTE_REMOVE = "execute_remove"
     EXECUTE_REPAIR = "execute_repair"
     EXECUTE_RESET = "execute_reset"
+    EXECUTE_START = "execute_start"
     EXECUTE_STOP = "execute_stop"
     EXECUTE_UPDATE = "execute_update"
     REGISTRY_LOGIN = "registry_login"

supervisor/resolution/fixups/addon_disable_boot.py

@@ -0,0 +1,48 @@
+"""Helpers to fix addon by disabling boot."""
+
+import logging
+
+from ...const import AddonBoot
+from ...coresys import CoreSys
+from ..const import ContextType, IssueType, SuggestionType
+from .base import FixupBase
+
+_LOGGER: logging.Logger = logging.getLogger(__name__)
+
+
+def setup(coresys: CoreSys) -> FixupBase:
+    """Check setup function."""
+    return FixupAddonDisableBoot(coresys)
+
+
+class FixupAddonDisableBoot(FixupBase):
+    """Storage class for fixup."""
+
+    async def process_fixup(self, reference: str | None = None) -> None:
+        """Initialize the fixup class."""
+        if not (addon := self.sys_addons.get(reference, local_only=True)):
+            _LOGGER.info("Cannot change addon %s as it does not exist", reference)
+            return
+
+        # Disable boot on addon
+        addon.boot = AddonBoot.MANUAL
+
+    @property
+    def suggestion(self) -> SuggestionType:
+        """Return a SuggestionType enum."""
+        return SuggestionType.DISABLE_BOOT
+
+    @property
+    def context(self) -> ContextType:
+        """Return a ContextType enum."""
+        return ContextType.ADDON
+
+    @property
+    def issues(self) -> list[IssueType]:
+        """Return a IssueType enum list."""
+        return [IssueType.BOOT_FAIL]
+
+    @property
+    def auto(self) -> bool:
+        """Return if a fixup can be apply as auto fix."""
+        return False

supervisor/resolution/fixups/addon_execute_start.py

@@ -0,0 +1,59 @@
+"""Helpers to fix addon by starting it."""
+
+import logging
+
+from ...const import AddonState
+from ...coresys import CoreSys
+from ...exceptions import AddonsError, ResolutionFixupError
+from ..const import ContextType, IssueType, SuggestionType
+from .base import FixupBase
+
+_LOGGER: logging.Logger = logging.getLogger(__name__)
+
+
+def setup(coresys: CoreSys) -> FixupBase:
+    """Check setup function."""
+    return FixupAddonExecuteStart(coresys)
+
+
+class FixupAddonExecuteStart(FixupBase):
+    """Storage class for fixup."""
+
+    async def process_fixup(self, reference: str | None = None) -> None:
+        """Initialize the fixup class."""
+        if not (addon := self.sys_addons.get(reference, local_only=True)):
+            _LOGGER.info("Cannot start addon %s as it does not exist", reference)
+            return
+
+        # Start addon
+        try:
+            start_task = await addon.start()
+        except AddonsError as err:
+            _LOGGER.error("Could not start %s due to %s", reference, err)
+            raise ResolutionFixupError() from None
+
+        # Wait for addon start. If it ends up in error or unknown state it's not fixed
+        await start_task
+        if addon.state in {AddonState.ERROR, AddonState.UNKNOWN}:
+            _LOGGER.error("Addon %s could not start successfully", reference)
+            raise ResolutionFixupError()
+
+    @property
+    def suggestion(self) -> SuggestionType:
+        """Return a SuggestionType enum."""
+        return SuggestionType.EXECUTE_START
+
+    @property
+    def context(self) -> ContextType:
+        """Return a ContextType enum."""
+        return ContextType.ADDON
+
+    @property
+    def issues(self) -> list[IssueType]:
+        """Return a IssueType enum list."""
+        return [IssueType.BOOT_FAIL]
+
+    @property
+    def auto(self) -> bool:
+        """Return if a fixup can be apply as auto fix."""
+        return False

tests/addons/test_addon.py

@@ -14,7 +14,7 @@
 from supervisor.addons.addon import Addon
 from supervisor.addons.const import AddonBackupMode
 from supervisor.addons.model import AddonModel
-from supervisor.const import AddonState, BusEvent
+from supervisor.const import AddonBoot, AddonState, BusEvent
 from supervisor.coresys import CoreSys
 from supervisor.docker.addon import DockerAddon
 from supervisor.docker.const import ContainerState
@@ -24,6 +24,8 @@
 from supervisor.store.repository import Repository
 from supervisor.utils.dt import utcnow
 
+from .test_manager import BOOT_FAIL_ISSUE, BOOT_FAIL_SUGGESTIONS
+
 from tests.common import get_fixture_path
 from tests.const import TEST_ADDON_SLUG
 
@@ -895,3 +897,32 @@ async def test_addon_manual_only_boot(coresys: CoreSys, install_addon_example: A
     # However boot mode can change on update and user may have set auto before, ensure it is ignored
     install_addon_example.boot = "auto"
     assert install_addon_example.boot == "manual"
+
+
+async def test_addon_start_dismisses_boot_fail(
+    coresys: CoreSys, install_addon_ssh: Addon
+):
+    """Test a successful start dismisses the boot fail issue."""
+    install_addon_ssh.state = AddonState.ERROR
+    coresys.resolution.add_issue(
+        BOOT_FAIL_ISSUE, [suggestion.type for suggestion in BOOT_FAIL_SUGGESTIONS]
+    )
+
+    install_addon_ssh.state = AddonState.STARTED
+    assert coresys.resolution.issues == []
+    assert coresys.resolution.suggestions == []
+
+
+async def test_addon_disable_boot_dismisses_boot_fail(
+    coresys: CoreSys, install_addon_ssh: Addon
+):
+    """Test a disabling boot dismisses the boot fail issue."""
+    install_addon_ssh.boot = AddonBoot.AUTO
+    install_addon_ssh.state = AddonState.ERROR
+    coresys.resolution.add_issue(
+        BOOT_FAIL_ISSUE, [suggestion.type for suggestion in BOOT_FAIL_SUGGESTIONS]
+    )
+
+    install_addon_ssh.boot = AddonBoot.MANUAL
+    assert coresys.resolution.issues == []
+    assert coresys.resolution.suggestions == []