Report stage with error in jobs

Author: mdegat01

supervisor/backups/manager.py

@@ -7,7 +7,7 @@
 import errno
 import logging
 from pathlib import Path
-from shutil import copy
+import shutil
 
 from ..addons.addon import Addon
 from ..const import (
@@ -368,14 +368,18 @@ async def _copy_to_additional_locations(
     ):
         """Copy a backup file to additional locations."""
 
+        all_new_locations: dict[str | None, Path] = {}
+
         def copy_to_additional_locations() -> dict[str | None, Path]:
             """Copy backup file to additional locations."""
-            all_locations: dict[str | None, Path] = {}
+            nonlocal all_new_locations
             for location in locations:
                 try:
                     if location == LOCATION_CLOUD_BACKUP:
-                        all_locations[LOCATION_CLOUD_BACKUP] = Path(
-                            copy(backup.tarfile, self.sys_config.path_core_backup)
+                        all_new_locations[LOCATION_CLOUD_BACKUP] = Path(
+                            shutil.copy(
+                                backup.tarfile, self.sys_config.path_core_backup
+                            )
                         )
                     elif location:
                         location_mount: Mount = location
@@ -384,12 +388,12 @@ def copy_to_additional_locations() -> dict[str | None, Path]:
                                 f"{location_mount.name} is down, cannot copy to it",
                                 _LOGGER.error,
                             )
-                        all_locations[location_mount.name] = Path(
-                            copy(backup.tarfile, location_mount.local_where)
+                        all_new_locations[location_mount.name] = Path(
+                            shutil.copy(backup.tarfile, location_mount.local_where)
                         )
                     else:
-                        all_locations[None] = Path(
-                            copy(backup.tarfile, self.sys_config.path_backup)
+                        all_new_locations[None] = Path(
+                            shutil.copy(backup.tarfile, self.sys_config.path_backup)
                         )
                 except OSError as err:
                     msg = f"Could not copy backup to {location.name if isinstance(location, Mount) else location} due to: {err!s}"
@@ -401,12 +405,8 @@ def copy_to_additional_locations() -> dict[str | None, Path]:
                         raise BackupDataDiskBadMessageError(msg, _LOGGER.error) from err
                     raise BackupError(msg, _LOGGER.error) from err
 
-            return all_locations
-
         try:
-            all_new_locations = await self.sys_run_in_executor(
-                copy_to_additional_locations
-            )
+            await self.sys_run_in_executor(copy_to_additional_locations)
         except BackupDataDiskBadMessageError:
             self.sys_resolution.add_unhealthy_reason(
                 UnhealthyReason.OSERROR_BAD_MESSAGE

supervisor/jobs/__init__.py

@@ -73,10 +73,15 @@ class SupervisorJobError:
 
     type_: type[HassioError] = HassioError
     message: str = "Unknown error, see supervisor logs"
+    stage: str | None = None
 
     def as_dict(self) -> dict[str, str]:
         """Return dictionary representation."""
-        return {"type": self.type_.__name__, "message": self.message}
+        return {
+            "type": self.type_.__name__,
+            "message": self.message,
+            "stage": self.stage,
+        }
 
 
 @define(order=True)
@@ -126,9 +131,9 @@ def as_dict(self) -> dict[str, Any]:
     def capture_error(self, err: HassioError | None = None) -> None:
         """Capture an error or record that an unknown error has occurred."""
         if err:
-            new_error = SupervisorJobError(type(err), str(err))
+            new_error = SupervisorJobError(type(err), str(err), self.stage)
         else:
-            new_error = SupervisorJobError()
+            new_error = SupervisorJobError(stage=self.stage)
         self.errors += [new_error]
 
     @contextmanager

tests/api/test_backups.py

@@ -658,6 +658,56 @@ async def test_backup_to_multiple_locations(
     assert coresys.backups.get(slug).location is None
 
 
+@pytest.mark.usefixtures("path_extern", "tmp_supervisor_data")
+@pytest.mark.parametrize(
+    ("backup_type", "inputs"), [("full", {}), ("partial", {"folders": ["ssl"]})]
+)
+async def test_backup_to_multiple_locations_error_on_copy(
+    api_client: TestClient,
+    coresys: CoreSys,
+    backup_type: str,
+    inputs: dict[str, Any],
+):
+    """Test making a backup to multiple locations that fails during copy stage."""
+    await coresys.core.set_state(CoreState.RUNNING)
+    coresys.hardware.disk.get_disk_free_space = lambda x: 5000
+
+    with patch("supervisor.backups.manager.shutil.copy", side_effect=OSError):
+        resp = await api_client.post(
+            f"/backups/new/{backup_type}",
+            json={
+                "name": "Multiple locations test",
+                "location": [None, ".cloud_backup"],
+            }
+            | inputs,
+        )
+    assert resp.status == 200
+    result = await resp.json()
+    assert result["result"] == "ok"
+    slug = result["data"]["slug"]
+
+    orig_backup = coresys.config.path_backup / f"{slug}.tar"
+    assert await coresys.run_in_executor(orig_backup.exists)
+    assert coresys.backups.get(slug).all_locations == {
+        None: {"path": orig_backup, "protected": False, "size_bytes": 10240},
+    }
+    assert coresys.backups.get(slug).location is None
+
+    resp = await api_client.get("/jobs/info")
+    assert resp.status == 200
+    result = await resp.json()
+    assert result["data"]["jobs"][0]["name"] == f"backup_manager_{backup_type}_backup"
+    assert result["data"]["jobs"][0]["reference"] == slug
+    assert result["data"]["jobs"][0]["done"] is True
+    assert result["data"]["jobs"][0]["errors"] == [
+        {
+            "type": "BackupError",
+            "message": "Could not copy backup to .cloud_backup due to: ",
+            "stage": "copy_additional_locations",
+        }
+    ]
+
+
 @pytest.mark.parametrize(
     ("backup_type", "inputs"), [("full", {}), ("partial", {"folders": ["ssl"]})]
 )

tests/api/test_jobs.py

@@ -7,6 +7,7 @@
 import pytest
 
 from supervisor.coresys import CoreSys
+from supervisor.exceptions import SupervisorError
 from supervisor.jobs.const import ATTR_IGNORE_CONDITIONS, JobCondition
 from supervisor.jobs.decorator import Job
 
@@ -317,3 +318,72 @@ async def test_jobs_sorted_2(self):
             ],
         },
     ]
+
+
+async def test_job_with_error(
+    api_client: TestClient,
+    coresys: CoreSys,
+):
+    """Test job output with an error."""
+
+    class TestClass:
+        """Test class."""
+
+        def __init__(self, coresys: CoreSys):
+            """Initialize the test class."""
+            self.coresys = coresys
+
+        @Job(name="test_jobs_api_error_outer", cleanup=False)
+        async def test_jobs_api_error_outer(self):
+            """Error test outer method."""
+            coresys.jobs.current.stage = "test"
+            await self.test_jobs_api_error_inner()
+
+        @Job(name="test_jobs_api_error_inner", cleanup=False)
+        async def test_jobs_api_error_inner(self):
+            """Error test inner method."""
+            raise SupervisorError("bad")
+
+    test = TestClass(coresys)
+    with pytest.raises(SupervisorError):
+        await test.test_jobs_api_error_outer()
+
+    resp = await api_client.get("/jobs/info")
+    result = await resp.json()
+    assert result["data"]["jobs"] == [
+        {
+            "created": ANY,
+            "name": "test_jobs_api_error_outer",
+            "reference": None,
+            "uuid": ANY,
+            "progress": 0,
+            "stage": "test",
+            "done": True,
+            "errors": [
+                {
+                    "type": "SupervisorError",
+                    "message": "bad",
+                    "stage": "test",
+                }
+            ],
+            "child_jobs": [
+                {
+                    "created": ANY,
+                    "name": "test_jobs_api_error_inner",
+                    "reference": None,
+                    "uuid": ANY,
+                    "progress": 0,
+                    "stage": None,
+                    "done": True,
+                    "errors": [
+                        {
+                            "type": "SupervisorError",
+                            "message": "bad",
+                            "stage": None,
+                        }
+                    ],
+                    "child_jobs": [],
+                },
+            ],
+        },
+    ]

tests/jobs/test_job_manager.py

@@ -195,6 +195,7 @@ async def test_notify_on_change(coresys: CoreSys, ha_ws_client: AsyncMock):
                             {
                                 "type": "HassioError",
                                 "message": "Unknown error, see supervisor logs",
+                                "stage": "test",
                             }
                         ],
                         "created": ANY,
@@ -221,6 +222,7 @@ async def test_notify_on_change(coresys: CoreSys, ha_ws_client: AsyncMock):
                         {
                             "type": "HassioError",
                             "message": "Unknown error, see supervisor logs",
+                            "stage": "test",
                         }
                     ],
                     "created": ANY,