fixed issues with failing services

Author: Andrei Neagu

packages/service-library/src/servicelib/long_running_tasks/_rabbit/_models.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+
+class RPCErrorResponse(BaseModel):
+    str_traceback: str
+    error_object: str

packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py

@@ -10,6 +10,7 @@
 from ...rabbitmq._client_rpc import RabbitMQRPCClient
 from .._serialization import string_to_object
 from ..models import RabbitNamespace, TaskBase, TaskContext, TaskId, TaskStatus
+from ._models import RPCErrorResponse
 from .namespace import get_namespace
 
 _logger = logging.getLogger(__name__)
@@ -99,12 +100,18 @@ async def get_task_result(
         task_id=task_id,
         timeout_s=_RPC_TIMEOUT_NORMAL_REQUEST,
     )
-    assert isinstance(serialized_result, str)  # nosec
-    task_result = string_to_object(serialized_result)
+    assert isinstance(serialized_result, RPCErrorResponse | str)  # nosec
+    if isinstance(serialized_result, RPCErrorResponse):
+        error = string_to_object(serialized_result.error_object)
+        _logger.warning(
+            "Remote task finished with error: '%s: %s'\n%s",
+            error.__class__.__name__,
+            error,
+            serialized_result.str_traceback,
+        )
+        raise error
 
-    if isinstance(task_result, Exception):
-        raise task_result
-    return task_result
+    return string_to_object(serialized_result)
 
 
 @log_decorator(_logger, level=logging.DEBUG)

packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py

@@ -1,4 +1,5 @@
 import logging
+import traceback
 from typing import Any
 
 from ...rabbitmq import RPCRouter
@@ -7,6 +8,7 @@
 from ..errors import BaseLongRunningError, TaskNotCompletedError, TaskNotFoundError
 from ..models import TaskBase, TaskContext, TaskId, TaskStatus
 from ..task import RegisteredTaskName
+from ._models import RPCErrorResponse
 
 _logger = logging.getLogger(__name__)
 
@@ -85,15 +87,18 @@ async def get_task_result(
     *,
     task_context: TaskContext,
     task_id: TaskId,
-) -> str:
+) -> RPCErrorResponse | str:
     try:
         return object_to_string(
             await _get_transferarble_task_result(
                 long_running_manager, task_context=task_context, task_id=task_id
             )
         )
     except Exception as exc:  # pylint:disable=broad-exception-caught
-        return object_to_string(exc)
+        return RPCErrorResponse(
+            str_traceback="".join(traceback.format_tb(exc.__traceback__)),
+            error_object=object_to_string(exc),
+        )
 
 
 @router.expose(reraise_if_error_type=(BaseLongRunningError,))