add configurable timeout

Author: bigabig

backend/src/modules/llm_assistant/llm_job.py

@@ -13,6 +13,7 @@
     router=router,
     device="gpu",
     result_ttl=JobResultTTL.NINETY_DAYS,
+    timeout=-1,  # infinite/no timeout
 )
 def llm_assistant(
     payload: LLMJobInput,

backend/src/systems/job_system/job_register_decorator.py

@@ -26,6 +26,7 @@ def register_job(
     router: APIRouter | None = None,
     result_ttl: JobResultTTL = JobResultTTL.DEFAULT,
     retry: tuple[int, int] | None = None,
+    timeout: int = 1800,  # default timeout of 30 min (RQ default is 3 min [180])
 ):
     def decorator(func: Callable[[InputT, Job], OutputT | None]):
         from systems.job_system.job_service import JobService
@@ -41,6 +42,7 @@ def decorator(func: Callable[[InputT, Job], OutputT | None]):
             router=router,
             result_ttl=result_ttl,
             retry=retry,
+            timeout=timeout,
         )
         return func
 

backend/src/systems/job_system/job_service.py

@@ -42,9 +42,9 @@ class RegisteredJob(TypedDict):
     result_ttl: JobResultTTL  # how long to keep successful jobs and their results (defaults to 500 seconds)
     # failure_ttl  # how long to keep failed jobs (defaults to 1 year)
     # ttl  # how long to keep jobs in queue before they are discarded (defaults to infinite)
-    # timeout  # specifies maximum runtime before the job is interrupted and marked as failed
     # see https://python-rq.org/docs/jobs/#job-creation
     retry: tuple[int, int] | None
+    timeout: int  # maximum runtime before the job is interrupted and marked as failed (in seconds)
 
 
 class JobService(metaclass=SingletonMeta):
@@ -103,6 +103,7 @@ def register_job(
         router: APIRouter | None,
         result_ttl: JobResultTTL,
         retry: tuple[int, int] | None,
+        timeout: int,
     ) -> None:
         # Enforce that the only parameter is named 'payload'
         sig = inspect.signature(handler_func)
@@ -130,6 +131,7 @@ def register_job(
             "router": router,
             "result_ttl": result_ttl,
             "retry": retry,
+            "timeout": timeout,
         }
 
     def start_job(