fix: add retry for get runtime and pipeline

(cherry picked from commit 7e63c9c2d85a7306a3faa577cf26a5d4e6cbfa45)

Author: innsd

agentkit/toolkit/builders/ve_pipeline.py

@@ -32,7 +32,11 @@
 from agentkit.toolkit.models import BuildResult, ImageInfo
 from agentkit.toolkit.reporter import Reporter
 from agentkit.toolkit.errors import ErrorCode
-from agentkit.utils.misc import generate_random_id, calculate_nonlinear_progress
+from agentkit.utils.misc import (
+    generate_random_id,
+    calculate_nonlinear_progress,
+    retry,
+)
 from agentkit.toolkit.volcengine.services import CRServiceConfig
 from .base import Builder
 
@@ -1303,11 +1307,12 @@ def download_and_show_logs(run_id: str):
 
                 while True:
                     try:
-                        # Get pipeline run status
-                        status = cp_client.get_pipeline_run_status(
-                            workspace_id=workspace_id,
-                            pipeline_id=pipeline_id,
-                            run_id=run_id,
+                        status = retry(
+                            lambda: cp_client.get_pipeline_run_status(
+                                workspace_id=workspace_id,
+                                pipeline_id=pipeline_id,
+                                run_id=run_id,
+                            )
                         )
 
                         # Update progress description

agentkit/toolkit/runners/ve_agentkit.py

@@ -39,6 +39,7 @@
     generate_apikey_name,
     generate_client_token,
     calculate_nonlinear_progress,
+    retry,
 )
 from agentkit.sdk.runtime.client import AgentkitRuntimeClient
 from agentkit.toolkit.volcengine.iam import VeIAM
@@ -834,8 +835,10 @@ def _wait_for_runtime_status_multiple(
         # Use reporter.long_task() for progress tracking
         with self.reporter.long_task(task_description, total=total_time) as task:
             while True:
-                runtime = self.agentkit_runtime_client.get_runtime(
-                    runtime_types.GetRuntimeRequest(runtime_id=runtime_id)
+                runtime = retry(
+                    lambda: self.agentkit_runtime_client.get_runtime(
+                        runtime_types.GetRuntimeRequest(runtime_id=runtime_id)
+                    )
                 )
 
                 # Check if target status reached

agentkit/toolkit/utils/__init__.py

@@ -16,4 +16,5 @@
 
 from .agent_parser import AgentParser
 
+
 __all__ = ["AgentParser"]

agentkit/utils/misc.py

@@ -15,6 +15,10 @@
 import math
 import string
 import random
+import time
+from typing import Callable, TypeVar
+
+T = TypeVar("T")
 
 
 def generate_random_id(length=8):
@@ -102,3 +106,17 @@ def calculate_nonlinear_progress(
     """
     progress = max_time * (1 - math.exp(-elapsed / expected_time))
     return min(progress, max_time * max_ratio)
+
+
+def retry(
+    func: Callable[[], T],
+    retries: int = 3,
+    delay: float = 1.0,
+) -> T:
+    for attempt in range(retries):
+        try:
+            return func()
+        except Exception:  # noqa: BLE001
+            if attempt == retries - 1:
+                raise
+            time.sleep(delay)