Proposition: retry on rate limit errors (#1801)

Author: aymeric-roucher

README.md

@@ -95,7 +95,7 @@ model = InferenceClientModel(
 from smolagents import LiteLLMModel
 
 model = LiteLLMModel(
-    model_id="anthropic/claude-3-5-sonnet-latest",
+    model_id="anthropic/claude-4-sonnet-latest",
     temperature=0.2,
     api_key=os.environ["ANTHROPIC_API_KEY"]
 )
@@ -106,9 +106,9 @@ model = LiteLLMModel(
 
 ```py
 import os
-from smolagents import OpenAIServerModel
+from smolagents import OpenAIModel
 
-model = OpenAIServerModel(
+model = OpenAIModel(
     model_id="deepseek-ai/DeepSeek-R1",
     api_base="https://api.together.xyz/v1/", # Leave this blank to query OpenAI servers.
     api_key=os.environ["TOGETHER_API_KEY"], # Switch to the API key for the server you're targeting.
@@ -120,9 +120,9 @@ model = OpenAIServerModel(
 
 ```py
 import os
-from smolagents import OpenAIServerModel
+from smolagents import OpenAIModel
 
-model = OpenAIServerModel(
+model = OpenAIModel(
     model_id="openai/gpt-4o",
     api_base="https://openrouter.ai/api/v1", # Leave this blank to query OpenAI servers.
     api_key=os.environ["OPENROUTER_API_KEY"], # Switch to the API key for the server you're targeting.
@@ -137,7 +137,7 @@ model = OpenAIServerModel(
 from smolagents import TransformersModel
 
 model = TransformersModel(
-    model_id="Qwen/Qwen2.5-Coder-32B-Instruct",
+    model_id="Qwen/Qwen3-4B-Instruct-2507",
     max_new_tokens=4096,
     device_map="auto"
 )
@@ -148,9 +148,9 @@ model = TransformersModel(
 
 ```py
 import os
-from smolagents import AzureOpenAIServerModel
+from smolagents import AzureOpenAIModel
 
-model = AzureOpenAIServerModel(
+model = AzureOpenAIModel(
     model_id = os.environ.get("AZURE_OPENAI_MODEL"),
     azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"),
     api_key=os.environ.get("AZURE_OPENAI_API_KEY"),
@@ -163,9 +163,9 @@ model = AzureOpenAIServerModel(
 
 ```py
 import os
-from smolagents import AmazonBedrockServerModel
+from smolagents import AmazonBedrockModel
 
-model = AmazonBedrockServerModel(
+model = AmazonBedrockModel(
     model_id = os.environ.get("AMAZON_BEDROCK_MODEL_ID") 
 )
 ```
@@ -178,14 +178,14 @@ You can run agents from CLI using two commands: `smolagent` and `webagent`.
 `smolagent` is a generalist command to run a multi-step `CodeAgent` that can be equipped with various tools.
 
 ```bash
-smolagent "Plan a trip to Tokyo, Kyoto and Osaka between Mar 28 and Apr 7."  --model-type "InferenceClientModel" --model-id "Qwen/Qwen2.5-Coder-32B-Instruct" --imports "pandas numpy" --tools "web_search"
+smolagent "Plan a trip to Tokyo, Kyoto and Osaka between Mar 28 and Apr 7."  --model-type "InferenceClientModel" --model-id "Qwen/Qwen3-Next-80B-A3B-Instruct" --imports pandas numpy --tools web_search
 ```
 
 Meanwhile `webagent` is a specific web-browsing agent using [helium](https://github.com/mherrmann/helium) (read more [here](https://github.com/huggingface/smolagents/blob/main/src/smolagents/vision_web_browser.py)).
 
 For instance:
 ```bash
-webagent "go to xyz.com/men, get to sale section, click the first clothing item you see. Get the product details, and the price, return them. note that I'm shopping from France" --model-type "LiteLLMModel" --model-id "gpt-4o"
+webagent "go to xyz.com/men, get to sale section, click the first clothing item you see. Get the product details, and the price, return them. note that I'm shopping from France" --model-type "LiteLLMModel" --model-id "gpt-5"
 ```
 
 ## How do Code agents work?

examples/multiple_tools.py

@@ -9,7 +9,7 @@
 # model = TransformersModel(model_id="meta-llama/Llama-3.2-2B-Instruct")
 
 # For anthropic: change model_id below to 'anthropic/claude-3-5-sonnet-20240620'
-# model = LiteLLMModel(model_id="gpt-4o")
+# model = LiteLLMModel(model_id="gpt-5")
 
 
 @tool

examples/rag.py

@@ -58,7 +58,7 @@ def forward(self, query: str) -> str:
 retriever_tool = RetrieverTool(docs_processed)
 agent = CodeAgent(
     tools=[retriever_tool],
-    model=InferenceClientModel(model_id="Qwen/Qwen2.5-Coder-32B-Instruct"),
+    model=InferenceClientModel(model_id="Qwen/Qwen3-Next-80B-A3B-Instruct"),
     max_steps=4,
     verbosity_level=2,
     stream_outputs=True,

examples/rag_using_chromadb.py

@@ -98,14 +98,14 @@ def forward(self, query: str) -> str:
 # Choose which LLM engine to use!
 
 # from smolagents import InferenceClientModel
-# model = InferenceClientModel(model_id="meta-llama/Llama-3.3-70B-Instruct")
+# model = InferenceClientModel(model_id="Qwen/Qwen3-Next-80B-A3B-Instruct")
 
 # from smolagents import TransformersModel
-# model = TransformersModel(model_id="meta-llama/Llama-3.2-2B-Instruct")
+# model = TransformersModel(model_id="Qwen/Qwen3-4B-Instruct-2507")
 
-# For anthropic: change model_id below to 'anthropic/claude-3-5-sonnet-20240620' and also change 'os.environ.get("ANTHROPIC_API_KEY")'
+# For anthropic: change model_id below to 'anthropic/claude-4-sonnet-latest' and also change 'os.environ.get("ANTHROPIC_API_KEY")'
 model = LiteLLMModel(
-    model_id="groq/llama-3.3-70b-versatile",
+    model_id="groq/openai/gpt-oss-120b",
     api_key=os.environ.get("GROQ_API_KEY"),
 )
 

pyproject.toml

@@ -16,8 +16,9 @@ dependencies = [
   "requests>=2.32.3",
   "rich>=13.9.4",
   "jinja2>=3.1.4",
-  "pillow>=10.0.1",  # Security fix for CVE-2023-4863: https://pillow.readthedocs.io/en/stable/releasenotes/10.0.1.html
-  "python-dotenv"
+  "pillow>=10.0.1",
+  # Security fix for CVE-2023-4863: https://pillow.readthedocs.io/en/stable/releasenotes/10.0.1.html
+  "python-dotenv",
 ]
 
 [project.optional-dependencies]

src/smolagents/models.py

@@ -26,7 +26,7 @@
 
 from .monitoring import TokenUsage
 from .tools import Tool
-from .utils import RateLimiter, _is_package_available, encode_image_base64, make_image_url, parse_json_blob
+from .utils import RateLimiter, Retrying, _is_package_available, encode_image_base64, make_image_url, parse_json_blob
 
 
 if TYPE_CHECKING:
@@ -35,6 +35,8 @@
 
 logger = logging.getLogger(__name__)
 
+RETRY_WAIT = 120
+RETRY_MAX_ATTEMPTS = 3
 STRUCTURED_GENERATION_PROVIDERS = ["cerebras", "fireworks-ai"]
 CODEAGENT_RESPONSE_FORMAT = {
     "type": "json_schema",
@@ -1078,6 +1080,8 @@ class ApiModel(Model):
             Pre-configured API client instance. If not provided, a default client will be created. Defaults to None.
         requests_per_minute (`float`, **optional**):
             Rate limit in requests per minute.
+        retry (`bool`, **optional**):
+            Wether to retry on rate limit errors, up to RETRY_MAX_ATTEMPTS times. Defaults to True.
         **kwargs:
             Additional keyword arguments to forward to the underlying model completion call.
     """
@@ -1088,12 +1092,21 @@ def __init__(
         custom_role_conversions: dict[str, str] | None = None,
         client: Any | None = None,
         requests_per_minute: float | None = None,
+        retry: bool = True,
         **kwargs,
     ):
         super().__init__(model_id=model_id, **kwargs)
         self.custom_role_conversions = custom_role_conversions or {}
         self.client = client or self.create_client()
         self.rate_limiter = RateLimiter(requests_per_minute)
+        self.retryer = Retrying(
+            max_attempts=RETRY_MAX_ATTEMPTS if retry else 1,
+            wait_seconds=RETRY_WAIT,
+            retry_predicate=is_rate_limit_error,
+            reraise=True,
+            before_sleep_logger=(logger, logging.INFO),
+            after_logger=(logger, logging.INFO),
+        )
 
     def create_client(self):
         """Create the API client for the specific service."""
@@ -1104,6 +1117,17 @@ def _apply_rate_limit(self):
         self.rate_limiter.throttle()
 
 
+def is_rate_limit_error(exception: BaseException) -> bool:
+    """Check if the exception is a rate limit error."""
+    error_str = str(exception).lower()
+    return (
+        "429" in error_str
+        or "rate limit" in error_str
+        or "too many requests" in error_str
+        or "rate_limit" in error_str
+    )
+
+
 class LiteLLMModel(ApiModel):
     """Model to use [LiteLLM Python SDK](https://docs.litellm.ai/docs/#litellm-python-sdk) to access hundreds of LLMs.
 
@@ -1186,7 +1210,8 @@ def generate(
             **kwargs,
         )
         self._apply_rate_limit()
-        response = self.client.completion(**completion_kwargs)
+        response = self.retryer(self.client.completion, **completion_kwargs)
+
         if not response.choices:
             raise RuntimeError(
                 f"Unexpected API response: model '{self.model_id}' returned no choices. "
@@ -1228,7 +1253,9 @@ def generate_stream(
             **kwargs,
         )
         self._apply_rate_limit()
-        for event in self.client.completion(**completion_kwargs, stream=True, stream_options={"include_usage": True}):
+        for event in self.retryer(
+            self.client.completion, **completion_kwargs, stream=True, stream_options={"include_usage": True}
+        ):
             if getattr(event, "usage", None):
                 yield ChatMessageStreamDelta(
                     content="",
@@ -1398,8 +1425,8 @@ class InferenceClientModel(ApiModel):
     Example:
     ```python
     >>> engine = InferenceClientModel(
-    ...     model_id="Qwen/Qwen2.5-Coder-32B-Instruct",
-    ...     provider="nebius",
+    ...     model_id="Qwen/Qwen3-Next-80B-A3B-Thinking",
+    ...     provider="hyperbolic",
     ...     token="your_hf_token_here",
     ...     max_tokens=5000,
     ... )
@@ -1412,7 +1439,7 @@ class InferenceClientModel(ApiModel):
 
     def __init__(
         self,
-        model_id: str = "Qwen/Qwen2.5-Coder-32B-Instruct",
+        model_id: str = "Qwen/Qwen3-Next-80B-A3B-Instruct",
         provider: str | None = None,
         token: str | None = None,
         timeout: int = 120,
@@ -1472,7 +1499,7 @@ def generate(
             **kwargs,
         )
         self._apply_rate_limit()
-        response = self.client.chat_completion(**completion_kwargs)
+        response = self.retryer(self.client.chat_completion, **completion_kwargs)
         content = response.choices[0].message.content
         if stop_sequences is not None and not self.supports_stop_parameter:
             content = remove_content_after_stop_sequences(content, stop_sequences)
@@ -1506,8 +1533,11 @@ def generate_stream(
             **kwargs,
         )
         self._apply_rate_limit()
-        for event in self.client.chat.completions.create(
-            **completion_kwargs, stream=True, stream_options={"include_usage": True}
+        for event in self.retryer(
+            self.client.chat.completions.create,
+            **completion_kwargs,
+            stream=True,
+            stream_options={"include_usage": True},
         ):
             if getattr(event, "usage", None):
                 yield ChatMessageStreamDelta(
@@ -1539,12 +1569,12 @@ def generate_stream(
                         raise ValueError(f"No content or tool calls in event: {event}")
 
 
-class OpenAIServerModel(ApiModel):
+class OpenAIModel(ApiModel):
     """This model connects to an OpenAI-compatible API server.
 
     Parameters:
         model_id (`str`):
-            The model identifier to use on the server (e.g. "gpt-3.5-turbo").
+            The model identifier to use on the server (e.g. "gpt-5").
         api_base (`str`, *optional*):
             The base URL of the OpenAI-compatible API server.
         api_key (`str`, *optional*):
@@ -1595,7 +1625,7 @@ def create_client(self):
             import openai
         except ModuleNotFoundError as e:
             raise ModuleNotFoundError(
-                "Please install 'openai' extra to use OpenAIServerModel: `pip install 'smolagents[openai]'`"
+                "Please install 'openai' extra to use OpenAIModel: `pip install 'smolagents[openai]'`"
             ) from e
 
         return openai.OpenAI(**self.client_kwargs)
@@ -1619,8 +1649,11 @@ def generate_stream(
             **kwargs,
         )
         self._apply_rate_limit()
-        for event in self.client.chat.completions.create(
-            **completion_kwargs, stream=True, stream_options={"include_usage": True}
+        for event in self.retryer(
+            self.client.chat.completions.create,
+            **completion_kwargs,
+            stream=True,
+            stream_options={"include_usage": True},
         ):
             if event.usage:
                 yield ChatMessageStreamDelta(
@@ -1670,7 +1703,7 @@ def generate(
             **kwargs,
         )
         self._apply_rate_limit()
-        response = self.client.chat.completions.create(**completion_kwargs)
+        response = self.retryer(self.client.chat.completions.create, **completion_kwargs)
         content = response.choices[0].message.content
         if stop_sequences is not None and not self.supports_stop_parameter:
             content = remove_content_after_stop_sequences(content, stop_sequences)
@@ -1686,10 +1719,10 @@ def generate(
         )
 
 
-OpenAIModel = OpenAIServerModel
+OpenAIServerModel = OpenAIModel
 
 
-class AzureOpenAIServerModel(OpenAIServerModel):
+class AzureOpenAIModel(OpenAIModel):
     """This model connects to an Azure OpenAI deployment.
 
     Parameters:
@@ -1740,16 +1773,16 @@ def create_client(self):
             import openai
         except ModuleNotFoundError as e:
             raise ModuleNotFoundError(
-                "Please install 'openai' extra to use AzureOpenAIServerModel: `pip install 'smolagents[openai]'`"
+                "Please install 'openai' extra to use AzureOpenAIModel: `pip install 'smolagents[openai]'`"
             ) from e
 
         return openai.AzureOpenAI(**self.client_kwargs)
 
 
-AzureOpenAIModel = AzureOpenAIServerModel
+AzureOpenAIServerModel = AzureOpenAIModel
 
 
-class AmazonBedrockServerModel(ApiModel):
+class AmazonBedrockModel(ApiModel):
     """
     A model class for interacting with Amazon Bedrock Server models through the Bedrock API.
 
@@ -1789,7 +1822,7 @@ class AmazonBedrockServerModel(ApiModel):
     Examples:
         Creating a model instance with default settings:
         ```python
-        >>> bedrock_model = AmazonBedrockServerModel(
+        >>> bedrock_model = AmazonBedrockModel(
         ...     model_id='us.amazon.nova-pro-v1:0'
         ... )
         ```
@@ -1798,15 +1831,15 @@ class AmazonBedrockServerModel(ApiModel):
         ```python
         >>> import boto3
         >>> client = boto3.client('bedrock-runtime', region_name='us-west-2')
-        >>> bedrock_model = AmazonBedrockServerModel(
+        >>> bedrock_model = AmazonBedrockModel(
         ...     model_id='us.amazon.nova-pro-v1:0',
         ...     client=client
         ... )
         ```
 
         Creating a model instance with client_kwargs for internal client creation:
         ```python
-        >>> bedrock_model = AmazonBedrockServerModel(
+        >>> bedrock_model = AmazonBedrockModel(
         ...     model_id='us.amazon.nova-pro-v1:0',
         ...     client_kwargs={'region_name': 'us-west-2', 'endpoint_url': 'https://custom-endpoint.com'}
         ... )
@@ -1823,7 +1856,7 @@ class AmazonBedrockServerModel(ApiModel):
         ...         "guardrailVersion": 'v1'
         ...     },
         ... }
-        >>> bedrock_model = AmazonBedrockServerModel(
+        >>> bedrock_model = AmazonBedrockModel(
         ...     model_id='anthropic.claude-3-haiku-20240307-v1:0',
         ...     **additional_api_config
         ... )
@@ -1929,7 +1962,7 @@ def generate(
         )
         self._apply_rate_limit()
         # self.client is created in ApiModel class
-        response = self.client.converse(**completion_kwargs)
+        response = self.retryer(self.client.converse, **completion_kwargs)
 
         # Get content blocks with "text" key: in case thinking blocks are present, discard them
         message_content_blocks_with_text = [
@@ -1953,7 +1986,7 @@ def generate(
         )
 
 
-AmazonBedrockModel = AmazonBedrockServerModel
+AmazonBedrockServerModel = AmazonBedrockModel
 
 __all__ = [
     "REMOVE_PARAMETER",