From 46cac667509b5c4ca24997b7850f9879e8ff89db Mon Sep 17 00:00:00 2001 From: Surya Balakrishnan Date: Thu, 2 Oct 2025 19:39:55 +0000 Subject: [PATCH 1/8] Improve error handling for short output token lengths --- src/smolagents/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/smolagents/utils.py b/src/smolagents/utils.py index 3fa22fec2..3d0c8c688 100644 --- a/src/smolagents/utils.py +++ b/src/smolagents/utils.py @@ -155,7 +155,7 @@ def parse_json_blob(json_blob: str) -> tuple[dict[str, str], str]: json_data = json.loads(json_str, strict=False) return json_data, json_blob[:first_accolade_index] except IndexError: - raise ValueError("The model output does not contain any JSON blob.") + raise ValueError("The model output does not contain any JSON blob. Try increasing the maximum output token length.") except json.JSONDecodeError as e: place = e.pos if json_blob[place - 1 : place + 2] == "},\n": From e99af0346b6bf9a0d2f14dfecc4597f4204d2293 Mon Sep 17 00:00:00 2001 From: Surya Balakrishnan Date: Sat, 11 Oct 2025 07:17:47 +0000 Subject: [PATCH 2/8] Remove warning from agent memory --- src/smolagents/agents.py | 8 ++++++++ src/smolagents/monitoring.py | 3 +++ src/smolagents/utils.py | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/smolagents/agents.py b/src/smolagents/agents.py index ae2c9344a..cb0638a6a 100644 --- a/src/smolagents/agents.py +++ b/src/smolagents/agents.py @@ -1302,6 +1302,10 @@ def _step_stream( try: chat_message = self.model.parse_tool_calls(chat_message) except Exception as e: + if isinstance(e, ValueError) and "does not contain any JSON blob" in str(e): + self.logger.log_warning( + "The model's output is missing JSON data. This could be due to an insufficient number of output tokens set in the model configuration" + ) raise AgentParsingError(f"Error while parsing tool call from model output: {e}", self.logger) else: for tool_call in chat_message.tool_calls: @@ -1679,6 +1683,10 @@ def _step_stream( code_action = fix_final_answer_code(code_action) memory_step.code_action = code_action except Exception as e: + if isinstance(e, ValueError) and "Make sure to include code with the correct pattern" in str(e): + self.logger.log_warning( + "The model's output is missing a Code output. This could be due to an insufficient number of output tokens set in the model configuration" + ) error_msg = f"Error in code parsing:\n{e}\nMake sure to provide correct code blobs." raise AgentParsingError(error_msg, self.logger) diff --git a/src/smolagents/monitoring.py b/src/smolagents/monitoring.py index 02959edd2..16c6ec7ea 100644 --- a/src/smolagents/monitoring.py +++ b/src/smolagents/monitoring.py @@ -187,6 +187,9 @@ def log_code(self, title: str, content: str, level: int = LogLevel.INFO) -> None level=level, ) + def log_warning(self, title: str, level: int = LogLevel.INFO) -> None: + self.log("Warning: " + title, style=YELLOW_HEX, level=LogLevel.INFO) + def log_rule(self, title: str, level: int = LogLevel.INFO) -> None: self.log( Rule( diff --git a/src/smolagents/utils.py b/src/smolagents/utils.py index 3d0c8c688..3fa22fec2 100644 --- a/src/smolagents/utils.py +++ b/src/smolagents/utils.py @@ -155,7 +155,7 @@ def parse_json_blob(json_blob: str) -> tuple[dict[str, str], str]: json_data = json.loads(json_str, strict=False) return json_data, json_blob[:first_accolade_index] except IndexError: - raise ValueError("The model output does not contain any JSON blob. Try increasing the maximum output token length.") + raise ValueError("The model output does not contain any JSON blob.") except json.JSONDecodeError as e: place = e.pos if json_blob[place - 1 : place + 2] == "},\n": From 33ceea0760064acfcdd1c0d99df2daf1f6319214 Mon Sep 17 00:00:00 2001 From: Surya Balakrishnan Date: Sat, 11 Oct 2025 07:55:58 +0000 Subject: [PATCH 3/8] Add documentation --- docs/source/en/tutorials/building_good_agents.md | 13 +++++++++++++ src/smolagents/agents.py | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/source/en/tutorials/building_good_agents.md b/docs/source/en/tutorials/building_good_agents.md index cbfcd1e32..d1f101163 100644 --- a/docs/source/en/tutorials/building_good_agents.md +++ b/docs/source/en/tutorials/building_good_agents.md @@ -422,3 +422,16 @@ result = agent.run( "How long would a cheetah at full speed take to run the length of Pont Alexandre III?", ) ``` + +### 5. Increase maximum output tokens + +If you see warnings indicating that the Code or JSON data is missing in the output, It might be because the output tokens configured for your model might be too less. The model is not generating a code snippet or a tool call that can be parsed by the framework. Please try increasing the model output token configuration (`num_ctx`, `max_tokens` etc) + +For example in a CodeAgent's logs you should see +``` +Warning: The model's output is missing a Code output. This could be due to an insufficient number of output tokens set in the model configuration. + +and for a ToolCallingAgent +``` +Warning: The model's output is missing JSON data. This could be due to an insufficient number of output tokens set in the model configuration +``` diff --git a/src/smolagents/agents.py b/src/smolagents/agents.py index cb0638a6a..0a177c08b 100644 --- a/src/smolagents/agents.py +++ b/src/smolagents/agents.py @@ -1304,7 +1304,7 @@ def _step_stream( except Exception as e: if isinstance(e, ValueError) and "does not contain any JSON blob" in str(e): self.logger.log_warning( - "The model's output is missing JSON data. This could be due to an insufficient number of output tokens set in the model configuration" + "The model's output is missing JSON data. This could be due to an insufficient number of output tokens set in the model configuration." ) raise AgentParsingError(f"Error while parsing tool call from model output: {e}", self.logger) else: @@ -1685,7 +1685,7 @@ def _step_stream( except Exception as e: if isinstance(e, ValueError) and "Make sure to include code with the correct pattern" in str(e): self.logger.log_warning( - "The model's output is missing a Code output. This could be due to an insufficient number of output tokens set in the model configuration" + "The model's output is missing a Code output. This could be due to an insufficient number of output tokens set in the model configuration." ) error_msg = f"Error in code parsing:\n{e}\nMake sure to provide correct code blobs." raise AgentParsingError(error_msg, self.logger) From 78475f205966ef3080b0809288a8a8c78ef6c1a0 Mon Sep 17 00:00:00 2001 From: Surya Balakrishnan Date: Sat, 11 Oct 2025 09:05:34 +0000 Subject: [PATCH 4/8] Add unit tests --- src/smolagents/agents.py | 2 +- tests/test_utils.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/smolagents/agents.py b/src/smolagents/agents.py index fc0d69df9..299e073b0 100644 --- a/src/smolagents/agents.py +++ b/src/smolagents/agents.py @@ -1302,7 +1302,7 @@ def _step_stream( try: chat_message = self.model.parse_tool_calls(chat_message) except Exception as e: - if isinstance(e, ValueError) and "does not contain any JSON blob" in str(e): + if isinstance(e, ValueError) and "The model output does not contain any JSON blob" in str(e): self.logger.log_warning( "The model's output is missing JSON data. This could be due to an insufficient number of output tokens set in the model configuration." ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 740ee3943..7acf7734c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -142,6 +142,18 @@ def test_multiple_code_blobs(self): result = parse_code_blobs(test_input, ("", "")) assert result == "Foo\n\ncode_a\n\ncode_b" +@pytest.mark.parametrize( + "raw_text", + [ + "This is just some conversational text.", + "Here is an invalid code snippet `x = 10", + ], +) +def test_parse_code_blobs_without_valid_code(raw_text): + # Note: If the exact string is changed a parsing warning needs to be modified in agents.py + with pytest.raises(ValueError, match="Make sure to include code with the correct pattern"): + parse_code_blobs(raw_text, ("", "")) + @pytest.fixture(scope="function") def ipython_shell(): @@ -485,6 +497,19 @@ def test_parse_json_blob_with_invalid_json(raw_json): with pytest.raises(Exception): parse_json_blob(raw_json) +@pytest.mark.parametrize( + "raw_json", + [ + "this string has no json blob", + "this string has an opening brace { but no closing one", + "", + "some text {", + ], +) +def test_parse_json_blob_without_json_blob(raw_json): + # Note: If the exact string is changed a parsing warning needs to be modified in agents.py + with pytest.raises(ValueError, match="The model output does not contain any JSON blob."): + parse_json_blob(raw_json) @pytest.mark.parametrize( "name,expected", From 4dfca234429f1821060b3d9b7239dc1e1e236de8 Mon Sep 17 00:00:00 2001 From: Surya Balakrishnan Date: Sun, 12 Oct 2025 05:50:48 +0000 Subject: [PATCH 5/8] make style --- tests/test_monitoring.py | 16 ++++------------ tests/test_utils.py | 3 +++ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py index df91baebf..8cbf85619 100644 --- a/tests/test_monitoring.py +++ b/tests/test_monitoring.py @@ -23,6 +23,7 @@ ToolCallingAgent, stream_to_gradio, ) +from smolagents.memory import ActionStep, AgentMemory from smolagents.models import ( ChatMessage, ChatMessageToolCall, @@ -31,16 +32,7 @@ Model, TokenUsage, ) - -from smolagents.memory import ( - AgentMemory, - ActionStep -) from smolagents.monitoring import AgentLogger -from smolagents.models import ( - ChatMessage, - MessageRole -) class FakeLLMModel(Model): @@ -196,14 +188,14 @@ def test_code_agent_metrics(agent_class): class ReplayTester(unittest.TestCase): def test_replay_with_chatmessage(self): - """ Regression test for dict(message) to message.dict() fix """ + """Regression test for dict(message) to message.dict() fix""" logger = AgentLogger() memory = AgentMemory(system_prompt="test") - step = ActionStep(step_number=1, timing = 0) + step = ActionStep(step_number=1, timing=0) step.model_input_messages = [ChatMessage(role=MessageRole.USER, content="Hello")] memory.steps.append(step) try: memory.replay(logger, detailed=True) except TypeError as e: - self.fail(f"Replay raised an error: {e}") \ No newline at end of file + self.fail(f"Replay raised an error: {e}") diff --git a/tests/test_utils.py b/tests/test_utils.py index 7acf7734c..c76e21772 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -142,6 +142,7 @@ def test_multiple_code_blobs(self): result = parse_code_blobs(test_input, ("", "")) assert result == "Foo\n\ncode_a\n\ncode_b" + @pytest.mark.parametrize( "raw_text", [ @@ -497,6 +498,7 @@ def test_parse_json_blob_with_invalid_json(raw_json): with pytest.raises(Exception): parse_json_blob(raw_json) + @pytest.mark.parametrize( "raw_json", [ @@ -511,6 +513,7 @@ def test_parse_json_blob_without_json_blob(raw_json): with pytest.raises(ValueError, match="The model output does not contain any JSON blob."): parse_json_blob(raw_json) + @pytest.mark.parametrize( "name,expected", [ From 4e630bfdcc125178604a0945e68e020d63ed9e2d Mon Sep 17 00:00:00 2001 From: Surya Balakrishnan Date: Mon, 20 Oct 2025 12:39:49 +0000 Subject: [PATCH 6/8] Add missing backticks to docs --- docs/source/en/tutorials/building_good_agents.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/en/tutorials/building_good_agents.md b/docs/source/en/tutorials/building_good_agents.md index d1f101163..98899210c 100644 --- a/docs/source/en/tutorials/building_good_agents.md +++ b/docs/source/en/tutorials/building_good_agents.md @@ -430,6 +430,7 @@ If you see warnings indicating that the Code or JSON data is missing in the outp For example in a CodeAgent's logs you should see ``` Warning: The model's output is missing a Code output. This could be due to an insufficient number of output tokens set in the model configuration. +``` and for a ToolCallingAgent ``` From a8357632619f247e1c43fb54ae99bd11265fb1ed Mon Sep 17 00:00:00 2001 From: Surya Balakrishnan Date: Fri, 24 Oct 2025 10:25:48 +0000 Subject: [PATCH 7/8] Remove documentation --- docs/source/en/tutorials/building_good_agents.md | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/docs/source/en/tutorials/building_good_agents.md b/docs/source/en/tutorials/building_good_agents.md index 98899210c..e7d2ce118 100644 --- a/docs/source/en/tutorials/building_good_agents.md +++ b/docs/source/en/tutorials/building_good_agents.md @@ -421,18 +421,4 @@ agent = CodeAgent( result = agent.run( "How long would a cheetah at full speed take to run the length of Pont Alexandre III?", ) -``` - -### 5. Increase maximum output tokens - -If you see warnings indicating that the Code or JSON data is missing in the output, It might be because the output tokens configured for your model might be too less. The model is not generating a code snippet or a tool call that can be parsed by the framework. Please try increasing the model output token configuration (`num_ctx`, `max_tokens` etc) - -For example in a CodeAgent's logs you should see -``` -Warning: The model's output is missing a Code output. This could be due to an insufficient number of output tokens set in the model configuration. -``` - -and for a ToolCallingAgent -``` -Warning: The model's output is missing JSON data. This could be due to an insufficient number of output tokens set in the model configuration -``` +``` \ No newline at end of file From 01fe39a42624687ad8e9998de4050a61202ce728 Mon Sep 17 00:00:00 2001 From: Surya Balakrishnan Date: Fri, 24 Oct 2025 10:26:40 +0000 Subject: [PATCH 8/8] Add newline --- docs/source/en/tutorials/building_good_agents.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/tutorials/building_good_agents.md b/docs/source/en/tutorials/building_good_agents.md index e7d2ce118..cbfcd1e32 100644 --- a/docs/source/en/tutorials/building_good_agents.md +++ b/docs/source/en/tutorials/building_good_agents.md @@ -421,4 +421,4 @@ agent = CodeAgent( result = agent.run( "How long would a cheetah at full speed take to run the length of Pont Alexandre III?", ) -``` \ No newline at end of file +```