diff --git a/src/smolagents/agents.py b/src/smolagents/agents.py index 8b1f8dab9..299e073b0 100644 --- a/src/smolagents/agents.py +++ b/src/smolagents/agents.py @@ -1302,6 +1302,10 @@ def _step_stream( try: chat_message = self.model.parse_tool_calls(chat_message) except Exception as e: + if isinstance(e, ValueError) and "The model output does not contain any JSON blob" in str(e): + self.logger.log_warning( + "The model's output is missing JSON data. This could be due to an insufficient number of output tokens set in the model configuration." + ) raise AgentParsingError(f"Error while parsing tool call from model output: {e}", self.logger) else: for tool_call in chat_message.tool_calls: @@ -1682,6 +1686,10 @@ def _step_stream( code_action = fix_final_answer_code(code_action) memory_step.code_action = code_action except Exception as e: + if isinstance(e, ValueError) and "Make sure to include code with the correct pattern" in str(e): + self.logger.log_warning( + "The model's output is missing a Code output. This could be due to an insufficient number of output tokens set in the model configuration." + ) error_msg = f"Error in code parsing:\n{e}\nMake sure to provide correct code blobs." raise AgentParsingError(error_msg, self.logger) diff --git a/src/smolagents/monitoring.py b/src/smolagents/monitoring.py index e264e8472..62edaaf37 100644 --- a/src/smolagents/monitoring.py +++ b/src/smolagents/monitoring.py @@ -187,6 +187,9 @@ def log_code(self, title: str, content: str, level: int = LogLevel.INFO) -> None level=level, ) + def log_warning(self, title: str, level: int = LogLevel.INFO) -> None: + self.log("Warning: " + title, style=YELLOW_HEX, level=LogLevel.INFO) + def log_rule(self, title: str, level: int = LogLevel.INFO) -> None: self.log( Rule( diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py index df91baebf..8cbf85619 100644 --- a/tests/test_monitoring.py +++ b/tests/test_monitoring.py @@ -23,6 +23,7 @@ ToolCallingAgent, stream_to_gradio, ) +from smolagents.memory import ActionStep, AgentMemory from smolagents.models import ( ChatMessage, ChatMessageToolCall, @@ -31,16 +32,7 @@ Model, TokenUsage, ) - -from smolagents.memory import ( - AgentMemory, - ActionStep -) from smolagents.monitoring import AgentLogger -from smolagents.models import ( - ChatMessage, - MessageRole -) class FakeLLMModel(Model): @@ -196,14 +188,14 @@ def test_code_agent_metrics(agent_class): class ReplayTester(unittest.TestCase): def test_replay_with_chatmessage(self): - """ Regression test for dict(message) to message.dict() fix """ + """Regression test for dict(message) to message.dict() fix""" logger = AgentLogger() memory = AgentMemory(system_prompt="test") - step = ActionStep(step_number=1, timing = 0) + step = ActionStep(step_number=1, timing=0) step.model_input_messages = [ChatMessage(role=MessageRole.USER, content="Hello")] memory.steps.append(step) try: memory.replay(logger, detailed=True) except TypeError as e: - self.fail(f"Replay raised an error: {e}") \ No newline at end of file + self.fail(f"Replay raised an error: {e}") diff --git a/tests/test_utils.py b/tests/test_utils.py index 740ee3943..c76e21772 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -143,6 +143,19 @@ def test_multiple_code_blobs(self): assert result == "Foo\n\ncode_a\n\ncode_b" +@pytest.mark.parametrize( + "raw_text", + [ + "This is just some conversational text.", + "Here is an invalid code snippet `x = 10", + ], +) +def test_parse_code_blobs_without_valid_code(raw_text): + # Note: If the exact string is changed a parsing warning needs to be modified in agents.py + with pytest.raises(ValueError, match="Make sure to include code with the correct pattern"): + parse_code_blobs(raw_text, ("", "")) + + @pytest.fixture(scope="function") def ipython_shell(): """Reset IPython shell before and after each test.""" @@ -486,6 +499,21 @@ def test_parse_json_blob_with_invalid_json(raw_json): parse_json_blob(raw_json) +@pytest.mark.parametrize( + "raw_json", + [ + "this string has no json blob", + "this string has an opening brace { but no closing one", + "", + "some text {", + ], +) +def test_parse_json_blob_without_json_blob(raw_json): + # Note: If the exact string is changed a parsing warning needs to be modified in agents.py + with pytest.raises(ValueError, match="The model output does not contain any JSON blob."): + parse_json_blob(raw_json) + + @pytest.mark.parametrize( "name,expected", [