Improve error messaging for short output token lengths

docs/source/en/tutorials/building_good_agents.md

@@ -422,3 +422,17 @@ result = agent.run(
     "How long would a cheetah at full speed take to run the length of Pont Alexandre III?",
 )
 ```
+
+### 5. Increase maximum output tokens
+
+If you see warnings indicating that the Code or JSON data is missing in the output, It might be because the output tokens configured for your model might be too less. The model is not generating a code snippet or a tool call that can be parsed by the framework. Please try increasing the model output token configuration (`num_ctx`, `max_tokens` etc)
+
+For example in a CodeAgent's logs you should see
+```
+Warning: The model's output is missing a Code output. This could be due to an insufficient number of output tokens set in the model configuration.
+```
+
+and for a ToolCallingAgent
+```
+Warning: The model's output is missing JSON data. This could be due to an insufficient number of output tokens set in the model configuration
+```

src/smolagents/agents.py

@@ -1302,6 +1302,10 @@ def _step_stream(
             try:
                 chat_message = self.model.parse_tool_calls(chat_message)
             except Exception as e:
+                if isinstance(e, ValueError) and "The model output does not contain any JSON blob" in str(e):
+                    self.logger.log_warning(
+                        "The model's output is missing JSON data. This could be due to an insufficient number of output tokens set in the model configuration."
+                    )
                 raise AgentParsingError(f"Error while parsing tool call from model output: {e}", self.logger)
         else:
             for tool_call in chat_message.tool_calls:
@@ -1682,6 +1686,10 @@ def _step_stream(
             code_action = fix_final_answer_code(code_action)
             memory_step.code_action = code_action
         except Exception as e:
+            if isinstance(e, ValueError) and "Make sure to include code with the correct pattern" in str(e):
+                self.logger.log_warning(
+                    "The model's output is missing a Code output. This could be due to an insufficient number of output tokens set in the model configuration."
+                )
             error_msg = f"Error in code parsing:\n{e}\nMake sure to provide correct code blobs."
             raise AgentParsingError(error_msg, self.logger)
 

src/smolagents/monitoring.py

@@ -187,6 +187,9 @@ def log_code(self, title: str, content: str, level: int = LogLevel.INFO) -> None
             level=level,
         )
 
+    def log_warning(self, title: str, level: int = LogLevel.INFO) -> None:
+        self.log("Warning: " + title, style=YELLOW_HEX, level=LogLevel.INFO)
+
     def log_rule(self, title: str, level: int = LogLevel.INFO) -> None:
         self.log(
             Rule(

tests/test_monitoring.py

@@ -23,6 +23,7 @@
     ToolCallingAgent,
     stream_to_gradio,
 )
+from smolagents.memory import ActionStep, AgentMemory
 from smolagents.models import (
     ChatMessage,
     ChatMessageToolCall,
@@ -31,16 +32,7 @@
     Model,
     TokenUsage,
 )
-
-from smolagents.memory import (
-    AgentMemory, 
-    ActionStep
-)
 from smolagents.monitoring import AgentLogger
-from smolagents.models import (
-    ChatMessage, 
-    MessageRole
-)
 
 
 class FakeLLMModel(Model):
@@ -196,14 +188,14 @@ def test_code_agent_metrics(agent_class):
 
 class ReplayTester(unittest.TestCase):
     def test_replay_with_chatmessage(self):
-        """ Regression test for dict(message) to message.dict() fix """
+        """Regression test for dict(message) to message.dict() fix"""
         logger = AgentLogger()
         memory = AgentMemory(system_prompt="test")
-        step = ActionStep(step_number=1, timing = 0)
+        step = ActionStep(step_number=1, timing=0)
         step.model_input_messages = [ChatMessage(role=MessageRole.USER, content="Hello")]
         memory.steps.append(step)
 
         try:
             memory.replay(logger, detailed=True)
         except TypeError as e:
-            self.fail(f"Replay raised an error: {e}")
+            self.fail(f"Replay raised an error: {e}")

tests/test_utils.py

@@ -143,6 +143,19 @@ def test_multiple_code_blobs(self):
         assert result == "Foo\n\ncode_a\n\ncode_b"
 
 
+@pytest.mark.parametrize(
+    "raw_text",
+    [
+        "This is just some conversational text.",
+        "Here is an invalid code snippet `x = 10",
+    ],
+)
+def test_parse_code_blobs_without_valid_code(raw_text):
+    # Note: If the exact string is changed a parsing warning needs to be modified in agents.py
+    with pytest.raises(ValueError, match="Make sure to include code with the correct pattern"):
+        parse_code_blobs(raw_text, ("<code>", "</code>"))
+
+
 @pytest.fixture(scope="function")
 def ipython_shell():
     """Reset IPython shell before and after each test."""
@@ -486,6 +499,21 @@ def test_parse_json_blob_with_invalid_json(raw_json):
         parse_json_blob(raw_json)
 
 
+@pytest.mark.parametrize(
+    "raw_json",
+    [
+        "this string has no json blob",
+        "this string has an opening brace { but no closing one",
+        "",
+        "some text {",
+    ],
+)
+def test_parse_json_blob_without_json_blob(raw_json):
+    # Note: If the exact string is changed a parsing warning needs to be modified in agents.py
+    with pytest.raises(ValueError, match="The model output does not contain any JSON blob."):
+        parse_json_blob(raw_json)
+
+
 @pytest.mark.parametrize(
     "name,expected",
     [