diff --git a/EvoScientist/stream/formatter.py b/EvoScientist/stream/formatter.py index f0fdf5e..8035482 100644 --- a/EvoScientist/stream/formatter.py +++ b/EvoScientist/stream/formatter.py @@ -109,12 +109,15 @@ class ToolResultFormatter: return False def _is_error(self, content: str) -> bool: + head = "\n".join(content.splitlines()[:3]) error_patterns = [ 'Traceback (most recent call last)', 'Exception:', 'Error:', + 'Error invoking tool', + 'Failed ', ] - return any(pattern in content for pattern in error_patterns) + return any(pattern in head for pattern in error_patterns) def _is_markdown(self, content: str) -> bool: md_patterns = ['```', '**', '##', '- **'] diff --git a/EvoScientist/stream/utils.py b/EvoScientist/stream/utils.py index 4244a3d..e125303 100644 --- a/EvoScientist/stream/utils.py +++ b/EvoScientist/stream/utils.py @@ -64,18 +64,27 @@ def has_args(args) -> bool: def is_success(content: str) -> bool: - """Determine if tool output indicates successful execution.""" + """Determine if tool output indicates successful execution. + + Only checks the first few lines of output for error patterns to avoid + false positives from code/docs that contain "Error:" as literal text. + """ content = content.strip() if content.startswith(SUCCESS_PREFIX): return True if content.startswith(FAILURE_PREFIX): return False + # Only check the first 3 lines — real tool errors appear at the top, + # not buried deep inside file content or command output. + head = "\n".join(content.splitlines()[:3]) error_patterns = [ 'Traceback (most recent call last)', 'Exception:', 'Error:', + 'Error invoking tool', + 'Failed ', ] - return not any(pattern in content for pattern in error_patterns) + return not any(pattern in head for pattern in error_patterns) def truncate(content: str, max_length: int, suffix: str = "\n... (truncated)") -> str: diff --git a/tests/test_stream_utils.py b/tests/test_stream_utils.py index edc2f0d..f67cb5b 100644 --- a/tests/test_stream_utils.py +++ b/tests/test_stream_utils.py @@ -35,6 +35,31 @@ class TestIsSuccess: def test_whitespace_stripped(self): assert is_success(" [OK] with spaces ") is True + def test_error_in_code_content_not_false_positive(self): + # read_file returning code with "Error:" deep inside should be success + content = '#!/usr/bin/env python3\n"""\nSkill Packager\n"""\n\nprint(f"Error: not found")' + assert is_success(content) is True + + def test_error_on_line4_not_false_positive(self): + content = "line1\nline2\nline3\nError: buried deep\nline5" + assert is_success(content) is True + + def test_error_on_first_line(self): + assert is_success("Error: file not found\nsome detail") is False + + def test_error_invoking_tool(self): + assert is_success("Error invoking tool 'write_file'") is False + + def test_failed_to_uninstall(self): + assert is_success("Failed to uninstall skill: Skill not found: latex-paper-en") is False + + def test_failed_to_install(self): + assert is_success("Failed to install skill: git clone failed: ...") is False + + def test_failed_in_code_content_not_false_positive(self): + content = '#!/usr/bin/env python3\n"""Helper"""\n\nif x:\n print("Failed to connect")' + assert is_success(content) is True + # === format_tool_compact ===