Teknium
2a95791992
run_agent.py: delete the `# noqa: F401` re-export block (agent.process_bootstrap
OpenAI/_SafeWriter/_get_proxy_*, model_tools get_tool_definitions/
handle_function_call/check_toolset_requirements, FailoverReason,
_qwen_portal_headers/_routermint_headers, session_persistence names,
estimate_request_tokens_rough, ContextCompressor + friends, jittered_backoff,
prompt_builder names, message_sanitization names, tool_dispatch_helpers
names) — 41 names run_agent never used itself — and the `_STREAM_DIAG_HEADERS`
back-compat class alias (no in-tree reader). run_agent now imports only what
it uses (get_toolset_for_tool, is_local_endpoint, coalesce/uniquify tool-call
ids, cleanup_vm/get_active_env from terminal_tool_lifecycle).
agent/*: `_ra().X` late-binds that only reached a re-export now import the
defining module directly (agent_runtime_helpers -> process_bootstrap.OpenAI,
model_tools.handle_function_call, session_persistence._safe_session_filename_component;
agent_init -> model_tools.get_tool_definitions/check_toolset_requirements,
_lazy_headers("agent.client_lifecycle", ...) for qwen/routermint;
system_prompt -> agent.prompt_builder / model_tools directly, dropping its
own _ra() shim and the `_r` parameter threading). `_ra()` stays for
run_agent-resident names (logger, AIAgent, _hermes_home, _set_interrupt, ...).
toolsets.py: remove resolve_multiple_toolsets (shim-only, restored by
34abf954bd); tests/test_toolsets.py pins the same union behavior via
resolve_toolset over each name.
providers/__init__.py: drop the OMIT_TEMPERATURE re-export (no callers via the
package); ProviderProfile stays because __init__ uses it for annotations —
2 tests repointed to providers.base.
agent/iteration_budget.py: drop the "run_agent re-exports the class"
docstring pointer; 4 tests import IterationBudget from its home.
model_tools.py (arg_coercion names), agent/tool_executor.py, and
hermes_cli/cli_session_mixin.py repoints landed via a sibling commit on this
shared worktree.
Callers repointed: gateway/run.py, hermes_cli/cli_chat_turn_mixin.py,
hermes_cli/cli_tui_mixin.py, tui_gateway/session_workdir.py,
agent/transports/codex.py (one-line imports) + comment pointers in
tools/file_state.py, tools/schema_sanitizer.py, scripts/tool_search_livetest.py.
Tests: patch("run_agent.X") / monkeypatch.setattr(run_agent, "X") /
`from run_agent import X` -> defining module across 99 test files.
396 lines
15 KiB
Python
396 lines
15 KiB
Python
"""Empty-transcript sanitizer: stop the per-send heal/WARNING loop (#96870).
|
|
|
|
``repair_empty_non_final_messages`` is still the single owner of the
|
|
wire-copy substitution. The send-time projection now fills empty non-final
|
|
turns first so the owner heals 0 on the main loop (same pattern as #88955).
|
|
When a caller still hits the owner repeatedly, logging escalates once per
|
|
session window instead of flooding errors.log.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
|
|
import pytest
|
|
|
|
from agent.agent_runtime_helpers import (
|
|
_INTERRUPTED_PLACEHOLDER,
|
|
_empty_heal_log_state,
|
|
_empty_heal_pending_notice,
|
|
_empty_heal_user_notified,
|
|
consume_pending_sanitizer_heal_notice,
|
|
fill_empty_non_final_wire_payload,
|
|
get_sanitizer_heal_stats,
|
|
repair_empty_non_final_messages,
|
|
)
|
|
from hermes_logging import clear_session_context, set_session_context
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_heal_log():
|
|
_empty_heal_log_state.clear()
|
|
_empty_heal_pending_notice.clear()
|
|
_empty_heal_user_notified.clear()
|
|
clear_session_context()
|
|
yield
|
|
_empty_heal_log_state.clear()
|
|
_empty_heal_pending_notice.clear()
|
|
_empty_heal_user_notified.clear()
|
|
clear_session_context()
|
|
|
|
|
|
def _poisoned_rows():
|
|
return [
|
|
{"role": "user", "content": "hi"},
|
|
{"role": "assistant", "content": ""},
|
|
{"role": "user", "content": "next"},
|
|
]
|
|
|
|
|
|
class TestFillEmptyNonFinalWirePayload:
|
|
def test_fills_empty_non_final_assistant(self):
|
|
msg = {"role": "assistant", "content": ""}
|
|
assert fill_empty_non_final_wire_payload(msg, is_final=False) is True
|
|
assert msg["content"] == _INTERRUPTED_PLACEHOLDER
|
|
|
|
def test_fills_empty_non_final_user(self):
|
|
msg = {"role": "user", "content": None}
|
|
assert fill_empty_non_final_wire_payload(msg, is_final=False) is True
|
|
assert msg["content"] == _INTERRUPTED_PLACEHOLDER
|
|
|
|
def test_skips_final_turn(self):
|
|
msg = {"role": "assistant", "content": ""}
|
|
assert fill_empty_non_final_wire_payload(msg, is_final=True) is False
|
|
assert msg["content"] == ""
|
|
|
|
def test_skips_tool_call_turn(self):
|
|
msg = {
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [{"id": "c1", "function": {"name": "x", "arguments": "{}"}}],
|
|
}
|
|
assert fill_empty_non_final_wire_payload(msg, is_final=False) is False
|
|
assert msg["content"] == ""
|
|
|
|
def test_skips_codex_commentary_carrier(self):
|
|
msg = {
|
|
"role": "assistant",
|
|
"content": "",
|
|
"codex_message_items": [{"type": "text", "text": "hi"}],
|
|
}
|
|
assert fill_empty_non_final_wire_payload(msg, is_final=False) is False
|
|
assert msg["content"] == ""
|
|
|
|
def test_skips_already_populated_content(self):
|
|
msg = {"role": "assistant", "content": "hello"}
|
|
assert fill_empty_non_final_wire_payload(msg, is_final=False) is False
|
|
assert msg["content"] == "hello"
|
|
|
|
|
|
class TestHealLogEscalation:
|
|
def test_warning_then_one_error_then_silence(self, monkeypatch, caplog):
|
|
import agent.agent_runtime_helpers as arh
|
|
|
|
monkeypatch.setattr(arh, "_heal_escalation_threshold", lambda: 3)
|
|
set_session_context("sess-heal")
|
|
durable = _poisoned_rows()
|
|
|
|
with caplog.at_level(logging.WARNING, logger="run_agent"):
|
|
for _ in range(5):
|
|
out = repair_empty_non_final_messages(
|
|
[dict(m) for m in durable]
|
|
)
|
|
assert out[1]["content"] == _INTERRUPTED_PLACEHOLDER
|
|
assert durable[1]["content"] == ""
|
|
|
|
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
|
errors = [r for r in caplog.records if r.levelno == logging.ERROR]
|
|
assert len(warnings) == 2
|
|
assert len(errors) == 1
|
|
assert "healed" in warnings[0].getMessage()
|
|
assert "session window" in errors[0].getMessage()
|
|
assert "/new" in errors[0].getMessage()
|
|
|
|
def test_sessions_do_not_share_heal_counters(self, monkeypatch, caplog):
|
|
import agent.agent_runtime_helpers as arh
|
|
|
|
monkeypatch.setattr(arh, "_heal_escalation_threshold", lambda: 3)
|
|
with caplog.at_level(logging.WARNING, logger="run_agent"):
|
|
set_session_context("sess-a")
|
|
repair_empty_non_final_messages([dict(m) for m in _poisoned_rows()])
|
|
set_session_context("sess-b")
|
|
repair_empty_non_final_messages([dict(m) for m in _poisoned_rows()])
|
|
|
|
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
|
errors = [r for r in caplog.records if r.levelno == logging.ERROR]
|
|
assert len(warnings) == 2
|
|
assert errors == []
|
|
|
|
def test_owner_still_heals_wire_copy_only(self):
|
|
durable = _poisoned_rows()
|
|
out = repair_empty_non_final_messages(durable)
|
|
assert out[1]["content"] == _INTERRUPTED_PLACEHOLDER
|
|
assert durable[1]["content"] == ""
|
|
assert out is not durable
|
|
|
|
|
|
class TestOneTimeUserNotice:
|
|
def _heal_n(self, n):
|
|
for _ in range(n):
|
|
repair_empty_non_final_messages(
|
|
[dict(m) for m in _poisoned_rows()]
|
|
)
|
|
|
|
def test_notice_queued_once_at_threshold(self, monkeypatch):
|
|
import agent.agent_runtime_helpers as arh
|
|
|
|
monkeypatch.setattr(arh, "_heal_escalation_threshold", lambda: 3)
|
|
set_session_context("sess-notice")
|
|
|
|
self._heal_n(2)
|
|
assert consume_pending_sanitizer_heal_notice() is None
|
|
|
|
self._heal_n(1) # crosses threshold
|
|
notice = consume_pending_sanitizer_heal_notice()
|
|
assert notice is not None
|
|
assert "repeated repair" in notice
|
|
assert "/debug share" in notice
|
|
assert "hermes doctor" in notice
|
|
|
|
# drained: never delivered twice
|
|
assert consume_pending_sanitizer_heal_notice() is None
|
|
|
|
def test_notice_never_rearms_in_new_window(self, monkeypatch):
|
|
import agent.agent_runtime_helpers as arh
|
|
|
|
monkeypatch.setattr(arh, "_heal_escalation_threshold", lambda: 2)
|
|
set_session_context("sess-rearm")
|
|
|
|
self._heal_n(2)
|
|
assert consume_pending_sanitizer_heal_notice() is not None
|
|
|
|
# Simulate window expiry: force a fresh window, cross threshold again.
|
|
arh._empty_heal_log_state["sess-rearm"]["window_start"] -= (
|
|
arh._EMPTY_HEAL_WINDOW_S + 1
|
|
)
|
|
self._heal_n(2)
|
|
assert consume_pending_sanitizer_heal_notice() is None
|
|
|
|
def test_notice_scoped_to_session(self, monkeypatch):
|
|
import agent.agent_runtime_helpers as arh
|
|
|
|
monkeypatch.setattr(arh, "_heal_escalation_threshold", lambda: 2)
|
|
set_session_context("sess-x")
|
|
self._heal_n(2)
|
|
set_session_context("sess-y")
|
|
# sess-y never escalated; its consume must not steal sess-x's notice
|
|
assert consume_pending_sanitizer_heal_notice() is None
|
|
set_session_context("sess-x")
|
|
assert consume_pending_sanitizer_heal_notice() is not None
|
|
|
|
def test_threshold_zero_disables_escalation(self, monkeypatch, caplog):
|
|
import agent.agent_runtime_helpers as arh
|
|
|
|
monkeypatch.setattr(arh, "_heal_escalation_threshold", lambda: 0)
|
|
set_session_context("sess-off")
|
|
with caplog.at_level(logging.WARNING, logger="run_agent"):
|
|
self._heal_n(6)
|
|
assert consume_pending_sanitizer_heal_notice() is None
|
|
assert not [r for r in caplog.records if r.levelno == logging.ERROR]
|
|
|
|
def test_threshold_read_from_config(self, monkeypatch):
|
|
import agent.agent_runtime_helpers as arh
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.load_config_readonly",
|
|
lambda: {"agent": {"sanitizer_heal_escalation_threshold": 7}},
|
|
)
|
|
assert arh._heal_escalation_threshold() == 7
|
|
|
|
def test_threshold_defaults_when_config_unreadable(self, monkeypatch):
|
|
import agent.agent_runtime_helpers as arh
|
|
|
|
def _boom():
|
|
raise RuntimeError("no config")
|
|
|
|
monkeypatch.setattr("hermes_cli.config.load_config_readonly", _boom)
|
|
assert (
|
|
arh._heal_escalation_threshold() == arh._EMPTY_HEAL_ESCALATE_AFTER
|
|
)
|
|
|
|
|
|
class TestHealStatsSurface:
|
|
def test_counters_visible_and_escalation_flagged(self, monkeypatch):
|
|
import agent.agent_runtime_helpers as arh
|
|
|
|
monkeypatch.setattr(arh, "_heal_escalation_threshold", lambda: 3)
|
|
set_session_context("sess-stats")
|
|
for _ in range(4):
|
|
repair_empty_non_final_messages(
|
|
[dict(m) for m in _poisoned_rows()]
|
|
)
|
|
|
|
stats = get_sanitizer_heal_stats()
|
|
assert stats["sess-stats"]["heal_events"] == 4
|
|
assert stats["sess-stats"]["messages_healed"] == 4
|
|
assert stats["sess-stats"]["escalated"] is True
|
|
|
|
def test_debug_report_includes_heal_counters(self, monkeypatch):
|
|
import agent.agent_runtime_helpers as arh
|
|
from hermes_cli.debug import collect_debug_report, LogSnapshot
|
|
|
|
monkeypatch.setattr(arh, "_heal_escalation_threshold", lambda: 2)
|
|
set_session_context("sess-report")
|
|
for _ in range(2):
|
|
repair_empty_non_final_messages(
|
|
[dict(m) for m in _poisoned_rows()]
|
|
)
|
|
|
|
empty = LogSnapshot(path=None, tail_text="", full_text="")
|
|
report = collect_debug_report(
|
|
log_lines=5,
|
|
dump_text="dump",
|
|
log_snapshots={
|
|
k: empty
|
|
for k in ("agent", "errors", "gateway", "gui", "desktop")
|
|
},
|
|
)
|
|
assert "transcript sanitiser heal counters" in report
|
|
assert "sess-report: 2 heal events" in report
|
|
assert "escalated=True" in report
|
|
|
|
|
|
class TestProjectionStopsReheal:
|
|
def _loop_agent(self):
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from run_agent import AIAgent
|
|
|
|
with (
|
|
patch("model_tools.get_tool_definitions", return_value=[]),
|
|
patch("model_tools.check_toolset_requirements", return_value={}),
|
|
patch("agent.process_bootstrap.OpenAI"),
|
|
):
|
|
agent = AIAgent(
|
|
api_key="test-key-1234567890",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
quiet_mode=True,
|
|
skip_context_files=True,
|
|
skip_memory=True,
|
|
)
|
|
agent.client = MagicMock()
|
|
agent._cached_system_prompt = "You are helpful."
|
|
agent._use_prompt_caching = False
|
|
agent.tool_delay = 0
|
|
agent.compression_enabled = False
|
|
agent.save_trajectories = False
|
|
return agent
|
|
|
|
def test_unmarked_empty_assistant_is_filled_before_sanitizer(self):
|
|
"""The per-turn WARNING spam IS the bug: projection must fill the
|
|
unmarked empty row so the sanitizer heals 0 (#96870 / #88955)."""
|
|
from unittest.mock import patch
|
|
|
|
import agent.agent_runtime_helpers as _arh
|
|
from tests.run_agent.test_run_agent import _mock_response
|
|
|
|
agent = self._loop_agent()
|
|
agent.client.chat.completions.create.side_effect = [
|
|
_mock_response(content="ok", finish_reason="stop"),
|
|
]
|
|
sanitizer_healed = []
|
|
_real_repair = _arh.repair_empty_non_final_messages
|
|
|
|
def _spy_repair(messages, *a, **k):
|
|
empty = [
|
|
(m.get("role"), m.get("content"))
|
|
for m in messages
|
|
if isinstance(m, dict)
|
|
and m.get("role") in ("user", "assistant")
|
|
and not (m.get("content") or "").strip()
|
|
and not m.get("tool_calls")
|
|
]
|
|
sanitizer_healed.append(empty)
|
|
return _real_repair(messages, *a, **k)
|
|
|
|
history = [
|
|
{"role": "user", "content": "start"},
|
|
{"role": "assistant", "content": ""},
|
|
{"role": "user", "content": "continue", "finish_reason": "stop"},
|
|
{"role": "assistant", "content": "earlier reply", "finish_reason": "stop"},
|
|
]
|
|
|
|
with (
|
|
patch.object(agent, "_flush_messages_to_session_db"),
|
|
patch.object(agent, "_persist_session"),
|
|
patch.object(agent, "_save_trajectory"),
|
|
patch.object(agent, "_cleanup_task_resources"),
|
|
patch.object(
|
|
_arh, "repair_empty_non_final_messages", side_effect=_spy_repair
|
|
),
|
|
):
|
|
agent.run_conversation("next question", conversation_history=history)
|
|
|
|
assert sanitizer_healed, "sanitizer was never invoked — test is vacuous"
|
|
for empty in sanitizer_healed:
|
|
assert empty == [], (
|
|
"sanitizer still received an empty non-final row — "
|
|
"the re-heal loop is back (#96870)"
|
|
)
|
|
|
|
wire = agent.client.chat.completions.create.call_args.kwargs["messages"]
|
|
wire_assistants = [m for m in wire if m.get("role") == "assistant"]
|
|
assert wire_assistants[0]["content"] == _INTERRUPTED_PLACEHOLDER
|
|
assert history[1]["content"] == ""
|
|
assert "api_content" not in history[1]
|
|
|
|
def test_pending_notice_delivered_out_of_band_not_in_context(self):
|
|
"""A queued escalation notice is emitted through _emit_warning (the
|
|
status/delivery channel) and NEVER appears in the wire messages or
|
|
the durable history — message-flow / caching invariants (#96870)."""
|
|
from unittest.mock import patch
|
|
|
|
import agent.agent_runtime_helpers as _arh
|
|
from tests.run_agent.test_run_agent import _mock_response
|
|
|
|
agent = self._loop_agent()
|
|
agent.client.chat.completions.create.side_effect = [
|
|
_mock_response(content="ok", finish_reason="stop"),
|
|
]
|
|
|
|
set_session_context("sess-loop-notice")
|
|
# The turn re-binds the log session context to the agent's own
|
|
# session id at turn start, so queue the notice under that key —
|
|
# exactly where the escalation path would have put it mid-session.
|
|
_live_key = str(getattr(agent, "session_id", None) or "-")
|
|
with _arh._empty_heal_log_lock:
|
|
_arh._empty_heal_user_notified.add(_live_key)
|
|
_arh._empty_heal_pending_notice[_live_key] = (
|
|
"⚠️ Your session transcript required repeated repair — "
|
|
"run /debug share or `hermes doctor`."
|
|
)
|
|
|
|
warned = []
|
|
history = [
|
|
{"role": "user", "content": "start"},
|
|
{"role": "assistant", "content": "earlier", "finish_reason": "stop"},
|
|
]
|
|
with (
|
|
patch.object(agent, "_flush_messages_to_session_db"),
|
|
patch.object(agent, "_persist_session"),
|
|
patch.object(agent, "_save_trajectory"),
|
|
patch.object(agent, "_cleanup_task_resources"),
|
|
patch.object(agent, "_emit_warning", side_effect=warned.append),
|
|
):
|
|
agent.run_conversation("next", conversation_history=history)
|
|
|
|
assert warned and "repeated repair" in warned[0]
|
|
# one-time: drained after delivery
|
|
assert _arh._empty_heal_pending_notice == {}
|
|
# never injected into the wire copy or durable history
|
|
wire = agent.client.chat.completions.create.call_args.kwargs["messages"]
|
|
assert all("repeated repair" not in str(m.get("content")) for m in wire)
|
|
assert all(
|
|
"repeated repair" not in str(m.get("content")) for m in history
|
|
)
|