From 8e4366d358bd93fd799fc22082e10ede69a21c6f Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 07:02:55 -0700 Subject: [PATCH] fix(tools): freeze tools[] across agent-cache eviction; make /reload-mcp the re-probe hatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Policy: availability-gated tools (check_fn probes — Docker, HASS_TOKEN, OAuth…) are frozen for the life of a session. tools[] only changes on /new, /reload-mcp, or compaction. Two doors remained after #100638: * Gateway agent-cache eviction (LRU/idle sweep/cross-process invalidation) rebuilds a fresh AIAgent for the SAME session and agent_init re-derives agent.tools from live probes with no predecessor to preserve. Persist the session's resolved tool-name order in a new `sessions.tool_names` JSON column (declarative reconciliation, SCHEMA_VERSION 28), written alongside the system prompt and re-pinned on every published refresh (so /reload-mcp and compaction naturally reset it; /new mints a new row). On restore-for-existing-session the fresh definitions are folded onto the saved order via the SAME `_merge_preserving_prefix` helper — a probe-flipped tool is carried forward from the registry schema, a deregistered one dropped, new tools appended at the tail. * /reload-mcp (CLI, gateway, TUI RPC) now also calls `reprobe_tool_availability()` — drops the check_fn verdict cache and the get_tool_definitions memo — so a user can consciously pick up a credential/daemon that appeared mid-session. Docs updated. --- agent/conversation_loop.py | 15 ++++ cli.py | 6 +- gateway/run.py | 4 +- hermes_state.py | 19 +++++ hermes_state_common.py | 3 +- tests/tools/test_refresh_agent_mcp_tools.py | 51 ++++++++++++++ tools/mcp_tool.py | 78 ++++++++++++++++++++- tui_gateway/methods_tools.py | 3 +- website/docs/reference/slash-commands.md | 4 +- website/docs/user-guide/features/mcp.md | 2 +- 10 files changed, 177 insertions(+), 8 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index 6c156523e9..76d944a961 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -989,6 +989,7 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) """ stored_prompt = None stored_state = "missing" + session_row = None if conversation_history and agent._session_db: try: session_row = agent._session_db.get_session(agent.session_id) @@ -1091,6 +1092,17 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) # Continuing session — reuse the exact system prompt from the # previous turn so the Anthropic cache prefix matches. agent._cached_system_prompt = stored_prompt + # Same contract for tools[]: a fresh AIAgent for an existing session + # (gateway agent-cache eviction) re-probed every check_fn, so pin the + # array back to the order this session already sent (tools freeze). + try: + saved_tools = session_row.get("tool_names") if session_row else None + if saved_tools: + from tools.mcp_tool import restore_agent_tool_prefix + + restore_agent_tool_prefix(agent, json.loads(saved_tools)) + except Exception: + logger.debug("tool prefix restore skipped", exc_info=True) # Prompt-section callbacks are new-session-only. Recover their frozen # bytes from the persisted full prompt so a later compression rebuild # keeps them without evaluating plugin state in this resumed process. @@ -1173,6 +1185,9 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) if agent._session_db: try: agent._session_db.update_system_prompt(agent.session_id, agent._cached_system_prompt) + from tools.mcp_tool import persist_agent_tool_names + + persist_agent_tool_names(agent) except Exception as exc: logger.warning( "Session DB update_system_prompt failed for session %s: " diff --git a/cli.py b/cli.py index bc8c66c7fe..ab2f2d3d7f 100644 --- a/cli.py +++ b/cli.py @@ -14764,7 +14764,9 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): sees the updated tools on the next turn. """ try: - from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools, _servers, _lock + from tools.mcp_tool import ( + shutdown_mcp_servers, discover_mcp_tools, reprobe_tool_availability, _servers, _lock, + ) # Capture old server names with _lock: @@ -14776,6 +14778,8 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin): # Shutdown existing connections shutdown_mcp_servers() + # Explicit reload also re-probes tool availability (check_fn). + reprobe_tool_availability() # Reconnect (reads config.yaml fresh) new_tools = discover_mcp_tools() diff --git a/gateway/run.py b/gateway/run.py index b0b799f385..392433f280 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -26230,7 +26230,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew return await self._execute_mcp_reload(event) try: from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools, _servers, _lock - from tools.mcp_tool import _server_scope_keys + from tools.mcp_tool import _server_scope_keys, reprobe_tool_availability from tools.registry import registry reload_scope = registry.current_scope_key() if multiplex else None @@ -26250,6 +26250,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew await self._run_in_executor_with_context( lambda: shutdown_mcp_servers(scope=reload_scope) ) + # Explicit reload also re-probes tool availability (check_fn). + reprobe_tool_availability() # Reconnect by discovering tools (reads config.yaml fresh) new_tools = await self._run_in_executor_with_context(discover_mcp_tools) diff --git a/hermes_state.py b/hermes_state.py index eb308a1e5b..06d7b19233 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -9741,6 +9741,25 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._delete_unreferenced_system_prompts(conn) self._execute_write(_do) + def update_session_tool_names( + self, session_id: str, tool_names: Optional[List[str]] + ) -> None: + """Persist the session's resolved ``tools[]`` name order (JSON array). + + Read back by ``tools.mcp_tool.restore_agent_tool_prefix`` when a fresh + ``AIAgent`` is rebuilt for an existing session (gateway agent-cache + eviction) so a flipped ``check_fn`` verdict can't fork the cached tool + prefix. ``None`` clears the pin. + """ + payload = json.dumps(list(tool_names)) if tool_names is not None else None + + def _do(conn): + conn.execute( + "UPDATE sessions SET tool_names = ? WHERE id = ?", + (payload, session_id), + ) + self._execute_write(_do) + def update_session_model( self, session_id: str, model: str, provider: Optional[str] = None ) -> None: diff --git a/hermes_state_common.py b/hermes_state_common.py index ae9e63af06..af12d322d3 100644 --- a/hermes_state_common.py +++ b/hermes_state_common.py @@ -354,7 +354,7 @@ def _sql_session_last_active_by_id(session_id_expr: str) -> str: ) -SCHEMA_VERSION = 27 +SCHEMA_VERSION = 28 # FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the @@ -452,6 +452,7 @@ CREATE TABLE IF NOT EXISTS sessions ( pinned INTEGER NOT NULL DEFAULT 0, hidden INTEGER NOT NULL DEFAULT 0, last_read_at REAL, + tool_names TEXT, FOREIGN KEY (parent_session_id) REFERENCES sessions(id), FOREIGN KEY (system_prompt_hash) REFERENCES system_prompts(hash) ); diff --git a/tests/tools/test_refresh_agent_mcp_tools.py b/tests/tools/test_refresh_agent_mcp_tools.py index 22203d93af..ee88a58ff9 100644 --- a/tests/tools/test_refresh_agent_mcp_tools.py +++ b/tests/tools/test_refresh_agent_mcp_tools.py @@ -286,3 +286,54 @@ def test_preserve_prefix_appends_late_arrivals_at_the_tail(monkeypatch): assert [t["function"]["name"] for t in agent.tools] == [ "read_file", "terminal", "aaa_mcp_late", ] + + +# --------------------------------------------------------------------------- +# tools[] freeze: eviction rebuild + the /reload-mcp re-probe hatch +# --------------------------------------------------------------------------- + + +def test_eviction_rebuild_restores_the_sessions_saved_tool_order(monkeypatch): + """A fresh AIAgent for an EXISTING session must keep the saved tools[] pin. + + Gateway agent-cache eviction rebuilds the agent; ``agent_init`` re-probes + every ``check_fn`` and ``browser_navigate``'s flips false. The persisted + name list stands in for the missing predecessor: the tool is carried + forward from the registry schema, byte-for-byte in its old slot. + """ + from tools import registry as registry_mod + + saved = ["read_file", "browser_navigate", "terminal"] + entries = {n: types.SimpleNamespace(name=n, schema=_tool(n)["function"]) for n in saved} + monkeypatch.setattr(registry_mod.registry, "get_all_entries", lambda: list(entries.values()), raising=False) + monkeypatch.setattr(registry_mod.registry, "get_entry", lambda name, **kw: entries.get(name), raising=False) + + rebuilt = _agent(["read_file", "terminal"]) # probe flipped: browser_navigate gone + changed = mcp_tool.restore_agent_tool_prefix(rebuilt, saved) + + assert changed is True + assert [t["function"]["name"] for t in rebuilt.tools] == saved + assert rebuilt.valid_tool_names == set(saved) + + +def test_reprobe_tool_availability_drops_cached_check_fn_verdicts(monkeypatch): + """/reload-mcp is the explicit hatch: a cached False must be re-probed.""" + from tools import registry as registry_mod + import model_tools + + verdict = {"ok": False} + + def probe(): + return verdict["ok"] + + monkeypatch.setattr(registry_mod, "check_fn_cache_scope", lambda: "test-scope") + assert registry_mod._check_fn_cached(probe) is False + verdict["ok"] = True + assert registry_mod._check_fn_cached(probe) is False # TTL cache replays stale verdict + with model_tools._tool_defs_cache_lock: + model_tools._tool_defs_cache[("sentinel",)] = [] + + mcp_tool.reprobe_tool_availability() + + assert registry_mod._check_fn_cached(probe) is True + assert ("sentinel",) not in model_tools._tool_defs_cache diff --git a/tools/mcp_tool.py b/tools/mcp_tool.py index cc69786304..01e7d9b3db 100644 --- a/tools/mcp_tool.py +++ b/tools/mcp_tool.py @@ -8509,7 +8509,83 @@ def refresh_agent_mcp_tools( engine_names.clear() engine_names.update(staged_engine_names) agent._tool_snapshot_generation = max(published_gen, snapshot_generation) - return new_names - current + added = new_names - current + # Every published snapshot re-pins the session's tool order so a later + # rebuild-for-existing-session (gateway agent-cache eviction) restores + # exactly these names — see ``restore_agent_tool_prefix``. + persist_agent_tool_names(agent) + return added + + +def reprobe_tool_availability() -> None: + """Explicit ``/reload-mcp`` hatch out of the tools[] freeze. + + Availability-gated tools (``check_fn``: Docker, HASS_TOKEN, OAuth…) are + frozen for the life of a session; a credential or daemon that appears + mid-session is only picked up when the user consciously asks. Drop the + ``check_fn`` verdict cache AND the ``get_tool_definitions`` memo (keyed on + registry generation, so it would otherwise replay the stale verdicts). + """ + from model_tools import _clear_tool_defs_cache + from tools.registry import invalidate_check_fn_cache + + invalidate_check_fn_cache() + _clear_tool_defs_cache() + + +def persist_agent_tool_names(agent) -> None: + """Best-effort: write ``agent.tools`` names to the session row (freeze pin).""" + db = getattr(agent, "_session_db", None) + session_id = getattr(agent, "session_id", None) + if not db or not session_id: + return + try: + db.update_session_tool_names( + session_id, + [t["function"]["name"] for t in (getattr(agent, "tools", None) or [])], + ) + except Exception: # noqa: BLE001 + logger.debug("tool_names persist skipped", exc_info=True) + + +def restore_agent_tool_prefix(agent, saved_names: list) -> bool: + """Fold a freshly built agent's ``tools`` onto the session's saved order. + + Closes the second door on the tools[] freeze: the gateway rebuilds a NEW + ``AIAgent`` for an existing session after agent-cache eviction, and + ``agent_init`` re-derives ``agent.tools`` from live ``check_fn`` probes + with no predecessor to preserve. The saved name list stands in for that + predecessor: a saved tool that is still registered but failed its probe + this time is carried forward from the registry's schema, a deregistered + one is dropped, and genuinely new tools append at the tail — the same + ``_merge_preserving_prefix`` rule the between-turns refresh uses. + Returns True when the snapshot was changed. + """ + if not saved_names: + return False + from tools.registry import registry + + fresh_defs = list(getattr(agent, "tools", None) or []) + fresh = {t["function"]["name"]: t for t in fresh_defs} + saved_defs = [] + for name in saved_names: + entry_def = fresh.get(name) + if entry_def is None: + entry = registry.get_entry(name) + if entry is None: + continue + entry_def = {"type": "function", "function": {**entry.schema, "name": entry.name}} + saved_defs.append(entry_def) + registered_names = {entry.name for entry in registry.get_all_entries()} + merged, merged_names = _merge_preserving_prefix(saved_defs, fresh_defs, registered_names) + with _agent_tools_lock: + if merged == fresh_defs: + return False + agent.tools = merged + agent.valid_tool_names = merged_names + if [t["function"]["name"] for t in merged] != list(saved_names): + persist_agent_tool_names(agent) + return True def _merge_preserving_prefix( diff --git a/tui_gateway/methods_tools.py b/tui_gateway/methods_tools.py index fb9fd3fdcd..e217ae5bc3 100644 --- a/tui_gateway/methods_tools.py +++ b/tui_gateway/methods_tools.py @@ -133,7 +133,7 @@ def _(rid, params: dict) -> dict: return _err(rid, 5019, f"compute-host reload_mcp failed: {exc}") return _ok(rid, {"status": "reloaded", "turn_isolation": True, "host_ack": ack}) - from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools + from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools, reprobe_tool_availability def _refresh_session_agent() -> None: """Rebuild THIS session's cached tool snapshot from the live @@ -184,6 +184,7 @@ def _(rid, params: dict) -> dict: loaded = _compute_mcp_rev() for _ in range(_MCP_RELOAD_MAX_PASSES): shutdown_mcp_servers() + reprobe_tool_availability() discover_mcp_tools() after = _compute_mcp_rev() if after == loaded: diff --git a/website/docs/reference/slash-commands.md b/website/docs/reference/slash-commands.md index ef939b4699..0bcdcb52cb 100644 --- a/website/docs/reference/slash-commands.md +++ b/website/docs/reference/slash-commands.md @@ -115,7 +115,7 @@ Type `/` in the CLI to open the autocomplete menu. Built-in commands are case-in | `/blueprint [name] [slot=value ...]` (alias: `/bp`) | Set up an automation from a blueprint template. Bare `/blueprint` lists the catalog; `/blueprint ` starts a guided slot-filling flow on the next agent turn; `/blueprint slot=value ...` creates the job directly. | | `/curator` | Background skill maintenance — `status`, `run`, `pin`, `archive`. See [Curator](/user-guide/features/curator). | | `/kanban ` | Drive the multi-profile, multi-project collaboration board without leaving chat. Full `hermes kanban` surface is available: `/kanban list`, `/kanban show t_abc`, `/kanban create "title" --assignee X`, `/kanban comment t_abc "text"`, `/kanban unblock t_abc`, `/kanban dispatch`, etc. Multi-board support included: `/kanban boards list`, `/kanban boards create `, `/kanban boards switch `, `/kanban --board `. See [Kanban slash command](/user-guide/features/kanban#kanban-slash-command). | -| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config.yaml | +| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config.yaml and re-probe tool availability (credentials/daemons that appeared mid-session) | | `/reload-skills` (alias: `/reload_skills`) | Re-scan `~/.hermes/skills/` for newly installed or removed skills | | `/reload` | Reload `.env` variables into the running session (picks up new API keys without restarting) | | `/plugins` | List installed plugins and their status | @@ -291,7 +291,7 @@ The messaging gateway supports the following built-in commands inside Telegram, | `/skills [pending\|approve\|reject\|diff\|approval]` | Review pending **skill** writes staged by the write-approval gate (`skills.write_approval`). Shows a one-line gist per staged write; `/skills diff ` is truncated for chat — read the full diff on the CLI or in `~/.hermes/pending/skills/.json`. Only appears when the gate is on (or staged writes remain); search/install stay CLI-only. | | `/kanban ` | Drive the multi-profile, multi-project collaboration board from chat — identical argument surface to the CLI. Bypasses the running-agent guard, so `/kanban unblock t_abc`, `/kanban comment t_abc "…"`, `/kanban list --mine`, `/kanban boards switch `, etc. work mid-turn. `/kanban create …` auto-subscribes the originating chat to the new task's terminal events. See [Kanban slash command](/user-guide/features/kanban#kanban-slash-command). | | `/platform [name]` | Operate a running gateway platform right from chat. `/platform list` shows every adapter and its state (running, paused-by-breaker, manually-paused); `/platform pause ` stops dispatching new messages to that adapter without unloading it; `/platform resume ` re-enables it and clears a tripped circuit breaker once the upstream is healthy. | -| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config. | +| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config and re-probe tool availability. | | `/verbose` | Cycle tool progress display. **Off by default on messaging** — enable with `display.tool_progress_command: true` in `config.yaml`. | | `/yolo` | Toggle YOLO mode — skip all dangerous command approval prompts. | | `/commands [page]` | Browse all commands and skills (paginated). | diff --git a/website/docs/user-guide/features/mcp.md b/website/docs/user-guide/features/mcp.md index 3d2e82a85f..a3fe5f0802 100644 --- a/website/docs/user-guide/features/mcp.md +++ b/website/docs/user-guide/features/mcp.md @@ -644,7 +644,7 @@ If you change MCP config, use: /reload-mcp ``` -This reloads MCP servers from config and refreshes the available tool list. For runtime tool changes pushed by the server itself, see [Dynamic Tool Discovery](#dynamic-tool-discovery) above. +This reloads MCP servers from config and refreshes the available tool list. It is also the explicit way to re-probe availability-gated tools (Docker, `HASS_TOKEN`, OAuth…): a session's tool set is otherwise frozen, so a credential or daemon that appears mid-session is only picked up on `/reload-mcp`, `/new`, or context compaction. For runtime tool changes pushed by the server itself, see [Dynamic Tool Discovery](#dynamic-tool-discovery) above. ### Toolsets