fix(tools): freeze tools[] across agent-cache eviction; make /reload-mcp the re-probe hatch
Policy: availability-gated tools (check_fn probes — Docker, HASS_TOKEN, OAuth…) are frozen for the life of a session. tools[] only changes on /new, /reload-mcp, or compaction. Two doors remained after #100638: * Gateway agent-cache eviction (LRU/idle sweep/cross-process invalidation) rebuilds a fresh AIAgent for the SAME session and agent_init re-derives agent.tools from live probes with no predecessor to preserve. Persist the session's resolved tool-name order in a new `sessions.tool_names` JSON column (declarative reconciliation, SCHEMA_VERSION 28), written alongside the system prompt and re-pinned on every published refresh (so /reload-mcp and compaction naturally reset it; /new mints a new row). On restore-for-existing-session the fresh definitions are folded onto the saved order via the SAME `_merge_preserving_prefix` helper — a probe-flipped tool is carried forward from the registry schema, a deregistered one dropped, new tools appended at the tail. * /reload-mcp (CLI, gateway, TUI RPC) now also calls `reprobe_tool_availability()` — drops the check_fn verdict cache and the get_tool_definitions memo — so a user can consciously pick up a credential/daemon that appeared mid-session. Docs updated.
This commit is contained in:
Teknium
@@ -989,6 +989,7 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history)
|
||||
"""
|
||||
stored_prompt = None
|
||||
stored_state = "missing"
|
||||
session_row = None
|
||||
if conversation_history and agent._session_db:
|
||||
try:
|
||||
session_row = agent._session_db.get_session(agent.session_id)
|
||||
@@ -1091,6 +1092,17 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history)
|
||||
# Continuing session — reuse the exact system prompt from the
|
||||
# previous turn so the Anthropic cache prefix matches.
|
||||
agent._cached_system_prompt = stored_prompt
|
||||
# Same contract for tools[]: a fresh AIAgent for an existing session
|
||||
# (gateway agent-cache eviction) re-probed every check_fn, so pin the
|
||||
# array back to the order this session already sent (tools freeze).
|
||||
try:
|
||||
saved_tools = session_row.get("tool_names") if session_row else None
|
||||
if saved_tools:
|
||||
from tools.mcp_tool import restore_agent_tool_prefix
|
||||
|
||||
restore_agent_tool_prefix(agent, json.loads(saved_tools))
|
||||
except Exception:
|
||||
logger.debug("tool prefix restore skipped", exc_info=True)
|
||||
# Prompt-section callbacks are new-session-only. Recover their frozen
|
||||
# bytes from the persisted full prompt so a later compression rebuild
|
||||
# keeps them without evaluating plugin state in this resumed process.
|
||||
@@ -1173,6 +1185,9 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history)
|
||||
if agent._session_db:
|
||||
try:
|
||||
agent._session_db.update_system_prompt(agent.session_id, agent._cached_system_prompt)
|
||||
from tools.mcp_tool import persist_agent_tool_names
|
||||
|
||||
persist_agent_tool_names(agent)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Session DB update_system_prompt failed for session %s: "
|
||||
|
||||
@@ -14764,7 +14764,9 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
||||
sees the updated tools on the next turn.
|
||||
"""
|
||||
try:
|
||||
from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools, _servers, _lock
|
||||
from tools.mcp_tool import (
|
||||
shutdown_mcp_servers, discover_mcp_tools, reprobe_tool_availability, _servers, _lock,
|
||||
)
|
||||
|
||||
# Capture old server names
|
||||
with _lock:
|
||||
@@ -14776,6 +14778,8 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin, CLIBillingMixin):
|
||||
# Shutdown existing connections
|
||||
shutdown_mcp_servers()
|
||||
|
||||
# Explicit reload also re-probes tool availability (check_fn).
|
||||
reprobe_tool_availability()
|
||||
# Reconnect (reads config.yaml fresh)
|
||||
new_tools = discover_mcp_tools()
|
||||
|
||||
|
||||
+3
-1
@@ -26230,7 +26230,7 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
return await self._execute_mcp_reload(event)
|
||||
try:
|
||||
from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools, _servers, _lock
|
||||
from tools.mcp_tool import _server_scope_keys
|
||||
from tools.mcp_tool import _server_scope_keys, reprobe_tool_availability
|
||||
from tools.registry import registry
|
||||
|
||||
reload_scope = registry.current_scope_key() if multiplex else None
|
||||
@@ -26250,6 +26250,8 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
|
||||
await self._run_in_executor_with_context(
|
||||
lambda: shutdown_mcp_servers(scope=reload_scope)
|
||||
)
|
||||
# Explicit reload also re-probes tool availability (check_fn).
|
||||
reprobe_tool_availability()
|
||||
|
||||
# Reconnect by discovering tools (reads config.yaml fresh)
|
||||
new_tools = await self._run_in_executor_with_context(discover_mcp_tools)
|
||||
|
||||
@@ -9741,6 +9741,25 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
|
||||
self._delete_unreferenced_system_prompts(conn)
|
||||
self._execute_write(_do)
|
||||
|
||||
def update_session_tool_names(
|
||||
self, session_id: str, tool_names: Optional[List[str]]
|
||||
) -> None:
|
||||
"""Persist the session's resolved ``tools[]`` name order (JSON array).
|
||||
|
||||
Read back by ``tools.mcp_tool.restore_agent_tool_prefix`` when a fresh
|
||||
``AIAgent`` is rebuilt for an existing session (gateway agent-cache
|
||||
eviction) so a flipped ``check_fn`` verdict can't fork the cached tool
|
||||
prefix. ``None`` clears the pin.
|
||||
"""
|
||||
payload = json.dumps(list(tool_names)) if tool_names is not None else None
|
||||
|
||||
def _do(conn):
|
||||
conn.execute(
|
||||
"UPDATE sessions SET tool_names = ? WHERE id = ?",
|
||||
(payload, session_id),
|
||||
)
|
||||
self._execute_write(_do)
|
||||
|
||||
def update_session_model(
|
||||
self, session_id: str, model: str, provider: Optional[str] = None
|
||||
) -> None:
|
||||
|
||||
@@ -354,7 +354,7 @@ def _sql_session_last_active_by_id(session_id_expr: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
SCHEMA_VERSION = 27
|
||||
SCHEMA_VERSION = 28
|
||||
|
||||
|
||||
# FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the
|
||||
@@ -452,6 +452,7 @@ CREATE TABLE IF NOT EXISTS sessions (
|
||||
pinned INTEGER NOT NULL DEFAULT 0,
|
||||
hidden INTEGER NOT NULL DEFAULT 0,
|
||||
last_read_at REAL,
|
||||
tool_names TEXT,
|
||||
FOREIGN KEY (parent_session_id) REFERENCES sessions(id),
|
||||
FOREIGN KEY (system_prompt_hash) REFERENCES system_prompts(hash)
|
||||
);
|
||||
|
||||
@@ -286,3 +286,54 @@ def test_preserve_prefix_appends_late_arrivals_at_the_tail(monkeypatch):
|
||||
assert [t["function"]["name"] for t in agent.tools] == [
|
||||
"read_file", "terminal", "aaa_mcp_late",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tools[] freeze: eviction rebuild + the /reload-mcp re-probe hatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_eviction_rebuild_restores_the_sessions_saved_tool_order(monkeypatch):
|
||||
"""A fresh AIAgent for an EXISTING session must keep the saved tools[] pin.
|
||||
|
||||
Gateway agent-cache eviction rebuilds the agent; ``agent_init`` re-probes
|
||||
every ``check_fn`` and ``browser_navigate``'s flips false. The persisted
|
||||
name list stands in for the missing predecessor: the tool is carried
|
||||
forward from the registry schema, byte-for-byte in its old slot.
|
||||
"""
|
||||
from tools import registry as registry_mod
|
||||
|
||||
saved = ["read_file", "browser_navigate", "terminal"]
|
||||
entries = {n: types.SimpleNamespace(name=n, schema=_tool(n)["function"]) for n in saved}
|
||||
monkeypatch.setattr(registry_mod.registry, "get_all_entries", lambda: list(entries.values()), raising=False)
|
||||
monkeypatch.setattr(registry_mod.registry, "get_entry", lambda name, **kw: entries.get(name), raising=False)
|
||||
|
||||
rebuilt = _agent(["read_file", "terminal"]) # probe flipped: browser_navigate gone
|
||||
changed = mcp_tool.restore_agent_tool_prefix(rebuilt, saved)
|
||||
|
||||
assert changed is True
|
||||
assert [t["function"]["name"] for t in rebuilt.tools] == saved
|
||||
assert rebuilt.valid_tool_names == set(saved)
|
||||
|
||||
|
||||
def test_reprobe_tool_availability_drops_cached_check_fn_verdicts(monkeypatch):
|
||||
"""/reload-mcp is the explicit hatch: a cached False must be re-probed."""
|
||||
from tools import registry as registry_mod
|
||||
import model_tools
|
||||
|
||||
verdict = {"ok": False}
|
||||
|
||||
def probe():
|
||||
return verdict["ok"]
|
||||
|
||||
monkeypatch.setattr(registry_mod, "check_fn_cache_scope", lambda: "test-scope")
|
||||
assert registry_mod._check_fn_cached(probe) is False
|
||||
verdict["ok"] = True
|
||||
assert registry_mod._check_fn_cached(probe) is False # TTL cache replays stale verdict
|
||||
with model_tools._tool_defs_cache_lock:
|
||||
model_tools._tool_defs_cache[("sentinel",)] = []
|
||||
|
||||
mcp_tool.reprobe_tool_availability()
|
||||
|
||||
assert registry_mod._check_fn_cached(probe) is True
|
||||
assert ("sentinel",) not in model_tools._tool_defs_cache
|
||||
|
||||
+77
-1
@@ -8509,7 +8509,83 @@ def refresh_agent_mcp_tools(
|
||||
engine_names.clear()
|
||||
engine_names.update(staged_engine_names)
|
||||
agent._tool_snapshot_generation = max(published_gen, snapshot_generation)
|
||||
return new_names - current
|
||||
added = new_names - current
|
||||
# Every published snapshot re-pins the session's tool order so a later
|
||||
# rebuild-for-existing-session (gateway agent-cache eviction) restores
|
||||
# exactly these names — see ``restore_agent_tool_prefix``.
|
||||
persist_agent_tool_names(agent)
|
||||
return added
|
||||
|
||||
|
||||
def reprobe_tool_availability() -> None:
|
||||
"""Explicit ``/reload-mcp`` hatch out of the tools[] freeze.
|
||||
|
||||
Availability-gated tools (``check_fn``: Docker, HASS_TOKEN, OAuth…) are
|
||||
frozen for the life of a session; a credential or daemon that appears
|
||||
mid-session is only picked up when the user consciously asks. Drop the
|
||||
``check_fn`` verdict cache AND the ``get_tool_definitions`` memo (keyed on
|
||||
registry generation, so it would otherwise replay the stale verdicts).
|
||||
"""
|
||||
from model_tools import _clear_tool_defs_cache
|
||||
from tools.registry import invalidate_check_fn_cache
|
||||
|
||||
invalidate_check_fn_cache()
|
||||
_clear_tool_defs_cache()
|
||||
|
||||
|
||||
def persist_agent_tool_names(agent) -> None:
|
||||
"""Best-effort: write ``agent.tools`` names to the session row (freeze pin)."""
|
||||
db = getattr(agent, "_session_db", None)
|
||||
session_id = getattr(agent, "session_id", None)
|
||||
if not db or not session_id:
|
||||
return
|
||||
try:
|
||||
db.update_session_tool_names(
|
||||
session_id,
|
||||
[t["function"]["name"] for t in (getattr(agent, "tools", None) or [])],
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("tool_names persist skipped", exc_info=True)
|
||||
|
||||
|
||||
def restore_agent_tool_prefix(agent, saved_names: list) -> bool:
|
||||
"""Fold a freshly built agent's ``tools`` onto the session's saved order.
|
||||
|
||||
Closes the second door on the tools[] freeze: the gateway rebuilds a NEW
|
||||
``AIAgent`` for an existing session after agent-cache eviction, and
|
||||
``agent_init`` re-derives ``agent.tools`` from live ``check_fn`` probes
|
||||
with no predecessor to preserve. The saved name list stands in for that
|
||||
predecessor: a saved tool that is still registered but failed its probe
|
||||
this time is carried forward from the registry's schema, a deregistered
|
||||
one is dropped, and genuinely new tools append at the tail — the same
|
||||
``_merge_preserving_prefix`` rule the between-turns refresh uses.
|
||||
Returns True when the snapshot was changed.
|
||||
"""
|
||||
if not saved_names:
|
||||
return False
|
||||
from tools.registry import registry
|
||||
|
||||
fresh_defs = list(getattr(agent, "tools", None) or [])
|
||||
fresh = {t["function"]["name"]: t for t in fresh_defs}
|
||||
saved_defs = []
|
||||
for name in saved_names:
|
||||
entry_def = fresh.get(name)
|
||||
if entry_def is None:
|
||||
entry = registry.get_entry(name)
|
||||
if entry is None:
|
||||
continue
|
||||
entry_def = {"type": "function", "function": {**entry.schema, "name": entry.name}}
|
||||
saved_defs.append(entry_def)
|
||||
registered_names = {entry.name for entry in registry.get_all_entries()}
|
||||
merged, merged_names = _merge_preserving_prefix(saved_defs, fresh_defs, registered_names)
|
||||
with _agent_tools_lock:
|
||||
if merged == fresh_defs:
|
||||
return False
|
||||
agent.tools = merged
|
||||
agent.valid_tool_names = merged_names
|
||||
if [t["function"]["name"] for t in merged] != list(saved_names):
|
||||
persist_agent_tool_names(agent)
|
||||
return True
|
||||
|
||||
|
||||
def _merge_preserving_prefix(
|
||||
|
||||
@@ -133,7 +133,7 @@ def _(rid, params: dict) -> dict:
|
||||
return _err(rid, 5019, f"compute-host reload_mcp failed: {exc}")
|
||||
return _ok(rid, {"status": "reloaded", "turn_isolation": True, "host_ack": ack})
|
||||
|
||||
from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools
|
||||
from tools.mcp_tool import shutdown_mcp_servers, discover_mcp_tools, reprobe_tool_availability
|
||||
|
||||
def _refresh_session_agent() -> None:
|
||||
"""Rebuild THIS session's cached tool snapshot from the live
|
||||
@@ -184,6 +184,7 @@ def _(rid, params: dict) -> dict:
|
||||
loaded = _compute_mcp_rev()
|
||||
for _ in range(_MCP_RELOAD_MAX_PASSES):
|
||||
shutdown_mcp_servers()
|
||||
reprobe_tool_availability()
|
||||
discover_mcp_tools()
|
||||
after = _compute_mcp_rev()
|
||||
if after == loaded:
|
||||
|
||||
@@ -115,7 +115,7 @@ Type `/` in the CLI to open the autocomplete menu. Built-in commands are case-in
|
||||
| `/blueprint [name] [slot=value ...]` (alias: `/bp`) | Set up an automation from a blueprint template. Bare `/blueprint` lists the catalog; `/blueprint <name>` starts a guided slot-filling flow on the next agent turn; `/blueprint <name> slot=value ...` creates the job directly. |
|
||||
| `/curator` | Background skill maintenance — `status`, `run`, `pin`, `archive`. See [Curator](/user-guide/features/curator). |
|
||||
| `/kanban <action>` | Drive the multi-profile, multi-project collaboration board without leaving chat. Full `hermes kanban` surface is available: `/kanban list`, `/kanban show t_abc`, `/kanban create "title" --assignee X`, `/kanban comment t_abc "text"`, `/kanban unblock t_abc`, `/kanban dispatch`, etc. Multi-board support included: `/kanban boards list`, `/kanban boards create <slug>`, `/kanban boards switch <slug>`, `/kanban --board <slug> <action>`. See [Kanban slash command](/user-guide/features/kanban#kanban-slash-command). |
|
||||
| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config.yaml |
|
||||
| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config.yaml and re-probe tool availability (credentials/daemons that appeared mid-session) |
|
||||
| `/reload-skills` (alias: `/reload_skills`) | Re-scan `~/.hermes/skills/` for newly installed or removed skills |
|
||||
| `/reload` | Reload `.env` variables into the running session (picks up new API keys without restarting) |
|
||||
| `/plugins` | List installed plugins and their status |
|
||||
@@ -291,7 +291,7 @@ The messaging gateway supports the following built-in commands inside Telegram,
|
||||
| `/skills [pending\|approve\|reject\|diff\|approval]` | Review pending **skill** writes staged by the write-approval gate (`skills.write_approval`). Shows a one-line gist per staged write; `/skills diff <id>` is truncated for chat — read the full diff on the CLI or in `~/.hermes/pending/skills/<id>.json`. Only appears when the gate is on (or staged writes remain); search/install stay CLI-only. |
|
||||
| `/kanban <action>` | Drive the multi-profile, multi-project collaboration board from chat — identical argument surface to the CLI. Bypasses the running-agent guard, so `/kanban unblock t_abc`, `/kanban comment t_abc "…"`, `/kanban list --mine`, `/kanban boards switch <slug>`, etc. work mid-turn. `/kanban create …` auto-subscribes the originating chat to the new task's terminal events. See [Kanban slash command](/user-guide/features/kanban#kanban-slash-command). |
|
||||
| `/platform <list\|pause\|resume> [name]` | Operate a running gateway platform right from chat. `/platform list` shows every adapter and its state (running, paused-by-breaker, manually-paused); `/platform pause <name>` stops dispatching new messages to that adapter without unloading it; `/platform resume <name>` re-enables it and clears a tripped circuit breaker once the upstream is healthy. |
|
||||
| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config. |
|
||||
| `/reload-mcp` (alias: `/reload_mcp`) | Reload MCP servers from config and re-probe tool availability. |
|
||||
| `/verbose` | Cycle tool progress display. **Off by default on messaging** — enable with `display.tool_progress_command: true` in `config.yaml`. |
|
||||
| `/yolo` | Toggle YOLO mode — skip all dangerous command approval prompts. |
|
||||
| `/commands [page]` | Browse all commands and skills (paginated). |
|
||||
|
||||
@@ -644,7 +644,7 @@ If you change MCP config, use:
|
||||
/reload-mcp
|
||||
```
|
||||
|
||||
This reloads MCP servers from config and refreshes the available tool list. For runtime tool changes pushed by the server itself, see [Dynamic Tool Discovery](#dynamic-tool-discovery) above.
|
||||
This reloads MCP servers from config and refreshes the available tool list. It is also the explicit way to re-probe availability-gated tools (Docker, `HASS_TOKEN`, OAuth…): a session's tool set is otherwise frozen, so a credential or daemon that appears mid-session is only picked up on `/reload-mcp`, `/new`, or context compaction. For runtime tool changes pushed by the server itself, see [Dynamic Tool Discovery](#dynamic-tool-discovery) above.
|
||||
|
||||
### Toolsets
|
||||
|
||||
|
||||
Reference in New Issue
Block a user