fix(gateway): serialize config mutations and finish the router off-loop sweep

Two follow-ups to the off-loop move, from external review (both verified,
the second larger than reported):

- Config read-modify-write handlers moved to worker threads could now
  interleave — _CONFIG_LOCK covers each load/save individually, never the
  span between them; the event loop used to serialize these accidentally.
  New _CONFIG_MUTATION_LOCK (worker-threads only, so it can never block
  the loop) held across the whole load→mutate→save span in all seven RMW
  handlers. update_config_raw skipped: it's a full-document replace with
  no server-side read, so a lock cannot close its client-side window.

- The review flagged two skills routes still taking _SKILLS_PROFILE_LOCK
  on the event loop; a systematic audit of hermes_cli/web_routers/ found
  24 on-loop routes (skills 5, mcp 9, tools 10, cron 1). All moved to the
  same inner-_run + asyncio.to_thread pattern, mutating ones under the
  mutation lock, uniform lock order (_SKILLS_PROFILE_LOCK →
  _CONFIG_MUTATION_LOCK). Await-safe _config_profile_scope routes, plain
  def routes, and already-threaded routes unchanged.

Regression tests: concurrent theme+font updates both survive (fails with
the lock nulled: "theme write lost to a concurrent font write"); event
loop stays responsive while the profile lock is held during GET
/api/skills. 214 tests passing across the touched suites.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Royalaid
2026-08-07 01:13:37 -07:00
committed by kshitij
parent 52c9aee3bf
commit 965a548788
6 changed files with 690 additions and 431 deletions
+1 -1
View File
@@ -146,7 +146,7 @@ async def cron_fire_webhook(request: Request):
auth = request.headers.get("Authorization", "")
token = auth[7:].strip() if auth.startswith("Bearer ") else ""
cfg = load_config()
cfg = await asyncio.to_thread(load_config)
claims = get_fire_verifier()(
token=token,
expected_audience=cfg_get(cfg, "cron", "chronos", "expected_audience", default=""),
+81 -41
View File
@@ -53,14 +53,20 @@ save_env_value = late("save_env_value")
_mcp_oauth_flows = LateState("_mcp_oauth_flows")
_mcp_oauth_flows_lock = LateState("_mcp_oauth_flows_lock")
_MAX_PENDING_MCP_OAUTH_FLOWS = LateState("_MAX_PENDING_MCP_OAUTH_FLOWS")
# Config read-modify-write serialization for off-loop handlers (defined in
# web_server.py; LateState supports ``with``-blocks, so this is the live lock).
_CONFIG_MUTATION_LOCK = LateState("_CONFIG_MUTATION_LOCK")
@router.get("/api/mcp/servers")
async def list_mcp_servers(profile: Optional[str] = None):
from hermes_cli.mcp_config import _get_mcp_servers
with _profile_scope(profile):
servers = _get_mcp_servers()
def _read():
with _profile_scope(profile):
return _get_mcp_servers()
servers = await asyncio.to_thread(_read)
return {
"servers": [
_mcp_server_summary(name, cfg) for name, cfg in sorted(servers.items())
@@ -81,20 +87,29 @@ async def add_mcp_server(body: MCPServerCreate, profile: Optional[str] = None):
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
with _profile_scope(body.profile or profile):
existing = _get_mcp_servers()
def _check_existing():
with _profile_scope(body.profile or profile):
return _get_mcp_servers()
existing = await asyncio.to_thread(_check_existing)
if name in existing:
raise HTTPException(status_code=409, detail=f"Server '{name}' already exists")
try:
def _save():
with _profile_scope(body.profile or profile):
if bearer_token is not None:
server_config["headers"] = _save_bearer_auth_token(name, bearer_token)
if not _save_mcp_server(name, server_config):
raise HTTPException(
status_code=400,
detail=f"Server '{name}' rejected: suspicious command/args configuration",
)
# _save_mcp_server does its own load→mutate→save of config.yaml;
# serialize the whole cycle against other off-loop config writers.
with _CONFIG_MUTATION_LOCK:
if bearer_token is not None:
server_config["headers"] = _save_bearer_auth_token(name, bearer_token)
if not _save_mcp_server(name, server_config):
raise HTTPException(
status_code=400,
detail=f"Server '{name}' rejected: suspicious command/args configuration",
)
try:
await asyncio.to_thread(_save)
except HTTPException:
raise
except Exception as exc:
@@ -116,8 +131,12 @@ async def replace_mcp_servers(body: MCPServersReplace, profile: Optional[str] =
"""
from hermes_cli.mcp_config import _replace_mcp_servers
with _profile_scope(body.profile or profile):
ok, issues = _replace_mcp_servers(body.servers)
def _run():
with _profile_scope(body.profile or profile):
with _CONFIG_MUTATION_LOCK:
return _replace_mcp_servers(body.servers)
ok, issues = await asyncio.to_thread(_run)
if not ok:
raise HTTPException(status_code=400, detail="; ".join(issues))
return {"ok": True}
@@ -127,8 +146,12 @@ async def replace_mcp_servers(body: MCPServersReplace, profile: Optional[str] =
async def remove_mcp_server(name: str, profile: Optional[str] = None):
from hermes_cli.mcp_config import _remove_mcp_server
with _profile_scope(profile):
removed = _remove_mcp_server(name)
def _run():
with _profile_scope(profile):
with _CONFIG_MUTATION_LOCK:
return _remove_mcp_server(name)
removed = await asyncio.to_thread(_run)
if not removed:
raise HTTPException(status_code=404, detail=f"Server '{name}' not found")
return {"ok": True}
@@ -143,8 +166,11 @@ async def test_mcp_server(name: str, profile: Optional[str] = None):
_probe_single_server,
)
with _profile_scope(profile):
servers = _get_mcp_servers()
def _read():
with _profile_scope(profile):
return _get_mcp_servers()
servers = await asyncio.to_thread(_read)
if name not in servers:
raise HTTPException(status_code=404, detail=f"Server '{name}' not found")
@@ -205,9 +231,12 @@ async def auth_mcp_server(name: str, request: Request, profile: Optional[str] =
from hermes_constants import get_hermes_home
process_home = str(get_hermes_home().expanduser().resolve(strict=False))
with _profile_scope(profile):
servers = _get_mcp_servers()
flow_home = str(get_hermes_home().expanduser().resolve(strict=False))
def _read():
with _profile_scope(profile):
return _get_mcp_servers(), str(get_hermes_home().expanduser().resolve(strict=False))
servers, flow_home = await asyncio.to_thread(_read)
if name not in servers:
raise HTTPException(status_code=404, detail=f"Server '{name}' not found")
cfg = dict(servers[name])
@@ -325,16 +354,20 @@ async def set_mcp_server_enabled(
flag the agent reads at startup. Disabled servers stay in config so they
can be re-enabled without re-entering their settings.
"""
with _profile_scope(body.profile or profile):
cfg = load_config()
servers = cfg.get("mcp_servers")
if not isinstance(servers, dict) or name not in servers:
raise HTTPException(status_code=404, detail=f"Server '{name}' not found")
if not isinstance(servers[name], dict):
raise HTTPException(status_code=400, detail="Malformed server config")
servers[name]["enabled"] = bool(body.enabled)
save_config(cfg)
return {"ok": True, "name": name, "enabled": bool(body.enabled)}
def _run():
with _profile_scope(body.profile or profile):
with _CONFIG_MUTATION_LOCK:
cfg = load_config()
servers = cfg.get("mcp_servers")
if not isinstance(servers, dict) or name not in servers:
raise HTTPException(status_code=404, detail=f"Server '{name}' not found")
if not isinstance(servers[name], dict):
raise HTTPException(status_code=400, detail="Malformed server config")
servers[name]["enabled"] = bool(body.enabled)
save_config(cfg)
return {"ok": True, "name": name, "enabled": bool(body.enabled)}
return await asyncio.to_thread(_run)
@router.get("/api/mcp/catalog")
@@ -355,12 +388,16 @@ async def list_mcp_catalog(profile: Optional[str] = None):
entries = []
try:
with _profile_scope(profile):
catalog_entries = list(mcp_catalog.list_catalog())
installed_state = {
e.name: (mcp_catalog.is_installed(e.name), mcp_catalog.is_enabled(e.name))
for e in catalog_entries
}
def _read():
with _profile_scope(profile):
catalog = list(mcp_catalog.list_catalog())
state = {
e.name: (mcp_catalog.is_installed(e.name), mcp_catalog.is_enabled(e.name))
for e in catalog
}
return catalog, state
catalog_entries, installed_state = await asyncio.to_thread(_read)
for entry in catalog_entries:
auth = entry.auth
transport = entry.transport
@@ -434,10 +471,13 @@ async def install_mcp_catalog_entry(body: MCPCatalogInstall, profile: Optional[s
# they need; we only write the ones the user provided).
effective_profile = body.profile or profile
if body.env:
with _profile_scope(effective_profile):
for k, v in body.env.items():
if v:
save_env_value(k, v)
def _write_env():
with _profile_scope(effective_profile):
for k, v in body.env.items():
if v:
save_env_value(k, v)
await asyncio.to_thread(_write_env)
# Git-bootstrap entries can take a while to clone — run via the background
# action path so the request returns immediately and the UI can tail logs.
+66 -45
View File
@@ -49,6 +49,10 @@ load_config = late("load_config")
# Live proxies for web_server-owned module state (mutations/monkeypatches
# on web_server remain authoritative; resolved at operation time).
_SKILL_HUB_SOURCE_LABELS = LateState("_SKILL_HUB_SOURCE_LABELS")
# Config read-modify-write serialization for off-loop handlers (see the
# definition in web_server.py). LateState supports ``with``-blocks, so this
# is the live lock object, not a frozen import-time copy.
_CONFIG_MUTATION_LOCK = LateState("_CONFIG_MUTATION_LOCK")
@hub_router.post("/api/skills/hub/install")
@@ -401,40 +405,48 @@ async def get_skills(profile: Optional[str] = None):
activity_count,
load_usage,
)
with _profile_scope(profile):
config = load_config()
disabled = get_disabled_skills(config)
skills = _find_all_skills(skip_disabled=True)
usage = load_usage()
# Set-based provenance (same classification as skill_usage.provenance,
# without a per-skill manifest read): hub > bundled > agent, where
# "agent" covers agent-authored AND local hand-made skills — the ones
# the user may edit/delete from the UI.
bundled_names = _read_bundled_manifest_names()
hub_names = _read_hub_installed_names()
for s in skills:
s["enabled"] = s["name"] not in disabled
s["usage"] = activity_count(usage.get(s["name"], {}))
s["provenance"] = (
"hub" if s["name"] in hub_names
else "bundled" if s["name"] in bundled_names
else "agent"
)
return skills
def _run():
with _profile_scope(profile):
config = load_config()
disabled = get_disabled_skills(config)
skills = _find_all_skills(skip_disabled=True)
usage = load_usage()
# Set-based provenance (same classification as skill_usage.provenance,
# without a per-skill manifest read): hub > bundled > agent, where
# "agent" covers agent-authored AND local hand-made skills — the ones
# the user may edit/delete from the UI.
bundled_names = _read_bundled_manifest_names()
hub_names = _read_hub_installed_names()
for s in skills:
s["enabled"] = s["name"] not in disabled
s["usage"] = activity_count(usage.get(s["name"], {}))
s["provenance"] = (
"hub" if s["name"] in hub_names
else "bundled" if s["name"] in bundled_names
else "agent"
)
return skills
return await asyncio.to_thread(_run)
@router.put("/api/skills/toggle")
async def toggle_skill(body: SkillToggle, profile: Optional[str] = None):
from hermes_cli.skills_config import get_disabled_skills, save_disabled_skills
with _profile_scope(body.profile or profile):
config = load_config()
disabled = get_disabled_skills(config)
if body.enabled:
disabled.discard(body.name)
else:
disabled.add(body.name)
save_disabled_skills(config, disabled)
return {"ok": True, "name": body.name, "enabled": body.enabled}
def _run():
with _profile_scope(body.profile or profile):
with _CONFIG_MUTATION_LOCK:
config = load_config()
disabled = get_disabled_skills(config)
if body.enabled:
disabled.discard(body.name)
else:
disabled.add(body.name)
save_disabled_skills(config, disabled)
return {"ok": True, "name": body.name, "enabled": body.enabled}
return await asyncio.to_thread(_run)
@router.get("/api/skills/content")
@@ -442,18 +454,21 @@ async def get_skill_content(name: str, profile: Optional[str] = None):
"""Return the raw SKILL.md text for a skill, for the dashboard editor."""
from tools.skill_manager_tool import _find_skill
with _profile_scope(profile):
found = _find_skill(name)
if not found:
raise HTTPException(status_code=404, detail=f"Skill '{name}' not found.")
skill_md = found["path"] / "SKILL.md"
if not skill_md.exists():
raise HTTPException(status_code=404, detail=f"Skill '{name}' has no SKILL.md.")
try:
content = skill_md.read_text(encoding="utf-8")
except OSError as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return {"name": name, "content": content, "path": str(skill_md)}
def _run():
with _profile_scope(profile):
found = _find_skill(name)
if not found:
raise HTTPException(status_code=404, detail=f"Skill '{name}' not found.")
skill_md = found["path"] / "SKILL.md"
if not skill_md.exists():
raise HTTPException(status_code=404, detail=f"Skill '{name}' has no SKILL.md.")
try:
content = skill_md.read_text(encoding="utf-8")
except OSError as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
return {"name": name, "content": content, "path": str(skill_md)}
return await asyncio.to_thread(_run)
@router.post("/api/skills")
@@ -467,8 +482,11 @@ async def create_skill(body: SkillCreate):
"""
from tools.skill_manager_tool import _create_skill
with _profile_scope(body.profile):
result = _create_skill(body.name, body.content, body.category or None)
def _run():
with _profile_scope(body.profile):
return _create_skill(body.name, body.content, body.category or None)
result = await asyncio.to_thread(_run)
if not result.get("success"):
raise HTTPException(status_code=400, detail=result.get("error", "Failed to create skill."))
_clear_skills_prompt_cache()
@@ -480,8 +498,11 @@ async def update_skill_content(body: SkillContentUpdate):
"""Replace the SKILL.md of an existing skill (full rewrite) from the editor."""
from tools.skill_manager_tool import _edit_skill
with _profile_scope(body.profile):
result = _edit_skill(body.name, body.content)
def _run():
with _profile_scope(body.profile):
return _edit_skill(body.name, body.content)
result = await asyncio.to_thread(_run)
if not result.get("success"):
err = result.get("error", "Failed to update skill.")
status = 404 if "not found" in str(err).lower() else 400
+328 -284
View File
@@ -10,6 +10,7 @@ late-binding seam in :mod:`hermes_cli.web_deps` (``late`` for callables,
stays authoritative.
"""
import asyncio
import logging
import sys # noqa: F401 — used by handlers
from typing import Any, Dict, List, Optional # noqa: F401
@@ -48,6 +49,9 @@ save_config = late("save_config")
_MODEL_CATALOG_TOOLSETS = LateState("_MODEL_CATALOG_TOOLSETS")
_TERMINAL_BACKENDS = LateState("_TERMINAL_BACKENDS")
_TERMINAL_BACKEND_NAMES = LateState("_TERMINAL_BACKEND_NAMES")
# Config read-modify-write serialization for off-loop handlers (defined in
# web_server.py; LateState supports ``with``-blocks, so this is the live lock).
_CONFIG_MUTATION_LOCK = LateState("_CONFIG_MUTATION_LOCK")
@router.get("/api/tools/toolsets")
@@ -64,21 +68,25 @@ async def get_toolsets(profile: Optional[str] = None):
from hermes_cli.platforms import platform_label
from toolsets import resolve_toolset
with _profile_scope(profile):
config = load_config()
toolset_rows = _get_effective_configurable_toolsets()
target_platforms = {
_toolset_configuration_platform(name) for name, _, _ in toolset_rows
}
enabled_by_platform = {
platform: _get_platform_tools(
config,
platform,
include_default_mcp_servers=False,
)
for platform in target_platforms
}
features = get_nous_subscription_features(config)
def _read():
with _profile_scope(profile):
config = load_config()
toolset_rows = _get_effective_configurable_toolsets()
target_platforms = {
_toolset_configuration_platform(name) for name, _, _ in toolset_rows
}
enabled_by_platform = {
platform: _get_platform_tools(
config,
platform,
include_default_mcp_servers=False,
)
for platform in target_platforms
}
features = get_nous_subscription_features(config)
return config, toolset_rows, enabled_by_platform, features
config, toolset_rows, enabled_by_platform, features = await asyncio.to_thread(_read)
result = []
for name, label, desc in toolset_rows:
try:
@@ -135,37 +143,38 @@ async def toggle_toolset(name: str, body: ToolsetToggle, profile: Optional[str]
raise HTTPException(status_code=400, detail=f"Unknown toolset: {name}")
target_platform = _toolset_configuration_platform(name)
if name in _CONFIG_ONLY_TOOLSETS:
# Config-only capabilities (stt) toggle their own config section's
# ``enabled`` flag — there is no platform_toolsets entry to write.
def _run():
if name in _CONFIG_ONLY_TOOLSETS:
# Config-only capabilities (stt) toggle their own config section's
# ``enabled`` flag — there is no platform_toolsets entry to write.
with _profile_scope(body.profile or profile):
with _CONFIG_MUTATION_LOCK:
config = load_config()
section = config.setdefault(name, {})
if not isinstance(section, dict):
section = {}
config[name] = section
section["enabled"] = bool(body.enabled)
save_config(config)
return
with _profile_scope(body.profile or profile):
config = load_config()
section = config.setdefault(name, {})
if not isinstance(section, dict):
section = {}
config[name] = section
section["enabled"] = bool(body.enabled)
save_config(config)
return {
"ok": True,
"name": name,
"platform": target_platform,
"enabled": body.enabled,
}
with _profile_scope(body.profile or profile):
config = load_config()
enabled = set(
_get_platform_tools(
config,
target_platform,
include_default_mcp_servers=False,
)
)
if body.enabled:
enabled.add(name)
else:
enabled.discard(name)
_save_platform_tools(config, target_platform, enabled)
with _CONFIG_MUTATION_LOCK:
config = load_config()
enabled = set(
_get_platform_tools(
config,
target_platform,
include_default_mcp_servers=False,
)
)
if body.enabled:
enabled.add(name)
else:
enabled.discard(name)
_save_platform_tools(config, target_platform, enabled)
await asyncio.to_thread(_run)
return {
"ok": True,
"name": name,
@@ -199,79 +208,84 @@ async def get_toolset_config(name: str, profile: Optional[str] = None):
if name not in valid:
raise HTTPException(status_code=400, detail=f"Unknown toolset: {name}")
with _profile_scope(profile):
config = load_config()
cat = TOOL_CATEGORIES.get(name)
providers = []
active_provider = None
active_search_backend = None
active_extract_backend = None
if cat:
# Fetch portal/entitlement state once for the whole matrix — the
# per-provider readiness computation below reuses it instead of
# re-probing per row.
features = get_nous_subscription_features(config, force_fresh=True)
for prov in _visible_providers(cat, config, force_fresh=True):
env_vars = [
{
"key": e["key"],
"prompt": e.get("prompt", e["key"]),
"url": e.get("url"),
"default": e.get("default"),
"is_set": bool(get_env_value(e["key"])),
def _read():
with _profile_scope(profile):
config = load_config()
cat = TOOL_CATEGORIES.get(name)
providers = []
active_provider = None
active_search_backend = None
active_extract_backend = None
if cat:
# Fetch portal/entitlement state once for the whole matrix — the
# per-provider readiness computation below reuses it instead of
# re-probing per row.
features = get_nous_subscription_features(config, force_fresh=True)
for prov in _visible_providers(cat, config, force_fresh=True):
env_vars = [
{
"key": e["key"],
"prompt": e.get("prompt", e["key"]),
"url": e.get("url"),
"default": e.get("default"),
"is_set": bool(get_env_value(e["key"])),
}
for e in prov.get("env_vars", [])
]
# Surface the same active-provider determination the CLI picker
# uses (``_is_provider_active``) so the GUI highlights the provider
# actually written to config (e.g. web.backend), not just the first
# keyless one in the list.
is_active = _is_provider_active(prov, config, force_fresh=True)
if is_active and active_provider is None:
active_provider = prov["name"]
row = {
"name": prov["name"],
"badge": prov.get("badge", ""),
"tag": prov.get("tag", ""),
"env_vars": env_vars,
"post_setup": prov.get("post_setup"),
"requires_nous_auth": bool(prov.get("requires_nous_auth")),
"is_active": is_active,
# Honest server-side readiness. The GUI's old client-side
# heuristic showed "Ready" for every zero-env-var row —
# including logged-out Nous Subscription rows and never-run
# post_setup installs (see provider_readiness_status).
"status": provider_readiness_status(
prov, config, features=features, is_active=is_active
),
}
for e in prov.get("env_vars", [])
]
# Surface the same active-provider determination the CLI picker
# uses (``_is_provider_active``) so the GUI highlights the provider
# actually written to config (e.g. web.backend), not just the first
# keyless one in the list.
is_active = _is_provider_active(prov, config, force_fresh=True)
if is_active and active_provider is None:
active_provider = prov["name"]
row = {
"name": prov["name"],
"badge": prov.get("badge", ""),
"tag": prov.get("tag", ""),
"env_vars": env_vars,
"post_setup": prov.get("post_setup"),
"requires_nous_auth": bool(prov.get("requires_nous_auth")),
"is_active": is_active,
# Honest server-side readiness. The GUI's old client-side
# heuristic showed "Ready" for every zero-env-var row —
# including logged-out Nous Subscription rows and never-run
# post_setup installs (see provider_readiness_status).
"status": provider_readiness_status(
prov, config, features=features, is_active=is_active
),
}
if name == "web" and prov.get("web_backend"):
# The runtime split web into two capabilities long ago
# (web.search_backend / web.extract_backend); surface each
# row's backend key and which capabilities it can serve so
# the GUI can offer per-capability selection.
row["web_backend"] = prov["web_backend"]
row["capabilities"] = web_provider_capabilities(prov["web_backend"])
if name == "tts" and prov.get("tts_provider"):
# The provider key written to tts.provider on selection.
# Doubles as the config section holding the provider's
# voice/model settings (tts.<key>.*) so the GUI can render
# those fields inline in the Capabilities panel.
row["tts_provider"] = prov["tts_provider"]
providers.append(row)
if name == "web":
# Resolve the per-capability active backends exactly the way the
# web_search / web_extract dispatchers do (per-capability key →
# shared web.backend → credential auto-detect), so the GUI badges
# reflect what a tool call would actually hit right now.
try:
from tools.web_tools import _get_extract_backend, _get_search_backend
if name == "web" and prov.get("web_backend"):
# The runtime split web into two capabilities long ago
# (web.search_backend / web.extract_backend); surface each
# row's backend key and which capabilities it can serve so
# the GUI can offer per-capability selection.
row["web_backend"] = prov["web_backend"]
row["capabilities"] = web_provider_capabilities(prov["web_backend"])
if name == "tts" and prov.get("tts_provider"):
# The provider key written to tts.provider on selection.
# Doubles as the config section holding the provider's
# voice/model settings (tts.<key>.*) so the GUI can render
# those fields inline in the Capabilities panel.
row["tts_provider"] = prov["tts_provider"]
providers.append(row)
if name == "web":
# Resolve the per-capability active backends exactly the way the
# web_search / web_extract dispatchers do (per-capability key →
# shared web.backend → credential auto-detect), so the GUI badges
# reflect what a tool call would actually hit right now.
try:
from tools.web_tools import _get_extract_backend, _get_search_backend
active_search_backend = _get_search_backend()
active_extract_backend = _get_extract_backend()
except Exception:
active_search_backend = None
active_extract_backend = None
return cat, providers, active_provider, active_search_backend, active_extract_backend
cat, providers, active_provider, active_search_backend, active_extract_backend = await asyncio.to_thread(_read)
active_search_backend = _get_search_backend()
active_extract_backend = _get_extract_backend()
except Exception:
active_search_backend = None
active_extract_backend = None
payload = {
"name": name,
"has_category": cat is not None,
@@ -300,28 +314,35 @@ async def get_toolset_models(
if section is None:
return {"name": name, "has_models": False, "models": [], "current": None, "default": None}
with _profile_scope(profile):
config = load_config()
row = _find_toolset_provider_row(name, config, provider)
plugin = _resolve_toolset_model_plugin(name, row) if row else None
if not plugin:
return {
"name": name,
"has_models": False,
"models": [],
"current": None,
"default": None,
}
def _read():
with _profile_scope(profile):
config = load_config()
row = _find_toolset_provider_row(name, config, provider)
plugin = _resolve_toolset_model_plugin(name, row) if row else None
if not plugin:
return None
catalog, default_model = _toolset_model_catalog(name, plugin)
section_cfg = config.get(section)
current = None
if isinstance(section_cfg, dict):
raw = section_cfg.get("model")
if isinstance(raw, str) and raw.strip():
current = raw.strip()
if current not in catalog:
current = default_model if default_model in catalog else None
catalog, default_model = _toolset_model_catalog(name, plugin)
section_cfg = config.get(section)
current = None
if isinstance(section_cfg, dict):
raw = section_cfg.get("model")
if isinstance(raw, str) and raw.strip():
current = raw.strip()
if current not in catalog:
current = default_model if default_model in catalog else None
return row, plugin, catalog, default_model, current
resolved = await asyncio.to_thread(_read)
if resolved is None:
return {
"name": name,
"has_models": False,
"models": [],
"current": None,
"default": None,
}
row, plugin, catalog, default_model, current = resolved
models = [
{
@@ -364,30 +385,34 @@ async def select_toolset_model(
if not model_id:
raise HTTPException(status_code=400, detail="model is required")
with _profile_scope(body.profile or profile):
config = load_config()
row = _find_toolset_provider_row(name, config, body.provider)
plugin = _resolve_toolset_model_plugin(name, row) if row else None
if not plugin:
raise HTTPException(
status_code=400,
detail=f"No model-capable backend is active for {name}",
)
def _run():
with _profile_scope(body.profile or profile):
with _CONFIG_MUTATION_LOCK:
config = load_config()
row = _find_toolset_provider_row(name, config, body.provider)
plugin = _resolve_toolset_model_plugin(name, row) if row else None
if not plugin:
raise HTTPException(
status_code=400,
detail=f"No model-capable backend is active for {name}",
)
catalog, _default = _toolset_model_catalog(name, plugin)
if model_id not in catalog:
raise HTTPException(
status_code=400,
detail=f"Unknown model {model_id!r} for backend {plugin!r}",
)
catalog, _default = _toolset_model_catalog(name, plugin)
if model_id not in catalog:
raise HTTPException(
status_code=400,
detail=f"Unknown model {model_id!r} for backend {plugin!r}",
)
section_cfg = config.setdefault(section, {})
if not isinstance(section_cfg, dict):
section_cfg = {}
config[section] = section_cfg
section_cfg["model"] = model_id
save_config(config)
section_cfg = config.setdefault(section, {})
if not isinstance(section_cfg, dict):
section_cfg = {}
config[section] = section_cfg
section_cfg["model"] = model_id
save_config(config)
return plugin
plugin = await asyncio.to_thread(_run)
return {"ok": True, "name": name, "model": model_id, "plugin": plugin}
@@ -450,77 +475,82 @@ async def select_toolset_provider(
detail=f"Unknown capability: {body.capability!r} (expected 'search' or 'extract')",
)
with _profile_scope(body.profile or profile):
config = load_config()
if body.capability is not None:
# Per-capability path: resolve the picker row to its backend key
# and write web.<capability>_backend. Does NOT touch web.backend,
# so the other capability keeps resolving through the shared
# fallback chain.
cat = TOOL_CATEGORIES.get(name)
providers = _visible_providers(cat, config, force_fresh=True) if cat else []
prov = next((p for p in providers if p.get("name") == body.provider), None)
if prov is None:
raise HTTPException(
status_code=400,
detail=f"Unknown provider {body.provider!r} for toolset {name!r}",
)
backend = prov.get("web_backend")
if not backend:
raise HTTPException(
status_code=400,
detail=f"Provider {body.provider!r} has no web backend key",
)
if body.capability not in web_provider_capabilities(backend):
raise HTTPException(
status_code=400,
detail=f"{body.provider} does not support {body.capability}",
)
web_cfg = config.setdefault("web", {})
if not isinstance(web_cfg, dict):
web_cfg = {}
config["web"] = web_cfg
web_cfg[f"{body.capability}_backend"] = backend
else:
try:
apply_provider_selection(name, body.provider, config)
except KeyError as exc:
raise HTTPException(status_code=400, detail=str(exc).strip('"'))
save_config(config)
response: Dict[str, Any] = {"ok": True, "name": name, "provider": body.provider}
if body.capability is not None:
response["capability"] = body.capability
def _run():
with _profile_scope(body.profile or profile):
with _CONFIG_MUTATION_LOCK:
config = load_config()
if body.capability is not None:
# Per-capability path: resolve the picker row to its backend key
# and write web.<capability>_backend. Does NOT touch web.backend,
# so the other capability keeps resolving through the shared
# fallback chain.
cat = TOOL_CATEGORIES.get(name)
providers = _visible_providers(cat, config, force_fresh=True) if cat else []
prov = next((p for p in providers if p.get("name") == body.provider), None)
if prov is None:
raise HTTPException(
status_code=400,
detail=f"Unknown provider {body.provider!r} for toolset {name!r}",
)
backend = prov.get("web_backend")
if not backend:
raise HTTPException(
status_code=400,
detail=f"Provider {body.provider!r} has no web backend key",
)
if body.capability not in web_provider_capabilities(backend):
raise HTTPException(
status_code=400,
detail=f"{body.provider} does not support {body.capability}",
)
web_cfg = config.setdefault("web", {})
if not isinstance(web_cfg, dict):
web_cfg = {}
config["web"] = web_cfg
web_cfg[f"{body.capability}_backend"] = backend
else:
try:
apply_provider_selection(name, body.provider, config)
except KeyError as exc:
raise HTTPException(status_code=400, detail=str(exc).strip('"'))
save_config(config)
response: Dict[str, Any] = {"ok": True, "name": name, "provider": body.provider}
if body.capability is not None:
response["capability"] = body.capability
# Entitlement check for managed Nous rows — mirrors the gate the CLI
# applies via ensure_nous_portal_access at selection time.
cat = TOOL_CATEGORIES.get(name)
row = None
if cat:
row = next(
(
p
for p in _visible_providers(cat, config, force_fresh=True)
if p.get("name") == body.provider
),
None,
)
managed_feature = (row or {}).get("managed_nous_feature")
if managed_feature:
features = get_nous_subscription_features(config, force_fresh=True)
acct = features.account_info
category = MANAGED_FEATURE_COVERAGE_CATEGORY.get(managed_feature)
entitled = bool(
acct
and acct.logged_in
and (
acct.tool_gateway_entitled_for(category)
if category
else acct.tool_gateway_entitled
)
)
if not entitled:
response["needs_nous_auth"] = True
response["feature"] = managed_feature
# Entitlement check for managed Nous rows — mirrors the gate the CLI
# applies via ensure_nous_portal_access at selection time.
cat = TOOL_CATEGORIES.get(name)
row = None
if cat:
row = next(
(
p
for p in _visible_providers(cat, config, force_fresh=True)
if p.get("name") == body.provider
),
None,
)
managed_feature = (row or {}).get("managed_nous_feature")
if managed_feature:
features = get_nous_subscription_features(config, force_fresh=True)
acct = features.account_info
category = MANAGED_FEATURE_COVERAGE_CATEGORY.get(managed_feature)
entitled = bool(
acct
and acct.logged_in
and (
acct.tool_gateway_entitled_for(category)
if category
else acct.tool_gateway_entitled
)
)
if not entitled:
response["needs_nous_auth"] = True
response["feature"] = managed_feature
return response
response = await asyncio.to_thread(_run)
return response
@@ -547,35 +577,39 @@ async def save_toolset_env(name: str, body: ToolsetEnvUpdate, profile: Optional[
if name not in valid_ts:
raise HTTPException(status_code=400, detail=f"Unknown toolset: {name}")
with _profile_scope(body.profile or profile):
config = load_config()
cat = TOOL_CATEGORIES.get(name)
allowed: set[str] = set()
if cat:
for prov in _visible_providers(cat, config, force_fresh=True):
for e in prov.get("env_vars", []):
allowed.add(e["key"])
def _run():
with _profile_scope(body.profile or profile):
config = load_config()
cat = TOOL_CATEGORIES.get(name)
allowed: set[str] = set()
if cat:
for prov in _visible_providers(cat, config, force_fresh=True):
for e in prov.get("env_vars", []):
allowed.add(e["key"])
unknown = [k for k in body.env if k not in allowed]
if unknown:
raise HTTPException(
status_code=400,
detail=f"Unknown env var(s) for toolset {name}: {', '.join(sorted(unknown))}",
)
unknown = [k for k in body.env if k not in allowed]
if unknown:
raise HTTPException(
status_code=400,
detail=f"Unknown env var(s) for toolset {name}: {', '.join(sorted(unknown))}",
)
saved: List[str] = []
skipped: List[str] = []
for key, value in body.env.items():
if value and value.strip():
try:
save_env_value(key, value.strip())
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc))
saved.append(key)
else:
skipped.append(key)
saved: List[str] = []
skipped: List[str] = []
for key, value in body.env.items():
if value and value.strip():
try:
save_env_value(key, value.strip())
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc))
saved.append(key)
else:
skipped.append(key)
status = {k: bool(get_env_value(k)) for k in allowed}
status = {k: bool(get_env_value(k)) for k in allowed}
return saved, skipped, status
saved, skipped, status = await asyncio.to_thread(_run)
return {"ok": True, "name": name, "saved": saved, "skipped": skipped, "is_set": status}
@@ -639,27 +673,30 @@ async def get_terminal_backends(profile: Optional[str] = None):
Probes are fast (<~2s each) and defensive — a probe failure surfaces as a
status, never an error response.
"""
with _profile_scope(profile):
config = load_config()
terminal_cfg = config.get("terminal")
if not isinstance(terminal_cfg, dict):
terminal_cfg = {}
active = str(terminal_cfg.get("backend") or "local").strip().lower()
if active not in _TERMINAL_BACKEND_NAMES:
active = "local"
def _read():
with _profile_scope(profile):
config = load_config()
terminal_cfg = config.get("terminal")
if not isinstance(terminal_cfg, dict):
terminal_cfg = {}
active = str(terminal_cfg.get("backend") or "local").strip().lower()
if active not in _TERMINAL_BACKEND_NAMES:
active = "local"
backends = []
for row in _TERMINAL_BACKENDS:
status, detail = _probe_terminal_backend(row["name"], terminal_cfg)
backends.append({
"name": row["name"],
"label": row["label"],
"description": row["description"],
"active": row["name"] == active,
"status": status,
"detail": detail,
})
return {"active": active, "backends": backends}
backends = []
for row in _TERMINAL_BACKENDS:
status, detail = _probe_terminal_backend(row["name"], terminal_cfg)
backends.append({
"name": row["name"],
"label": row["label"],
"description": row["description"],
"active": row["name"] == active,
"status": status,
"detail": detail,
})
return {"active": active, "backends": backends}
return await asyncio.to_thread(_read)
@router.put("/api/tools/terminal/backend")
@@ -680,14 +717,18 @@ async def select_terminal_backend(
f"Use one of: {', '.join(sorted(_TERMINAL_BACKEND_NAMES))}",
)
with _profile_scope(body.profile or profile):
config = load_config()
terminal_cfg = config.setdefault("terminal", {})
if not isinstance(terminal_cfg, dict):
terminal_cfg = {}
config["terminal"] = terminal_cfg
terminal_cfg["backend"] = backend
save_config(config)
def _run():
with _profile_scope(body.profile or profile):
with _CONFIG_MUTATION_LOCK:
config = load_config()
terminal_cfg = config.setdefault("terminal", {})
if not isinstance(terminal_cfg, dict):
terminal_cfg = {}
config["terminal"] = terminal_cfg
terminal_cfg["backend"] = backend
save_config(config)
await asyncio.to_thread(_run)
return {"ok": True, "backend": backend}
@@ -701,8 +742,11 @@ async def get_computer_use_status(profile: Optional[str] = None):
"""
from tools.computer_use.permissions import computer_use_status
with _profile_scope(profile):
return computer_use_status()
def _read():
with _profile_scope(profile):
return computer_use_status()
return await asyncio.to_thread(_read)
@router.post("/api/tools/computer-use/permissions/grant")
+77 -60
View File
@@ -2966,6 +2966,16 @@ _TOPOLOGY_CACHE: Dict[str, Any] = {"ts": 0.0, "data": None, "fn": None}
_TOPOLOGY_CACHE_LOCK = threading.Lock()
_TOPOLOGY_CACHE_TTL = 10.0
# Serializes read-modify-write cycles over config.yaml for handlers that run
# in worker threads (asyncio.to_thread). config.py's _CONFIG_LOCK covers each
# load_config()/save_config() call individually, not the span between them —
# when these handlers ran on the event loop the loop itself serialized the
# whole cycle, but off-loop two concurrent updates could interleave
# load→mutate→save and silently drop one another's writes. Held only in
# worker threads, so it can never block the event loop. RLock so a locked
# section that calls helpers which also take it can't self-deadlock.
_CONFIG_MUTATION_LOCK = threading.RLock()
def _topology_cache_get(fn: Any) -> Optional[Dict[str, Any]]:
if (
@@ -6959,9 +6969,10 @@ async def update_config(body: ConfigUpdate, profile: Optional[str] = None):
# is not sent in the PUT body. A full-replace save would silently
# drop those keys. Deep-merge incoming over what's on disk so the
# frontend can only overwrite what it explicitly sends.
existing = read_raw_config()
incoming = _denormalize_config_from_web(body.config)
save_config(_deep_merge(existing, incoming))
with _CONFIG_MUTATION_LOCK:
existing = read_raw_config()
incoming = _denormalize_config_from_web(body.config)
save_config(_deep_merge(existing, incoming))
return {"ok": True}
try:
@@ -12924,11 +12935,12 @@ async def set_memory_provider(body: MemoryProviderSelect):
def _run():
_require_memory_provider_ready(provider)
cfg = load_config()
if not isinstance(cfg.get("memory"), dict):
cfg["memory"] = {}
cfg["memory"]["provider"] = provider
save_config(cfg)
with _CONFIG_MUTATION_LOCK:
cfg = load_config()
if not isinstance(cfg.get("memory"), dict):
cfg["memory"] = {}
cfg["memory"]["provider"] = provider
save_config(cfg)
return {"ok": True, "active": provider}
return await asyncio.to_thread(_run)
@@ -13237,23 +13249,24 @@ async def create_hook(body: HookCreate):
pass
def _run():
cfg = load_config()
hooks_cfg = cfg.get("hooks")
if not isinstance(hooks_cfg, dict):
hooks_cfg = {}
cfg["hooks"] = hooks_cfg
entries = hooks_cfg.get(event)
if not isinstance(entries, list):
entries = []
hooks_cfg[event] = entries
with _CONFIG_MUTATION_LOCK:
cfg = load_config()
hooks_cfg = cfg.get("hooks")
if not isinstance(hooks_cfg, dict):
hooks_cfg = {}
cfg["hooks"] = hooks_cfg
entries = hooks_cfg.get(event)
if not isinstance(entries, list):
entries = []
hooks_cfg[event] = entries
new_entry: Dict[str, Any] = {"command": command}
if body.matcher:
new_entry["matcher"] = body.matcher
if body.timeout is not None:
new_entry["timeout"] = int(body.timeout)
entries.append(new_entry)
save_config(cfg)
new_entry: Dict[str, Any] = {"command": command}
if body.matcher:
new_entry["matcher"] = body.matcher
if body.timeout is not None:
new_entry["timeout"] = int(body.timeout)
entries.append(new_entry)
save_config(cfg)
approved = False
if body.approve:
@@ -13279,21 +13292,22 @@ async def delete_hook(body: HookDelete):
raise HTTPException(status_code=400, detail="event and command are required")
def _run():
cfg = load_config()
hooks_cfg = cfg.get("hooks")
removed = False
if isinstance(hooks_cfg, dict) and isinstance(hooks_cfg.get(event), list):
before = len(hooks_cfg[event])
hooks_cfg[event] = [
e for e in hooks_cfg[event]
if not (isinstance(e, dict) and e.get("command") == command)
]
removed = len(hooks_cfg[event]) < before
if not hooks_cfg[event]:
del hooks_cfg[event]
if not hooks_cfg:
cfg.pop("hooks", None)
save_config(cfg)
with _CONFIG_MUTATION_LOCK:
cfg = load_config()
hooks_cfg = cfg.get("hooks")
if isinstance(hooks_cfg, dict) and isinstance(hooks_cfg.get(event), list):
before = len(hooks_cfg[event])
hooks_cfg[event] = [
e for e in hooks_cfg[event]
if not (isinstance(e, dict) and e.get("command") == command)
]
removed = len(hooks_cfg[event]) < before
if not hooks_cfg[event]:
del hooks_cfg[event]
if not hooks_cfg:
cfg.pop("hooks", None)
save_config(cfg)
# Revoke consent regardless so a re-add re-prompts.
try:
@@ -16683,11 +16697,12 @@ async def get_dashboard_themes():
async def set_dashboard_theme(body: ThemeSetBody):
"""Set the active dashboard theme (persists to config.yaml)."""
def _run():
config = load_config()
if "dashboard" not in config:
config["dashboard"] = {}
config["dashboard"]["theme"] = body.name
save_config(config)
with _CONFIG_MUTATION_LOCK:
config = load_config()
if "dashboard" not in config:
config["dashboard"] = {}
config["dashboard"]["theme"] = body.name
save_config(config)
return {"ok": True, "theme": body.name}
return await asyncio.to_thread(_run)
@@ -16732,11 +16747,12 @@ async def set_dashboard_font(body: FontSetBody):
font = body.font if body.font in _FONT_CHOICES else _FONT_DEFAULT_ID
def _run():
config = load_config()
if "dashboard" not in config:
config["dashboard"] = {}
config["dashboard"]["font"] = font
save_config(config)
with _CONFIG_MUTATION_LOCK:
config = load_config()
if "dashboard" not in config:
config["dashboard"] = {}
config["dashboard"]["font"] = font
save_config(config)
return {"ok": True, "font": font}
return await asyncio.to_thread(_run)
@@ -17288,20 +17304,21 @@ async def post_plugin_visibility(request: Request, name: str, body: _PluginVisib
name = _validate_plugin_name(name)
def _run():
config = load_config()
if "dashboard" not in config or not isinstance(config.get("dashboard"), dict):
config["dashboard"] = {}
hidden_list: list = config["dashboard"].get("hidden_plugins") or []
if not isinstance(hidden_list, list):
hidden_list = []
with _CONFIG_MUTATION_LOCK:
config = load_config()
if "dashboard" not in config or not isinstance(config.get("dashboard"), dict):
config["dashboard"] = {}
hidden_list: list = config["dashboard"].get("hidden_plugins") or []
if not isinstance(hidden_list, list):
hidden_list = []
if body.hidden and name not in hidden_list:
hidden_list.append(name)
elif not body.hidden and name in hidden_list:
hidden_list.remove(name)
if body.hidden and name not in hidden_list:
hidden_list.append(name)
elif not body.hidden and name in hidden_list:
hidden_list.remove(name)
config["dashboard"]["hidden_plugins"] = hidden_list
save_config(config)
config["dashboard"]["hidden_plugins"] = hidden_list
save_config(config)
_invalidate_plugins_hub_cache()
return {"ok": True, "name": name, "hidden": body.hidden}
@@ -102,3 +102,140 @@ class TestGetConfigOffLoop:
"_SKILLS_PROFILE_LOCK was held — /api/config is blocking the "
"loop again"
)
class TestRouterOffLoop:
"""Mounted routers (skills/mcp/tools) hold the same locks — they must not
do it on the event loop either (same bug class as /api/config)."""
@pytest.fixture(autouse=True)
def _home(self, _isolate_hermes_home):
pass
def test_get_skills_loop_stays_responsive_while_profile_lock_held(self):
try:
import httpx
except ImportError:
pytest.skip("httpx not installed")
from hermes_cli import web_server
hold_s = 1.0
release = threading.Event()
def _holder():
with web_server._SKILLS_PROFILE_LOCK:
release.wait(hold_s)
async def _scenario():
holder = threading.Thread(target=_holder)
holder.start()
await asyncio.sleep(0.05)
ticks = 0
async def _heartbeat(stop: asyncio.Event):
nonlocal ticks
while not stop.is_set():
await asyncio.sleep(0.02)
ticks += 1
stop = asyncio.Event()
hb = asyncio.create_task(_heartbeat(stop))
await asyncio.sleep(0)
transport = httpx.ASGITransport(app=web_server.app)
try:
async with httpx.AsyncClient(
transport=transport, base_url="http://testserver"
) as client:
client.headers[web_server._SESSION_HEADER_NAME] = (
web_server._SESSION_TOKEN
)
resp = await client.get("/api/skills")
finally:
stop.set()
release.set()
await hb
holder.join()
assert resp.status_code == 200
return ticks
ticks = asyncio.run(_scenario())
assert ticks >= 10, (
f"event loop heartbeat only ticked {ticks} time(s) while "
"_SKILLS_PROFILE_LOCK was held — GET /api/skills is blocking "
"the loop"
)
class TestConfigMutationLock:
"""Off-loop read-modify-write handlers must not lose concurrent updates.
config.py's _CONFIG_LOCK covers each load/save individually; the span
between them is serialized by web_server._CONFIG_MUTATION_LOCK. Two
concurrent writers touching DIFFERENT keys must both survive."""
@pytest.fixture(autouse=True)
def _home(self, _isolate_hermes_home):
pass
def test_concurrent_distinct_updates_both_survive(self):
try:
from starlette.testclient import TestClient
except ImportError:
pytest.skip("fastapi/starlette not installed")
from hermes_cli import web_server
from hermes_cli.config import load_config
client = TestClient(web_server.app)
client.headers[web_server._SESSION_HEADER_NAME] = web_server._SESSION_TOKEN
# Interleave the two handlers' load→mutate→save spans deterministically:
# gate the FIRST writer inside its mutation-lock hold, and assert the
# second writer cannot slip its own load in between. With the lock,
# writer B waits for writer A's full span; without it, B loads the
# pre-A config and its save erases A's write.
results = []
def _put_theme():
resp = client.put("/api/dashboard/theme", json={"name": "midnight"})
results.append(("theme", resp.status_code))
def _put_font():
resp = client.put("/api/dashboard/font", json={"font": "jetbrains-mono"})
results.append(("font", resp.status_code))
# Widen the race window: make save_config slow so an unserialized
# interleave is near-certain, not just possible.
real_save = web_server.save_config
def _slow_save(cfg, **kwargs):
time.sleep(0.15)
return real_save(cfg, **kwargs)
threads = []
try:
web_server.save_config = _slow_save
threads = [
threading.Thread(target=_put_theme),
threading.Thread(target=_put_font),
]
for t in threads:
t.start()
finally:
for t in threads:
t.join()
web_server.save_config = real_save
assert all(code == 200 for _, code in results), results
cfg = load_config()
dashboard = cfg.get("dashboard") or {}
# Both writes must be present — a lost update drops exactly one.
assert dashboard.get("theme") == "midnight", (
"theme write lost to a concurrent font write — "
"read-modify-write span is not serialized"
)
assert dashboard.get("font") == "jetbrains-mono", (
"font write lost to a concurrent theme write — "
"read-modify-write span is not serialized"
)