fix(gateway): serialize config mutations and finish the router off-loop sweep
Two follow-ups to the off-loop move, from external review (both verified, the second larger than reported): - Config read-modify-write handlers moved to worker threads could now interleave — _CONFIG_LOCK covers each load/save individually, never the span between them; the event loop used to serialize these accidentally. New _CONFIG_MUTATION_LOCK (worker-threads only, so it can never block the loop) held across the whole load→mutate→save span in all seven RMW handlers. update_config_raw skipped: it's a full-document replace with no server-side read, so a lock cannot close its client-side window. - The review flagged two skills routes still taking _SKILLS_PROFILE_LOCK on the event loop; a systematic audit of hermes_cli/web_routers/ found 24 on-loop routes (skills 5, mcp 9, tools 10, cron 1). All moved to the same inner-_run + asyncio.to_thread pattern, mutating ones under the mutation lock, uniform lock order (_SKILLS_PROFILE_LOCK → _CONFIG_MUTATION_LOCK). Await-safe _config_profile_scope routes, plain def routes, and already-threaded routes unchanged. Regression tests: concurrent theme+font updates both survive (fails with the lock nulled: "theme write lost to a concurrent font write"); event loop stays responsive while the profile lock is held during GET /api/skills. 214 tests passing across the touched suites. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -146,7 +146,7 @@ async def cron_fire_webhook(request: Request):
|
||||
auth = request.headers.get("Authorization", "")
|
||||
token = auth[7:].strip() if auth.startswith("Bearer ") else ""
|
||||
|
||||
cfg = load_config()
|
||||
cfg = await asyncio.to_thread(load_config)
|
||||
claims = get_fire_verifier()(
|
||||
token=token,
|
||||
expected_audience=cfg_get(cfg, "cron", "chronos", "expected_audience", default=""),
|
||||
|
||||
@@ -53,14 +53,20 @@ save_env_value = late("save_env_value")
|
||||
_mcp_oauth_flows = LateState("_mcp_oauth_flows")
|
||||
_mcp_oauth_flows_lock = LateState("_mcp_oauth_flows_lock")
|
||||
_MAX_PENDING_MCP_OAUTH_FLOWS = LateState("_MAX_PENDING_MCP_OAUTH_FLOWS")
|
||||
# Config read-modify-write serialization for off-loop handlers (defined in
|
||||
# web_server.py; LateState supports ``with``-blocks, so this is the live lock).
|
||||
_CONFIG_MUTATION_LOCK = LateState("_CONFIG_MUTATION_LOCK")
|
||||
|
||||
|
||||
@router.get("/api/mcp/servers")
|
||||
async def list_mcp_servers(profile: Optional[str] = None):
|
||||
from hermes_cli.mcp_config import _get_mcp_servers
|
||||
|
||||
with _profile_scope(profile):
|
||||
servers = _get_mcp_servers()
|
||||
def _read():
|
||||
with _profile_scope(profile):
|
||||
return _get_mcp_servers()
|
||||
|
||||
servers = await asyncio.to_thread(_read)
|
||||
return {
|
||||
"servers": [
|
||||
_mcp_server_summary(name, cfg) for name, cfg in sorted(servers.items())
|
||||
@@ -81,20 +87,29 @@ async def add_mcp_server(body: MCPServerCreate, profile: Optional[str] = None):
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
with _profile_scope(body.profile or profile):
|
||||
existing = _get_mcp_servers()
|
||||
def _check_existing():
|
||||
with _profile_scope(body.profile or profile):
|
||||
return _get_mcp_servers()
|
||||
|
||||
existing = await asyncio.to_thread(_check_existing)
|
||||
if name in existing:
|
||||
raise HTTPException(status_code=409, detail=f"Server '{name}' already exists")
|
||||
|
||||
try:
|
||||
def _save():
|
||||
with _profile_scope(body.profile or profile):
|
||||
if bearer_token is not None:
|
||||
server_config["headers"] = _save_bearer_auth_token(name, bearer_token)
|
||||
if not _save_mcp_server(name, server_config):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Server '{name}' rejected: suspicious command/args configuration",
|
||||
)
|
||||
# _save_mcp_server does its own load→mutate→save of config.yaml;
|
||||
# serialize the whole cycle against other off-loop config writers.
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
if bearer_token is not None:
|
||||
server_config["headers"] = _save_bearer_auth_token(name, bearer_token)
|
||||
if not _save_mcp_server(name, server_config):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Server '{name}' rejected: suspicious command/args configuration",
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(_save)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
@@ -116,8 +131,12 @@ async def replace_mcp_servers(body: MCPServersReplace, profile: Optional[str] =
|
||||
"""
|
||||
from hermes_cli.mcp_config import _replace_mcp_servers
|
||||
|
||||
with _profile_scope(body.profile or profile):
|
||||
ok, issues = _replace_mcp_servers(body.servers)
|
||||
def _run():
|
||||
with _profile_scope(body.profile or profile):
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
return _replace_mcp_servers(body.servers)
|
||||
|
||||
ok, issues = await asyncio.to_thread(_run)
|
||||
if not ok:
|
||||
raise HTTPException(status_code=400, detail="; ".join(issues))
|
||||
return {"ok": True}
|
||||
@@ -127,8 +146,12 @@ async def replace_mcp_servers(body: MCPServersReplace, profile: Optional[str] =
|
||||
async def remove_mcp_server(name: str, profile: Optional[str] = None):
|
||||
from hermes_cli.mcp_config import _remove_mcp_server
|
||||
|
||||
with _profile_scope(profile):
|
||||
removed = _remove_mcp_server(name)
|
||||
def _run():
|
||||
with _profile_scope(profile):
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
return _remove_mcp_server(name)
|
||||
|
||||
removed = await asyncio.to_thread(_run)
|
||||
if not removed:
|
||||
raise HTTPException(status_code=404, detail=f"Server '{name}' not found")
|
||||
return {"ok": True}
|
||||
@@ -143,8 +166,11 @@ async def test_mcp_server(name: str, profile: Optional[str] = None):
|
||||
_probe_single_server,
|
||||
)
|
||||
|
||||
with _profile_scope(profile):
|
||||
servers = _get_mcp_servers()
|
||||
def _read():
|
||||
with _profile_scope(profile):
|
||||
return _get_mcp_servers()
|
||||
|
||||
servers = await asyncio.to_thread(_read)
|
||||
if name not in servers:
|
||||
raise HTTPException(status_code=404, detail=f"Server '{name}' not found")
|
||||
|
||||
@@ -205,9 +231,12 @@ async def auth_mcp_server(name: str, request: Request, profile: Optional[str] =
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
process_home = str(get_hermes_home().expanduser().resolve(strict=False))
|
||||
with _profile_scope(profile):
|
||||
servers = _get_mcp_servers()
|
||||
flow_home = str(get_hermes_home().expanduser().resolve(strict=False))
|
||||
|
||||
def _read():
|
||||
with _profile_scope(profile):
|
||||
return _get_mcp_servers(), str(get_hermes_home().expanduser().resolve(strict=False))
|
||||
|
||||
servers, flow_home = await asyncio.to_thread(_read)
|
||||
if name not in servers:
|
||||
raise HTTPException(status_code=404, detail=f"Server '{name}' not found")
|
||||
cfg = dict(servers[name])
|
||||
@@ -325,16 +354,20 @@ async def set_mcp_server_enabled(
|
||||
flag the agent reads at startup. Disabled servers stay in config so they
|
||||
can be re-enabled without re-entering their settings.
|
||||
"""
|
||||
with _profile_scope(body.profile or profile):
|
||||
cfg = load_config()
|
||||
servers = cfg.get("mcp_servers")
|
||||
if not isinstance(servers, dict) or name not in servers:
|
||||
raise HTTPException(status_code=404, detail=f"Server '{name}' not found")
|
||||
if not isinstance(servers[name], dict):
|
||||
raise HTTPException(status_code=400, detail="Malformed server config")
|
||||
servers[name]["enabled"] = bool(body.enabled)
|
||||
save_config(cfg)
|
||||
return {"ok": True, "name": name, "enabled": bool(body.enabled)}
|
||||
def _run():
|
||||
with _profile_scope(body.profile or profile):
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
cfg = load_config()
|
||||
servers = cfg.get("mcp_servers")
|
||||
if not isinstance(servers, dict) or name not in servers:
|
||||
raise HTTPException(status_code=404, detail=f"Server '{name}' not found")
|
||||
if not isinstance(servers[name], dict):
|
||||
raise HTTPException(status_code=400, detail="Malformed server config")
|
||||
servers[name]["enabled"] = bool(body.enabled)
|
||||
save_config(cfg)
|
||||
return {"ok": True, "name": name, "enabled": bool(body.enabled)}
|
||||
|
||||
return await asyncio.to_thread(_run)
|
||||
|
||||
|
||||
@router.get("/api/mcp/catalog")
|
||||
@@ -355,12 +388,16 @@ async def list_mcp_catalog(profile: Optional[str] = None):
|
||||
|
||||
entries = []
|
||||
try:
|
||||
with _profile_scope(profile):
|
||||
catalog_entries = list(mcp_catalog.list_catalog())
|
||||
installed_state = {
|
||||
e.name: (mcp_catalog.is_installed(e.name), mcp_catalog.is_enabled(e.name))
|
||||
for e in catalog_entries
|
||||
}
|
||||
def _read():
|
||||
with _profile_scope(profile):
|
||||
catalog = list(mcp_catalog.list_catalog())
|
||||
state = {
|
||||
e.name: (mcp_catalog.is_installed(e.name), mcp_catalog.is_enabled(e.name))
|
||||
for e in catalog
|
||||
}
|
||||
return catalog, state
|
||||
|
||||
catalog_entries, installed_state = await asyncio.to_thread(_read)
|
||||
for entry in catalog_entries:
|
||||
auth = entry.auth
|
||||
transport = entry.transport
|
||||
@@ -434,10 +471,13 @@ async def install_mcp_catalog_entry(body: MCPCatalogInstall, profile: Optional[s
|
||||
# they need; we only write the ones the user provided).
|
||||
effective_profile = body.profile or profile
|
||||
if body.env:
|
||||
with _profile_scope(effective_profile):
|
||||
for k, v in body.env.items():
|
||||
if v:
|
||||
save_env_value(k, v)
|
||||
def _write_env():
|
||||
with _profile_scope(effective_profile):
|
||||
for k, v in body.env.items():
|
||||
if v:
|
||||
save_env_value(k, v)
|
||||
|
||||
await asyncio.to_thread(_write_env)
|
||||
|
||||
# Git-bootstrap entries can take a while to clone — run via the background
|
||||
# action path so the request returns immediately and the UI can tail logs.
|
||||
|
||||
@@ -49,6 +49,10 @@ load_config = late("load_config")
|
||||
# Live proxies for web_server-owned module state (mutations/monkeypatches
|
||||
# on web_server remain authoritative; resolved at operation time).
|
||||
_SKILL_HUB_SOURCE_LABELS = LateState("_SKILL_HUB_SOURCE_LABELS")
|
||||
# Config read-modify-write serialization for off-loop handlers (see the
|
||||
# definition in web_server.py). LateState supports ``with``-blocks, so this
|
||||
# is the live lock object, not a frozen import-time copy.
|
||||
_CONFIG_MUTATION_LOCK = LateState("_CONFIG_MUTATION_LOCK")
|
||||
|
||||
|
||||
@hub_router.post("/api/skills/hub/install")
|
||||
@@ -401,40 +405,48 @@ async def get_skills(profile: Optional[str] = None):
|
||||
activity_count,
|
||||
load_usage,
|
||||
)
|
||||
with _profile_scope(profile):
|
||||
config = load_config()
|
||||
disabled = get_disabled_skills(config)
|
||||
skills = _find_all_skills(skip_disabled=True)
|
||||
usage = load_usage()
|
||||
# Set-based provenance (same classification as skill_usage.provenance,
|
||||
# without a per-skill manifest read): hub > bundled > agent, where
|
||||
# "agent" covers agent-authored AND local hand-made skills — the ones
|
||||
# the user may edit/delete from the UI.
|
||||
bundled_names = _read_bundled_manifest_names()
|
||||
hub_names = _read_hub_installed_names()
|
||||
for s in skills:
|
||||
s["enabled"] = s["name"] not in disabled
|
||||
s["usage"] = activity_count(usage.get(s["name"], {}))
|
||||
s["provenance"] = (
|
||||
"hub" if s["name"] in hub_names
|
||||
else "bundled" if s["name"] in bundled_names
|
||||
else "agent"
|
||||
)
|
||||
return skills
|
||||
def _run():
|
||||
with _profile_scope(profile):
|
||||
config = load_config()
|
||||
disabled = get_disabled_skills(config)
|
||||
skills = _find_all_skills(skip_disabled=True)
|
||||
usage = load_usage()
|
||||
# Set-based provenance (same classification as skill_usage.provenance,
|
||||
# without a per-skill manifest read): hub > bundled > agent, where
|
||||
# "agent" covers agent-authored AND local hand-made skills — the ones
|
||||
# the user may edit/delete from the UI.
|
||||
bundled_names = _read_bundled_manifest_names()
|
||||
hub_names = _read_hub_installed_names()
|
||||
for s in skills:
|
||||
s["enabled"] = s["name"] not in disabled
|
||||
s["usage"] = activity_count(usage.get(s["name"], {}))
|
||||
s["provenance"] = (
|
||||
"hub" if s["name"] in hub_names
|
||||
else "bundled" if s["name"] in bundled_names
|
||||
else "agent"
|
||||
)
|
||||
return skills
|
||||
|
||||
return await asyncio.to_thread(_run)
|
||||
|
||||
|
||||
@router.put("/api/skills/toggle")
|
||||
async def toggle_skill(body: SkillToggle, profile: Optional[str] = None):
|
||||
from hermes_cli.skills_config import get_disabled_skills, save_disabled_skills
|
||||
with _profile_scope(body.profile or profile):
|
||||
config = load_config()
|
||||
disabled = get_disabled_skills(config)
|
||||
if body.enabled:
|
||||
disabled.discard(body.name)
|
||||
else:
|
||||
disabled.add(body.name)
|
||||
save_disabled_skills(config, disabled)
|
||||
return {"ok": True, "name": body.name, "enabled": body.enabled}
|
||||
|
||||
def _run():
|
||||
with _profile_scope(body.profile or profile):
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
config = load_config()
|
||||
disabled = get_disabled_skills(config)
|
||||
if body.enabled:
|
||||
disabled.discard(body.name)
|
||||
else:
|
||||
disabled.add(body.name)
|
||||
save_disabled_skills(config, disabled)
|
||||
return {"ok": True, "name": body.name, "enabled": body.enabled}
|
||||
|
||||
return await asyncio.to_thread(_run)
|
||||
|
||||
|
||||
@router.get("/api/skills/content")
|
||||
@@ -442,18 +454,21 @@ async def get_skill_content(name: str, profile: Optional[str] = None):
|
||||
"""Return the raw SKILL.md text for a skill, for the dashboard editor."""
|
||||
from tools.skill_manager_tool import _find_skill
|
||||
|
||||
with _profile_scope(profile):
|
||||
found = _find_skill(name)
|
||||
if not found:
|
||||
raise HTTPException(status_code=404, detail=f"Skill '{name}' not found.")
|
||||
skill_md = found["path"] / "SKILL.md"
|
||||
if not skill_md.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Skill '{name}' has no SKILL.md.")
|
||||
try:
|
||||
content = skill_md.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
return {"name": name, "content": content, "path": str(skill_md)}
|
||||
def _run():
|
||||
with _profile_scope(profile):
|
||||
found = _find_skill(name)
|
||||
if not found:
|
||||
raise HTTPException(status_code=404, detail=f"Skill '{name}' not found.")
|
||||
skill_md = found["path"] / "SKILL.md"
|
||||
if not skill_md.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Skill '{name}' has no SKILL.md.")
|
||||
try:
|
||||
content = skill_md.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
return {"name": name, "content": content, "path": str(skill_md)}
|
||||
|
||||
return await asyncio.to_thread(_run)
|
||||
|
||||
|
||||
@router.post("/api/skills")
|
||||
@@ -467,8 +482,11 @@ async def create_skill(body: SkillCreate):
|
||||
"""
|
||||
from tools.skill_manager_tool import _create_skill
|
||||
|
||||
with _profile_scope(body.profile):
|
||||
result = _create_skill(body.name, body.content, body.category or None)
|
||||
def _run():
|
||||
with _profile_scope(body.profile):
|
||||
return _create_skill(body.name, body.content, body.category or None)
|
||||
|
||||
result = await asyncio.to_thread(_run)
|
||||
if not result.get("success"):
|
||||
raise HTTPException(status_code=400, detail=result.get("error", "Failed to create skill."))
|
||||
_clear_skills_prompt_cache()
|
||||
@@ -480,8 +498,11 @@ async def update_skill_content(body: SkillContentUpdate):
|
||||
"""Replace the SKILL.md of an existing skill (full rewrite) from the editor."""
|
||||
from tools.skill_manager_tool import _edit_skill
|
||||
|
||||
with _profile_scope(body.profile):
|
||||
result = _edit_skill(body.name, body.content)
|
||||
def _run():
|
||||
with _profile_scope(body.profile):
|
||||
return _edit_skill(body.name, body.content)
|
||||
|
||||
result = await asyncio.to_thread(_run)
|
||||
if not result.get("success"):
|
||||
err = result.get("error", "Failed to update skill.")
|
||||
status = 404 if "not found" in str(err).lower() else 400
|
||||
|
||||
+328
-284
@@ -10,6 +10,7 @@ late-binding seam in :mod:`hermes_cli.web_deps` (``late`` for callables,
|
||||
stays authoritative.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys # noqa: F401 — used by handlers
|
||||
from typing import Any, Dict, List, Optional # noqa: F401
|
||||
@@ -48,6 +49,9 @@ save_config = late("save_config")
|
||||
_MODEL_CATALOG_TOOLSETS = LateState("_MODEL_CATALOG_TOOLSETS")
|
||||
_TERMINAL_BACKENDS = LateState("_TERMINAL_BACKENDS")
|
||||
_TERMINAL_BACKEND_NAMES = LateState("_TERMINAL_BACKEND_NAMES")
|
||||
# Config read-modify-write serialization for off-loop handlers (defined in
|
||||
# web_server.py; LateState supports ``with``-blocks, so this is the live lock).
|
||||
_CONFIG_MUTATION_LOCK = LateState("_CONFIG_MUTATION_LOCK")
|
||||
|
||||
|
||||
@router.get("/api/tools/toolsets")
|
||||
@@ -64,21 +68,25 @@ async def get_toolsets(profile: Optional[str] = None):
|
||||
from hermes_cli.platforms import platform_label
|
||||
from toolsets import resolve_toolset
|
||||
|
||||
with _profile_scope(profile):
|
||||
config = load_config()
|
||||
toolset_rows = _get_effective_configurable_toolsets()
|
||||
target_platforms = {
|
||||
_toolset_configuration_platform(name) for name, _, _ in toolset_rows
|
||||
}
|
||||
enabled_by_platform = {
|
||||
platform: _get_platform_tools(
|
||||
config,
|
||||
platform,
|
||||
include_default_mcp_servers=False,
|
||||
)
|
||||
for platform in target_platforms
|
||||
}
|
||||
features = get_nous_subscription_features(config)
|
||||
def _read():
|
||||
with _profile_scope(profile):
|
||||
config = load_config()
|
||||
toolset_rows = _get_effective_configurable_toolsets()
|
||||
target_platforms = {
|
||||
_toolset_configuration_platform(name) for name, _, _ in toolset_rows
|
||||
}
|
||||
enabled_by_platform = {
|
||||
platform: _get_platform_tools(
|
||||
config,
|
||||
platform,
|
||||
include_default_mcp_servers=False,
|
||||
)
|
||||
for platform in target_platforms
|
||||
}
|
||||
features = get_nous_subscription_features(config)
|
||||
return config, toolset_rows, enabled_by_platform, features
|
||||
|
||||
config, toolset_rows, enabled_by_platform, features = await asyncio.to_thread(_read)
|
||||
result = []
|
||||
for name, label, desc in toolset_rows:
|
||||
try:
|
||||
@@ -135,37 +143,38 @@ async def toggle_toolset(name: str, body: ToolsetToggle, profile: Optional[str]
|
||||
raise HTTPException(status_code=400, detail=f"Unknown toolset: {name}")
|
||||
|
||||
target_platform = _toolset_configuration_platform(name)
|
||||
if name in _CONFIG_ONLY_TOOLSETS:
|
||||
# Config-only capabilities (stt) toggle their own config section's
|
||||
# ``enabled`` flag — there is no platform_toolsets entry to write.
|
||||
|
||||
def _run():
|
||||
if name in _CONFIG_ONLY_TOOLSETS:
|
||||
# Config-only capabilities (stt) toggle their own config section's
|
||||
# ``enabled`` flag — there is no platform_toolsets entry to write.
|
||||
with _profile_scope(body.profile or profile):
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
config = load_config()
|
||||
section = config.setdefault(name, {})
|
||||
if not isinstance(section, dict):
|
||||
section = {}
|
||||
config[name] = section
|
||||
section["enabled"] = bool(body.enabled)
|
||||
save_config(config)
|
||||
return
|
||||
with _profile_scope(body.profile or profile):
|
||||
config = load_config()
|
||||
section = config.setdefault(name, {})
|
||||
if not isinstance(section, dict):
|
||||
section = {}
|
||||
config[name] = section
|
||||
section["enabled"] = bool(body.enabled)
|
||||
save_config(config)
|
||||
return {
|
||||
"ok": True,
|
||||
"name": name,
|
||||
"platform": target_platform,
|
||||
"enabled": body.enabled,
|
||||
}
|
||||
with _profile_scope(body.profile or profile):
|
||||
config = load_config()
|
||||
enabled = set(
|
||||
_get_platform_tools(
|
||||
config,
|
||||
target_platform,
|
||||
include_default_mcp_servers=False,
|
||||
)
|
||||
)
|
||||
if body.enabled:
|
||||
enabled.add(name)
|
||||
else:
|
||||
enabled.discard(name)
|
||||
_save_platform_tools(config, target_platform, enabled)
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
config = load_config()
|
||||
enabled = set(
|
||||
_get_platform_tools(
|
||||
config,
|
||||
target_platform,
|
||||
include_default_mcp_servers=False,
|
||||
)
|
||||
)
|
||||
if body.enabled:
|
||||
enabled.add(name)
|
||||
else:
|
||||
enabled.discard(name)
|
||||
_save_platform_tools(config, target_platform, enabled)
|
||||
|
||||
await asyncio.to_thread(_run)
|
||||
return {
|
||||
"ok": True,
|
||||
"name": name,
|
||||
@@ -199,79 +208,84 @@ async def get_toolset_config(name: str, profile: Optional[str] = None):
|
||||
if name not in valid:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown toolset: {name}")
|
||||
|
||||
with _profile_scope(profile):
|
||||
config = load_config()
|
||||
cat = TOOL_CATEGORIES.get(name)
|
||||
providers = []
|
||||
active_provider = None
|
||||
active_search_backend = None
|
||||
active_extract_backend = None
|
||||
if cat:
|
||||
# Fetch portal/entitlement state once for the whole matrix — the
|
||||
# per-provider readiness computation below reuses it instead of
|
||||
# re-probing per row.
|
||||
features = get_nous_subscription_features(config, force_fresh=True)
|
||||
for prov in _visible_providers(cat, config, force_fresh=True):
|
||||
env_vars = [
|
||||
{
|
||||
"key": e["key"],
|
||||
"prompt": e.get("prompt", e["key"]),
|
||||
"url": e.get("url"),
|
||||
"default": e.get("default"),
|
||||
"is_set": bool(get_env_value(e["key"])),
|
||||
def _read():
|
||||
with _profile_scope(profile):
|
||||
config = load_config()
|
||||
cat = TOOL_CATEGORIES.get(name)
|
||||
providers = []
|
||||
active_provider = None
|
||||
active_search_backend = None
|
||||
active_extract_backend = None
|
||||
if cat:
|
||||
# Fetch portal/entitlement state once for the whole matrix — the
|
||||
# per-provider readiness computation below reuses it instead of
|
||||
# re-probing per row.
|
||||
features = get_nous_subscription_features(config, force_fresh=True)
|
||||
for prov in _visible_providers(cat, config, force_fresh=True):
|
||||
env_vars = [
|
||||
{
|
||||
"key": e["key"],
|
||||
"prompt": e.get("prompt", e["key"]),
|
||||
"url": e.get("url"),
|
||||
"default": e.get("default"),
|
||||
"is_set": bool(get_env_value(e["key"])),
|
||||
}
|
||||
for e in prov.get("env_vars", [])
|
||||
]
|
||||
# Surface the same active-provider determination the CLI picker
|
||||
# uses (``_is_provider_active``) so the GUI highlights the provider
|
||||
# actually written to config (e.g. web.backend), not just the first
|
||||
# keyless one in the list.
|
||||
is_active = _is_provider_active(prov, config, force_fresh=True)
|
||||
if is_active and active_provider is None:
|
||||
active_provider = prov["name"]
|
||||
row = {
|
||||
"name": prov["name"],
|
||||
"badge": prov.get("badge", ""),
|
||||
"tag": prov.get("tag", ""),
|
||||
"env_vars": env_vars,
|
||||
"post_setup": prov.get("post_setup"),
|
||||
"requires_nous_auth": bool(prov.get("requires_nous_auth")),
|
||||
"is_active": is_active,
|
||||
# Honest server-side readiness. The GUI's old client-side
|
||||
# heuristic showed "Ready" for every zero-env-var row —
|
||||
# including logged-out Nous Subscription rows and never-run
|
||||
# post_setup installs (see provider_readiness_status).
|
||||
"status": provider_readiness_status(
|
||||
prov, config, features=features, is_active=is_active
|
||||
),
|
||||
}
|
||||
for e in prov.get("env_vars", [])
|
||||
]
|
||||
# Surface the same active-provider determination the CLI picker
|
||||
# uses (``_is_provider_active``) so the GUI highlights the provider
|
||||
# actually written to config (e.g. web.backend), not just the first
|
||||
# keyless one in the list.
|
||||
is_active = _is_provider_active(prov, config, force_fresh=True)
|
||||
if is_active and active_provider is None:
|
||||
active_provider = prov["name"]
|
||||
row = {
|
||||
"name": prov["name"],
|
||||
"badge": prov.get("badge", ""),
|
||||
"tag": prov.get("tag", ""),
|
||||
"env_vars": env_vars,
|
||||
"post_setup": prov.get("post_setup"),
|
||||
"requires_nous_auth": bool(prov.get("requires_nous_auth")),
|
||||
"is_active": is_active,
|
||||
# Honest server-side readiness. The GUI's old client-side
|
||||
# heuristic showed "Ready" for every zero-env-var row —
|
||||
# including logged-out Nous Subscription rows and never-run
|
||||
# post_setup installs (see provider_readiness_status).
|
||||
"status": provider_readiness_status(
|
||||
prov, config, features=features, is_active=is_active
|
||||
),
|
||||
}
|
||||
if name == "web" and prov.get("web_backend"):
|
||||
# The runtime split web into two capabilities long ago
|
||||
# (web.search_backend / web.extract_backend); surface each
|
||||
# row's backend key and which capabilities it can serve so
|
||||
# the GUI can offer per-capability selection.
|
||||
row["web_backend"] = prov["web_backend"]
|
||||
row["capabilities"] = web_provider_capabilities(prov["web_backend"])
|
||||
if name == "tts" and prov.get("tts_provider"):
|
||||
# The provider key written to tts.provider on selection.
|
||||
# Doubles as the config section holding the provider's
|
||||
# voice/model settings (tts.<key>.*) so the GUI can render
|
||||
# those fields inline in the Capabilities panel.
|
||||
row["tts_provider"] = prov["tts_provider"]
|
||||
providers.append(row)
|
||||
if name == "web":
|
||||
# Resolve the per-capability active backends exactly the way the
|
||||
# web_search / web_extract dispatchers do (per-capability key →
|
||||
# shared web.backend → credential auto-detect), so the GUI badges
|
||||
# reflect what a tool call would actually hit right now.
|
||||
try:
|
||||
from tools.web_tools import _get_extract_backend, _get_search_backend
|
||||
if name == "web" and prov.get("web_backend"):
|
||||
# The runtime split web into two capabilities long ago
|
||||
# (web.search_backend / web.extract_backend); surface each
|
||||
# row's backend key and which capabilities it can serve so
|
||||
# the GUI can offer per-capability selection.
|
||||
row["web_backend"] = prov["web_backend"]
|
||||
row["capabilities"] = web_provider_capabilities(prov["web_backend"])
|
||||
if name == "tts" and prov.get("tts_provider"):
|
||||
# The provider key written to tts.provider on selection.
|
||||
# Doubles as the config section holding the provider's
|
||||
# voice/model settings (tts.<key>.*) so the GUI can render
|
||||
# those fields inline in the Capabilities panel.
|
||||
row["tts_provider"] = prov["tts_provider"]
|
||||
providers.append(row)
|
||||
if name == "web":
|
||||
# Resolve the per-capability active backends exactly the way the
|
||||
# web_search / web_extract dispatchers do (per-capability key →
|
||||
# shared web.backend → credential auto-detect), so the GUI badges
|
||||
# reflect what a tool call would actually hit right now.
|
||||
try:
|
||||
from tools.web_tools import _get_extract_backend, _get_search_backend
|
||||
|
||||
active_search_backend = _get_search_backend()
|
||||
active_extract_backend = _get_extract_backend()
|
||||
except Exception:
|
||||
active_search_backend = None
|
||||
active_extract_backend = None
|
||||
return cat, providers, active_provider, active_search_backend, active_extract_backend
|
||||
|
||||
cat, providers, active_provider, active_search_backend, active_extract_backend = await asyncio.to_thread(_read)
|
||||
|
||||
active_search_backend = _get_search_backend()
|
||||
active_extract_backend = _get_extract_backend()
|
||||
except Exception:
|
||||
active_search_backend = None
|
||||
active_extract_backend = None
|
||||
payload = {
|
||||
"name": name,
|
||||
"has_category": cat is not None,
|
||||
@@ -300,28 +314,35 @@ async def get_toolset_models(
|
||||
if section is None:
|
||||
return {"name": name, "has_models": False, "models": [], "current": None, "default": None}
|
||||
|
||||
with _profile_scope(profile):
|
||||
config = load_config()
|
||||
row = _find_toolset_provider_row(name, config, provider)
|
||||
plugin = _resolve_toolset_model_plugin(name, row) if row else None
|
||||
if not plugin:
|
||||
return {
|
||||
"name": name,
|
||||
"has_models": False,
|
||||
"models": [],
|
||||
"current": None,
|
||||
"default": None,
|
||||
}
|
||||
def _read():
|
||||
with _profile_scope(profile):
|
||||
config = load_config()
|
||||
row = _find_toolset_provider_row(name, config, provider)
|
||||
plugin = _resolve_toolset_model_plugin(name, row) if row else None
|
||||
if not plugin:
|
||||
return None
|
||||
|
||||
catalog, default_model = _toolset_model_catalog(name, plugin)
|
||||
section_cfg = config.get(section)
|
||||
current = None
|
||||
if isinstance(section_cfg, dict):
|
||||
raw = section_cfg.get("model")
|
||||
if isinstance(raw, str) and raw.strip():
|
||||
current = raw.strip()
|
||||
if current not in catalog:
|
||||
current = default_model if default_model in catalog else None
|
||||
catalog, default_model = _toolset_model_catalog(name, plugin)
|
||||
section_cfg = config.get(section)
|
||||
current = None
|
||||
if isinstance(section_cfg, dict):
|
||||
raw = section_cfg.get("model")
|
||||
if isinstance(raw, str) and raw.strip():
|
||||
current = raw.strip()
|
||||
if current not in catalog:
|
||||
current = default_model if default_model in catalog else None
|
||||
return row, plugin, catalog, default_model, current
|
||||
|
||||
resolved = await asyncio.to_thread(_read)
|
||||
if resolved is None:
|
||||
return {
|
||||
"name": name,
|
||||
"has_models": False,
|
||||
"models": [],
|
||||
"current": None,
|
||||
"default": None,
|
||||
}
|
||||
row, plugin, catalog, default_model, current = resolved
|
||||
|
||||
models = [
|
||||
{
|
||||
@@ -364,30 +385,34 @@ async def select_toolset_model(
|
||||
if not model_id:
|
||||
raise HTTPException(status_code=400, detail="model is required")
|
||||
|
||||
with _profile_scope(body.profile or profile):
|
||||
config = load_config()
|
||||
row = _find_toolset_provider_row(name, config, body.provider)
|
||||
plugin = _resolve_toolset_model_plugin(name, row) if row else None
|
||||
if not plugin:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"No model-capable backend is active for {name}",
|
||||
)
|
||||
def _run():
|
||||
with _profile_scope(body.profile or profile):
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
config = load_config()
|
||||
row = _find_toolset_provider_row(name, config, body.provider)
|
||||
plugin = _resolve_toolset_model_plugin(name, row) if row else None
|
||||
if not plugin:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"No model-capable backend is active for {name}",
|
||||
)
|
||||
|
||||
catalog, _default = _toolset_model_catalog(name, plugin)
|
||||
if model_id not in catalog:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unknown model {model_id!r} for backend {plugin!r}",
|
||||
)
|
||||
catalog, _default = _toolset_model_catalog(name, plugin)
|
||||
if model_id not in catalog:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unknown model {model_id!r} for backend {plugin!r}",
|
||||
)
|
||||
|
||||
section_cfg = config.setdefault(section, {})
|
||||
if not isinstance(section_cfg, dict):
|
||||
section_cfg = {}
|
||||
config[section] = section_cfg
|
||||
section_cfg["model"] = model_id
|
||||
save_config(config)
|
||||
section_cfg = config.setdefault(section, {})
|
||||
if not isinstance(section_cfg, dict):
|
||||
section_cfg = {}
|
||||
config[section] = section_cfg
|
||||
section_cfg["model"] = model_id
|
||||
save_config(config)
|
||||
return plugin
|
||||
|
||||
plugin = await asyncio.to_thread(_run)
|
||||
return {"ok": True, "name": name, "model": model_id, "plugin": plugin}
|
||||
|
||||
|
||||
@@ -450,77 +475,82 @@ async def select_toolset_provider(
|
||||
detail=f"Unknown capability: {body.capability!r} (expected 'search' or 'extract')",
|
||||
)
|
||||
|
||||
with _profile_scope(body.profile or profile):
|
||||
config = load_config()
|
||||
if body.capability is not None:
|
||||
# Per-capability path: resolve the picker row to its backend key
|
||||
# and write web.<capability>_backend. Does NOT touch web.backend,
|
||||
# so the other capability keeps resolving through the shared
|
||||
# fallback chain.
|
||||
cat = TOOL_CATEGORIES.get(name)
|
||||
providers = _visible_providers(cat, config, force_fresh=True) if cat else []
|
||||
prov = next((p for p in providers if p.get("name") == body.provider), None)
|
||||
if prov is None:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unknown provider {body.provider!r} for toolset {name!r}",
|
||||
)
|
||||
backend = prov.get("web_backend")
|
||||
if not backend:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Provider {body.provider!r} has no web backend key",
|
||||
)
|
||||
if body.capability not in web_provider_capabilities(backend):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"{body.provider} does not support {body.capability}",
|
||||
)
|
||||
web_cfg = config.setdefault("web", {})
|
||||
if not isinstance(web_cfg, dict):
|
||||
web_cfg = {}
|
||||
config["web"] = web_cfg
|
||||
web_cfg[f"{body.capability}_backend"] = backend
|
||||
else:
|
||||
try:
|
||||
apply_provider_selection(name, body.provider, config)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc).strip('"'))
|
||||
save_config(config)
|
||||
response: Dict[str, Any] = {"ok": True, "name": name, "provider": body.provider}
|
||||
if body.capability is not None:
|
||||
response["capability"] = body.capability
|
||||
def _run():
|
||||
with _profile_scope(body.profile or profile):
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
config = load_config()
|
||||
if body.capability is not None:
|
||||
# Per-capability path: resolve the picker row to its backend key
|
||||
# and write web.<capability>_backend. Does NOT touch web.backend,
|
||||
# so the other capability keeps resolving through the shared
|
||||
# fallback chain.
|
||||
cat = TOOL_CATEGORIES.get(name)
|
||||
providers = _visible_providers(cat, config, force_fresh=True) if cat else []
|
||||
prov = next((p for p in providers if p.get("name") == body.provider), None)
|
||||
if prov is None:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unknown provider {body.provider!r} for toolset {name!r}",
|
||||
)
|
||||
backend = prov.get("web_backend")
|
||||
if not backend:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Provider {body.provider!r} has no web backend key",
|
||||
)
|
||||
if body.capability not in web_provider_capabilities(backend):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"{body.provider} does not support {body.capability}",
|
||||
)
|
||||
web_cfg = config.setdefault("web", {})
|
||||
if not isinstance(web_cfg, dict):
|
||||
web_cfg = {}
|
||||
config["web"] = web_cfg
|
||||
web_cfg[f"{body.capability}_backend"] = backend
|
||||
else:
|
||||
try:
|
||||
apply_provider_selection(name, body.provider, config)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc).strip('"'))
|
||||
save_config(config)
|
||||
response: Dict[str, Any] = {"ok": True, "name": name, "provider": body.provider}
|
||||
if body.capability is not None:
|
||||
response["capability"] = body.capability
|
||||
|
||||
# Entitlement check for managed Nous rows — mirrors the gate the CLI
|
||||
# applies via ensure_nous_portal_access at selection time.
|
||||
cat = TOOL_CATEGORIES.get(name)
|
||||
row = None
|
||||
if cat:
|
||||
row = next(
|
||||
(
|
||||
p
|
||||
for p in _visible_providers(cat, config, force_fresh=True)
|
||||
if p.get("name") == body.provider
|
||||
),
|
||||
None,
|
||||
)
|
||||
managed_feature = (row or {}).get("managed_nous_feature")
|
||||
if managed_feature:
|
||||
features = get_nous_subscription_features(config, force_fresh=True)
|
||||
acct = features.account_info
|
||||
category = MANAGED_FEATURE_COVERAGE_CATEGORY.get(managed_feature)
|
||||
entitled = bool(
|
||||
acct
|
||||
and acct.logged_in
|
||||
and (
|
||||
acct.tool_gateway_entitled_for(category)
|
||||
if category
|
||||
else acct.tool_gateway_entitled
|
||||
)
|
||||
)
|
||||
if not entitled:
|
||||
response["needs_nous_auth"] = True
|
||||
response["feature"] = managed_feature
|
||||
# Entitlement check for managed Nous rows — mirrors the gate the CLI
|
||||
# applies via ensure_nous_portal_access at selection time.
|
||||
cat = TOOL_CATEGORIES.get(name)
|
||||
row = None
|
||||
if cat:
|
||||
row = next(
|
||||
(
|
||||
p
|
||||
for p in _visible_providers(cat, config, force_fresh=True)
|
||||
if p.get("name") == body.provider
|
||||
),
|
||||
None,
|
||||
)
|
||||
managed_feature = (row or {}).get("managed_nous_feature")
|
||||
if managed_feature:
|
||||
features = get_nous_subscription_features(config, force_fresh=True)
|
||||
acct = features.account_info
|
||||
category = MANAGED_FEATURE_COVERAGE_CATEGORY.get(managed_feature)
|
||||
entitled = bool(
|
||||
acct
|
||||
and acct.logged_in
|
||||
and (
|
||||
acct.tool_gateway_entitled_for(category)
|
||||
if category
|
||||
else acct.tool_gateway_entitled
|
||||
)
|
||||
)
|
||||
if not entitled:
|
||||
response["needs_nous_auth"] = True
|
||||
response["feature"] = managed_feature
|
||||
return response
|
||||
|
||||
response = await asyncio.to_thread(_run)
|
||||
return response
|
||||
|
||||
|
||||
@@ -547,35 +577,39 @@ async def save_toolset_env(name: str, body: ToolsetEnvUpdate, profile: Optional[
|
||||
if name not in valid_ts:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown toolset: {name}")
|
||||
|
||||
with _profile_scope(body.profile or profile):
|
||||
config = load_config()
|
||||
cat = TOOL_CATEGORIES.get(name)
|
||||
allowed: set[str] = set()
|
||||
if cat:
|
||||
for prov in _visible_providers(cat, config, force_fresh=True):
|
||||
for e in prov.get("env_vars", []):
|
||||
allowed.add(e["key"])
|
||||
def _run():
|
||||
with _profile_scope(body.profile or profile):
|
||||
config = load_config()
|
||||
cat = TOOL_CATEGORIES.get(name)
|
||||
allowed: set[str] = set()
|
||||
if cat:
|
||||
for prov in _visible_providers(cat, config, force_fresh=True):
|
||||
for e in prov.get("env_vars", []):
|
||||
allowed.add(e["key"])
|
||||
|
||||
unknown = [k for k in body.env if k not in allowed]
|
||||
if unknown:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unknown env var(s) for toolset {name}: {', '.join(sorted(unknown))}",
|
||||
)
|
||||
unknown = [k for k in body.env if k not in allowed]
|
||||
if unknown:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Unknown env var(s) for toolset {name}: {', '.join(sorted(unknown))}",
|
||||
)
|
||||
|
||||
saved: List[str] = []
|
||||
skipped: List[str] = []
|
||||
for key, value in body.env.items():
|
||||
if value and value.strip():
|
||||
try:
|
||||
save_env_value(key, value.strip())
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
saved.append(key)
|
||||
else:
|
||||
skipped.append(key)
|
||||
saved: List[str] = []
|
||||
skipped: List[str] = []
|
||||
for key, value in body.env.items():
|
||||
if value and value.strip():
|
||||
try:
|
||||
save_env_value(key, value.strip())
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
saved.append(key)
|
||||
else:
|
||||
skipped.append(key)
|
||||
|
||||
status = {k: bool(get_env_value(k)) for k in allowed}
|
||||
status = {k: bool(get_env_value(k)) for k in allowed}
|
||||
return saved, skipped, status
|
||||
|
||||
saved, skipped, status = await asyncio.to_thread(_run)
|
||||
return {"ok": True, "name": name, "saved": saved, "skipped": skipped, "is_set": status}
|
||||
|
||||
|
||||
@@ -639,27 +673,30 @@ async def get_terminal_backends(profile: Optional[str] = None):
|
||||
Probes are fast (<~2s each) and defensive — a probe failure surfaces as a
|
||||
status, never an error response.
|
||||
"""
|
||||
with _profile_scope(profile):
|
||||
config = load_config()
|
||||
terminal_cfg = config.get("terminal")
|
||||
if not isinstance(terminal_cfg, dict):
|
||||
terminal_cfg = {}
|
||||
active = str(terminal_cfg.get("backend") or "local").strip().lower()
|
||||
if active not in _TERMINAL_BACKEND_NAMES:
|
||||
active = "local"
|
||||
def _read():
|
||||
with _profile_scope(profile):
|
||||
config = load_config()
|
||||
terminal_cfg = config.get("terminal")
|
||||
if not isinstance(terminal_cfg, dict):
|
||||
terminal_cfg = {}
|
||||
active = str(terminal_cfg.get("backend") or "local").strip().lower()
|
||||
if active not in _TERMINAL_BACKEND_NAMES:
|
||||
active = "local"
|
||||
|
||||
backends = []
|
||||
for row in _TERMINAL_BACKENDS:
|
||||
status, detail = _probe_terminal_backend(row["name"], terminal_cfg)
|
||||
backends.append({
|
||||
"name": row["name"],
|
||||
"label": row["label"],
|
||||
"description": row["description"],
|
||||
"active": row["name"] == active,
|
||||
"status": status,
|
||||
"detail": detail,
|
||||
})
|
||||
return {"active": active, "backends": backends}
|
||||
backends = []
|
||||
for row in _TERMINAL_BACKENDS:
|
||||
status, detail = _probe_terminal_backend(row["name"], terminal_cfg)
|
||||
backends.append({
|
||||
"name": row["name"],
|
||||
"label": row["label"],
|
||||
"description": row["description"],
|
||||
"active": row["name"] == active,
|
||||
"status": status,
|
||||
"detail": detail,
|
||||
})
|
||||
return {"active": active, "backends": backends}
|
||||
|
||||
return await asyncio.to_thread(_read)
|
||||
|
||||
|
||||
@router.put("/api/tools/terminal/backend")
|
||||
@@ -680,14 +717,18 @@ async def select_terminal_backend(
|
||||
f"Use one of: {', '.join(sorted(_TERMINAL_BACKEND_NAMES))}",
|
||||
)
|
||||
|
||||
with _profile_scope(body.profile or profile):
|
||||
config = load_config()
|
||||
terminal_cfg = config.setdefault("terminal", {})
|
||||
if not isinstance(terminal_cfg, dict):
|
||||
terminal_cfg = {}
|
||||
config["terminal"] = terminal_cfg
|
||||
terminal_cfg["backend"] = backend
|
||||
save_config(config)
|
||||
def _run():
|
||||
with _profile_scope(body.profile or profile):
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
config = load_config()
|
||||
terminal_cfg = config.setdefault("terminal", {})
|
||||
if not isinstance(terminal_cfg, dict):
|
||||
terminal_cfg = {}
|
||||
config["terminal"] = terminal_cfg
|
||||
terminal_cfg["backend"] = backend
|
||||
save_config(config)
|
||||
|
||||
await asyncio.to_thread(_run)
|
||||
return {"ok": True, "backend": backend}
|
||||
|
||||
|
||||
@@ -701,8 +742,11 @@ async def get_computer_use_status(profile: Optional[str] = None):
|
||||
"""
|
||||
from tools.computer_use.permissions import computer_use_status
|
||||
|
||||
with _profile_scope(profile):
|
||||
return computer_use_status()
|
||||
def _read():
|
||||
with _profile_scope(profile):
|
||||
return computer_use_status()
|
||||
|
||||
return await asyncio.to_thread(_read)
|
||||
|
||||
|
||||
@router.post("/api/tools/computer-use/permissions/grant")
|
||||
|
||||
+77
-60
@@ -2966,6 +2966,16 @@ _TOPOLOGY_CACHE: Dict[str, Any] = {"ts": 0.0, "data": None, "fn": None}
|
||||
_TOPOLOGY_CACHE_LOCK = threading.Lock()
|
||||
_TOPOLOGY_CACHE_TTL = 10.0
|
||||
|
||||
# Serializes read-modify-write cycles over config.yaml for handlers that run
|
||||
# in worker threads (asyncio.to_thread). config.py's _CONFIG_LOCK covers each
|
||||
# load_config()/save_config() call individually, not the span between them —
|
||||
# when these handlers ran on the event loop the loop itself serialized the
|
||||
# whole cycle, but off-loop two concurrent updates could interleave
|
||||
# load→mutate→save and silently drop one another's writes. Held only in
|
||||
# worker threads, so it can never block the event loop. RLock so a locked
|
||||
# section that calls helpers which also take it can't self-deadlock.
|
||||
_CONFIG_MUTATION_LOCK = threading.RLock()
|
||||
|
||||
|
||||
def _topology_cache_get(fn: Any) -> Optional[Dict[str, Any]]:
|
||||
if (
|
||||
@@ -6959,9 +6969,10 @@ async def update_config(body: ConfigUpdate, profile: Optional[str] = None):
|
||||
# is not sent in the PUT body. A full-replace save would silently
|
||||
# drop those keys. Deep-merge incoming over what's on disk so the
|
||||
# frontend can only overwrite what it explicitly sends.
|
||||
existing = read_raw_config()
|
||||
incoming = _denormalize_config_from_web(body.config)
|
||||
save_config(_deep_merge(existing, incoming))
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
existing = read_raw_config()
|
||||
incoming = _denormalize_config_from_web(body.config)
|
||||
save_config(_deep_merge(existing, incoming))
|
||||
return {"ok": True}
|
||||
|
||||
try:
|
||||
@@ -12924,11 +12935,12 @@ async def set_memory_provider(body: MemoryProviderSelect):
|
||||
def _run():
|
||||
_require_memory_provider_ready(provider)
|
||||
|
||||
cfg = load_config()
|
||||
if not isinstance(cfg.get("memory"), dict):
|
||||
cfg["memory"] = {}
|
||||
cfg["memory"]["provider"] = provider
|
||||
save_config(cfg)
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
cfg = load_config()
|
||||
if not isinstance(cfg.get("memory"), dict):
|
||||
cfg["memory"] = {}
|
||||
cfg["memory"]["provider"] = provider
|
||||
save_config(cfg)
|
||||
return {"ok": True, "active": provider}
|
||||
|
||||
return await asyncio.to_thread(_run)
|
||||
@@ -13237,23 +13249,24 @@ async def create_hook(body: HookCreate):
|
||||
pass
|
||||
|
||||
def _run():
|
||||
cfg = load_config()
|
||||
hooks_cfg = cfg.get("hooks")
|
||||
if not isinstance(hooks_cfg, dict):
|
||||
hooks_cfg = {}
|
||||
cfg["hooks"] = hooks_cfg
|
||||
entries = hooks_cfg.get(event)
|
||||
if not isinstance(entries, list):
|
||||
entries = []
|
||||
hooks_cfg[event] = entries
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
cfg = load_config()
|
||||
hooks_cfg = cfg.get("hooks")
|
||||
if not isinstance(hooks_cfg, dict):
|
||||
hooks_cfg = {}
|
||||
cfg["hooks"] = hooks_cfg
|
||||
entries = hooks_cfg.get(event)
|
||||
if not isinstance(entries, list):
|
||||
entries = []
|
||||
hooks_cfg[event] = entries
|
||||
|
||||
new_entry: Dict[str, Any] = {"command": command}
|
||||
if body.matcher:
|
||||
new_entry["matcher"] = body.matcher
|
||||
if body.timeout is not None:
|
||||
new_entry["timeout"] = int(body.timeout)
|
||||
entries.append(new_entry)
|
||||
save_config(cfg)
|
||||
new_entry: Dict[str, Any] = {"command": command}
|
||||
if body.matcher:
|
||||
new_entry["matcher"] = body.matcher
|
||||
if body.timeout is not None:
|
||||
new_entry["timeout"] = int(body.timeout)
|
||||
entries.append(new_entry)
|
||||
save_config(cfg)
|
||||
|
||||
approved = False
|
||||
if body.approve:
|
||||
@@ -13279,21 +13292,22 @@ async def delete_hook(body: HookDelete):
|
||||
raise HTTPException(status_code=400, detail="event and command are required")
|
||||
|
||||
def _run():
|
||||
cfg = load_config()
|
||||
hooks_cfg = cfg.get("hooks")
|
||||
removed = False
|
||||
if isinstance(hooks_cfg, dict) and isinstance(hooks_cfg.get(event), list):
|
||||
before = len(hooks_cfg[event])
|
||||
hooks_cfg[event] = [
|
||||
e for e in hooks_cfg[event]
|
||||
if not (isinstance(e, dict) and e.get("command") == command)
|
||||
]
|
||||
removed = len(hooks_cfg[event]) < before
|
||||
if not hooks_cfg[event]:
|
||||
del hooks_cfg[event]
|
||||
if not hooks_cfg:
|
||||
cfg.pop("hooks", None)
|
||||
save_config(cfg)
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
cfg = load_config()
|
||||
hooks_cfg = cfg.get("hooks")
|
||||
if isinstance(hooks_cfg, dict) and isinstance(hooks_cfg.get(event), list):
|
||||
before = len(hooks_cfg[event])
|
||||
hooks_cfg[event] = [
|
||||
e for e in hooks_cfg[event]
|
||||
if not (isinstance(e, dict) and e.get("command") == command)
|
||||
]
|
||||
removed = len(hooks_cfg[event]) < before
|
||||
if not hooks_cfg[event]:
|
||||
del hooks_cfg[event]
|
||||
if not hooks_cfg:
|
||||
cfg.pop("hooks", None)
|
||||
save_config(cfg)
|
||||
|
||||
# Revoke consent regardless so a re-add re-prompts.
|
||||
try:
|
||||
@@ -16683,11 +16697,12 @@ async def get_dashboard_themes():
|
||||
async def set_dashboard_theme(body: ThemeSetBody):
|
||||
"""Set the active dashboard theme (persists to config.yaml)."""
|
||||
def _run():
|
||||
config = load_config()
|
||||
if "dashboard" not in config:
|
||||
config["dashboard"] = {}
|
||||
config["dashboard"]["theme"] = body.name
|
||||
save_config(config)
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
config = load_config()
|
||||
if "dashboard" not in config:
|
||||
config["dashboard"] = {}
|
||||
config["dashboard"]["theme"] = body.name
|
||||
save_config(config)
|
||||
return {"ok": True, "theme": body.name}
|
||||
|
||||
return await asyncio.to_thread(_run)
|
||||
@@ -16732,11 +16747,12 @@ async def set_dashboard_font(body: FontSetBody):
|
||||
font = body.font if body.font in _FONT_CHOICES else _FONT_DEFAULT_ID
|
||||
|
||||
def _run():
|
||||
config = load_config()
|
||||
if "dashboard" not in config:
|
||||
config["dashboard"] = {}
|
||||
config["dashboard"]["font"] = font
|
||||
save_config(config)
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
config = load_config()
|
||||
if "dashboard" not in config:
|
||||
config["dashboard"] = {}
|
||||
config["dashboard"]["font"] = font
|
||||
save_config(config)
|
||||
return {"ok": True, "font": font}
|
||||
|
||||
return await asyncio.to_thread(_run)
|
||||
@@ -17288,20 +17304,21 @@ async def post_plugin_visibility(request: Request, name: str, body: _PluginVisib
|
||||
name = _validate_plugin_name(name)
|
||||
|
||||
def _run():
|
||||
config = load_config()
|
||||
if "dashboard" not in config or not isinstance(config.get("dashboard"), dict):
|
||||
config["dashboard"] = {}
|
||||
hidden_list: list = config["dashboard"].get("hidden_plugins") or []
|
||||
if not isinstance(hidden_list, list):
|
||||
hidden_list = []
|
||||
with _CONFIG_MUTATION_LOCK:
|
||||
config = load_config()
|
||||
if "dashboard" not in config or not isinstance(config.get("dashboard"), dict):
|
||||
config["dashboard"] = {}
|
||||
hidden_list: list = config["dashboard"].get("hidden_plugins") or []
|
||||
if not isinstance(hidden_list, list):
|
||||
hidden_list = []
|
||||
|
||||
if body.hidden and name not in hidden_list:
|
||||
hidden_list.append(name)
|
||||
elif not body.hidden and name in hidden_list:
|
||||
hidden_list.remove(name)
|
||||
if body.hidden and name not in hidden_list:
|
||||
hidden_list.append(name)
|
||||
elif not body.hidden and name in hidden_list:
|
||||
hidden_list.remove(name)
|
||||
|
||||
config["dashboard"]["hidden_plugins"] = hidden_list
|
||||
save_config(config)
|
||||
config["dashboard"]["hidden_plugins"] = hidden_list
|
||||
save_config(config)
|
||||
_invalidate_plugins_hub_cache()
|
||||
return {"ok": True, "name": name, "hidden": body.hidden}
|
||||
|
||||
|
||||
@@ -102,3 +102,140 @@ class TestGetConfigOffLoop:
|
||||
"_SKILLS_PROFILE_LOCK was held — /api/config is blocking the "
|
||||
"loop again"
|
||||
)
|
||||
|
||||
|
||||
class TestRouterOffLoop:
|
||||
"""Mounted routers (skills/mcp/tools) hold the same locks — they must not
|
||||
do it on the event loop either (same bug class as /api/config)."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _home(self, _isolate_hermes_home):
|
||||
pass
|
||||
|
||||
def test_get_skills_loop_stays_responsive_while_profile_lock_held(self):
|
||||
try:
|
||||
import httpx
|
||||
except ImportError:
|
||||
pytest.skip("httpx not installed")
|
||||
from hermes_cli import web_server
|
||||
|
||||
hold_s = 1.0
|
||||
release = threading.Event()
|
||||
|
||||
def _holder():
|
||||
with web_server._SKILLS_PROFILE_LOCK:
|
||||
release.wait(hold_s)
|
||||
|
||||
async def _scenario():
|
||||
holder = threading.Thread(target=_holder)
|
||||
holder.start()
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
ticks = 0
|
||||
|
||||
async def _heartbeat(stop: asyncio.Event):
|
||||
nonlocal ticks
|
||||
while not stop.is_set():
|
||||
await asyncio.sleep(0.02)
|
||||
ticks += 1
|
||||
|
||||
stop = asyncio.Event()
|
||||
hb = asyncio.create_task(_heartbeat(stop))
|
||||
await asyncio.sleep(0)
|
||||
transport = httpx.ASGITransport(app=web_server.app)
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport, base_url="http://testserver"
|
||||
) as client:
|
||||
client.headers[web_server._SESSION_HEADER_NAME] = (
|
||||
web_server._SESSION_TOKEN
|
||||
)
|
||||
resp = await client.get("/api/skills")
|
||||
finally:
|
||||
stop.set()
|
||||
release.set()
|
||||
await hb
|
||||
holder.join()
|
||||
|
||||
assert resp.status_code == 200
|
||||
return ticks
|
||||
|
||||
ticks = asyncio.run(_scenario())
|
||||
assert ticks >= 10, (
|
||||
f"event loop heartbeat only ticked {ticks} time(s) while "
|
||||
"_SKILLS_PROFILE_LOCK was held — GET /api/skills is blocking "
|
||||
"the loop"
|
||||
)
|
||||
|
||||
|
||||
class TestConfigMutationLock:
|
||||
"""Off-loop read-modify-write handlers must not lose concurrent updates.
|
||||
|
||||
config.py's _CONFIG_LOCK covers each load/save individually; the span
|
||||
between them is serialized by web_server._CONFIG_MUTATION_LOCK. Two
|
||||
concurrent writers touching DIFFERENT keys must both survive."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _home(self, _isolate_hermes_home):
|
||||
pass
|
||||
|
||||
def test_concurrent_distinct_updates_both_survive(self):
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip("fastapi/starlette not installed")
|
||||
from hermes_cli import web_server
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
client = TestClient(web_server.app)
|
||||
client.headers[web_server._SESSION_HEADER_NAME] = web_server._SESSION_TOKEN
|
||||
|
||||
# Interleave the two handlers' load→mutate→save spans deterministically:
|
||||
# gate the FIRST writer inside its mutation-lock hold, and assert the
|
||||
# second writer cannot slip its own load in between. With the lock,
|
||||
# writer B waits for writer A's full span; without it, B loads the
|
||||
# pre-A config and its save erases A's write.
|
||||
results = []
|
||||
|
||||
def _put_theme():
|
||||
resp = client.put("/api/dashboard/theme", json={"name": "midnight"})
|
||||
results.append(("theme", resp.status_code))
|
||||
|
||||
def _put_font():
|
||||
resp = client.put("/api/dashboard/font", json={"font": "jetbrains-mono"})
|
||||
results.append(("font", resp.status_code))
|
||||
|
||||
# Widen the race window: make save_config slow so an unserialized
|
||||
# interleave is near-certain, not just possible.
|
||||
real_save = web_server.save_config
|
||||
|
||||
def _slow_save(cfg, **kwargs):
|
||||
time.sleep(0.15)
|
||||
return real_save(cfg, **kwargs)
|
||||
|
||||
threads = []
|
||||
try:
|
||||
web_server.save_config = _slow_save
|
||||
threads = [
|
||||
threading.Thread(target=_put_theme),
|
||||
threading.Thread(target=_put_font),
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
finally:
|
||||
for t in threads:
|
||||
t.join()
|
||||
web_server.save_config = real_save
|
||||
|
||||
assert all(code == 200 for _, code in results), results
|
||||
cfg = load_config()
|
||||
dashboard = cfg.get("dashboard") or {}
|
||||
# Both writes must be present — a lost update drops exactly one.
|
||||
assert dashboard.get("theme") == "midnight", (
|
||||
"theme write lost to a concurrent font write — "
|
||||
"read-modify-write span is not serialized"
|
||||
)
|
||||
assert dashboard.get("font") == "jetbrains-mono", (
|
||||
"font write lost to a concurrent theme write — "
|
||||
"read-modify-write span is not serialized"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user