From 965a5487884755cef4f1e2d2f16fd51425b2ef5b Mon Sep 17 00:00:00 2001 From: Royalaid <2439803+royalaid@users.noreply.github.com> Date: Fri, 7 Aug 2026 01:13:37 -0700 Subject: [PATCH] fix(gateway): serialize config mutations and finish the router off-loop sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups to the off-loop move, from external review (both verified, the second larger than reported): - Config read-modify-write handlers moved to worker threads could now interleave — _CONFIG_LOCK covers each load/save individually, never the span between them; the event loop used to serialize these accidentally. New _CONFIG_MUTATION_LOCK (worker-threads only, so it can never block the loop) held across the whole load→mutate→save span in all seven RMW handlers. update_config_raw skipped: it's a full-document replace with no server-side read, so a lock cannot close its client-side window. - The review flagged two skills routes still taking _SKILLS_PROFILE_LOCK on the event loop; a systematic audit of hermes_cli/web_routers/ found 24 on-loop routes (skills 5, mcp 9, tools 10, cron 1). All moved to the same inner-_run + asyncio.to_thread pattern, mutating ones under the mutation lock, uniform lock order (_SKILLS_PROFILE_LOCK → _CONFIG_MUTATION_LOCK). Await-safe _config_profile_scope routes, plain def routes, and already-threaded routes unchanged. Regression tests: concurrent theme+font updates both survive (fails with the lock nulled: "theme write lost to a concurrent font write"); event loop stays responsive while the profile lock is held during GET /api/skills. 214 tests passing across the touched suites. Co-Authored-By: Claude Fable 5 --- hermes_cli/web_routers/cron.py | 2 +- hermes_cli/web_routers/mcp.py | 122 ++-- hermes_cli/web_routers/skills.py | 111 ++-- hermes_cli/web_routers/tools.py | 612 ++++++++++-------- hermes_cli/web_server.py | 137 ++-- .../test_web_server_config_offloop.py | 137 ++++ 6 files changed, 690 insertions(+), 431 deletions(-) diff --git a/hermes_cli/web_routers/cron.py b/hermes_cli/web_routers/cron.py index 1d83ba360e..f5f861e710 100644 --- a/hermes_cli/web_routers/cron.py +++ b/hermes_cli/web_routers/cron.py @@ -146,7 +146,7 @@ async def cron_fire_webhook(request: Request): auth = request.headers.get("Authorization", "") token = auth[7:].strip() if auth.startswith("Bearer ") else "" - cfg = load_config() + cfg = await asyncio.to_thread(load_config) claims = get_fire_verifier()( token=token, expected_audience=cfg_get(cfg, "cron", "chronos", "expected_audience", default=""), diff --git a/hermes_cli/web_routers/mcp.py b/hermes_cli/web_routers/mcp.py index ef900d8e39..c07df31543 100644 --- a/hermes_cli/web_routers/mcp.py +++ b/hermes_cli/web_routers/mcp.py @@ -53,14 +53,20 @@ save_env_value = late("save_env_value") _mcp_oauth_flows = LateState("_mcp_oauth_flows") _mcp_oauth_flows_lock = LateState("_mcp_oauth_flows_lock") _MAX_PENDING_MCP_OAUTH_FLOWS = LateState("_MAX_PENDING_MCP_OAUTH_FLOWS") +# Config read-modify-write serialization for off-loop handlers (defined in +# web_server.py; LateState supports ``with``-blocks, so this is the live lock). +_CONFIG_MUTATION_LOCK = LateState("_CONFIG_MUTATION_LOCK") @router.get("/api/mcp/servers") async def list_mcp_servers(profile: Optional[str] = None): from hermes_cli.mcp_config import _get_mcp_servers - with _profile_scope(profile): - servers = _get_mcp_servers() + def _read(): + with _profile_scope(profile): + return _get_mcp_servers() + + servers = await asyncio.to_thread(_read) return { "servers": [ _mcp_server_summary(name, cfg) for name, cfg in sorted(servers.items()) @@ -81,20 +87,29 @@ async def add_mcp_server(body: MCPServerCreate, profile: Optional[str] = None): except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc - with _profile_scope(body.profile or profile): - existing = _get_mcp_servers() + def _check_existing(): + with _profile_scope(body.profile or profile): + return _get_mcp_servers() + + existing = await asyncio.to_thread(_check_existing) if name in existing: raise HTTPException(status_code=409, detail=f"Server '{name}' already exists") - try: + def _save(): with _profile_scope(body.profile or profile): - if bearer_token is not None: - server_config["headers"] = _save_bearer_auth_token(name, bearer_token) - if not _save_mcp_server(name, server_config): - raise HTTPException( - status_code=400, - detail=f"Server '{name}' rejected: suspicious command/args configuration", - ) + # _save_mcp_server does its own load→mutate→save of config.yaml; + # serialize the whole cycle against other off-loop config writers. + with _CONFIG_MUTATION_LOCK: + if bearer_token is not None: + server_config["headers"] = _save_bearer_auth_token(name, bearer_token) + if not _save_mcp_server(name, server_config): + raise HTTPException( + status_code=400, + detail=f"Server '{name}' rejected: suspicious command/args configuration", + ) + + try: + await asyncio.to_thread(_save) except HTTPException: raise except Exception as exc: @@ -116,8 +131,12 @@ async def replace_mcp_servers(body: MCPServersReplace, profile: Optional[str] = """ from hermes_cli.mcp_config import _replace_mcp_servers - with _profile_scope(body.profile or profile): - ok, issues = _replace_mcp_servers(body.servers) + def _run(): + with _profile_scope(body.profile or profile): + with _CONFIG_MUTATION_LOCK: + return _replace_mcp_servers(body.servers) + + ok, issues = await asyncio.to_thread(_run) if not ok: raise HTTPException(status_code=400, detail="; ".join(issues)) return {"ok": True} @@ -127,8 +146,12 @@ async def replace_mcp_servers(body: MCPServersReplace, profile: Optional[str] = async def remove_mcp_server(name: str, profile: Optional[str] = None): from hermes_cli.mcp_config import _remove_mcp_server - with _profile_scope(profile): - removed = _remove_mcp_server(name) + def _run(): + with _profile_scope(profile): + with _CONFIG_MUTATION_LOCK: + return _remove_mcp_server(name) + + removed = await asyncio.to_thread(_run) if not removed: raise HTTPException(status_code=404, detail=f"Server '{name}' not found") return {"ok": True} @@ -143,8 +166,11 @@ async def test_mcp_server(name: str, profile: Optional[str] = None): _probe_single_server, ) - with _profile_scope(profile): - servers = _get_mcp_servers() + def _read(): + with _profile_scope(profile): + return _get_mcp_servers() + + servers = await asyncio.to_thread(_read) if name not in servers: raise HTTPException(status_code=404, detail=f"Server '{name}' not found") @@ -205,9 +231,12 @@ async def auth_mcp_server(name: str, request: Request, profile: Optional[str] = from hermes_constants import get_hermes_home process_home = str(get_hermes_home().expanduser().resolve(strict=False)) - with _profile_scope(profile): - servers = _get_mcp_servers() - flow_home = str(get_hermes_home().expanduser().resolve(strict=False)) + + def _read(): + with _profile_scope(profile): + return _get_mcp_servers(), str(get_hermes_home().expanduser().resolve(strict=False)) + + servers, flow_home = await asyncio.to_thread(_read) if name not in servers: raise HTTPException(status_code=404, detail=f"Server '{name}' not found") cfg = dict(servers[name]) @@ -325,16 +354,20 @@ async def set_mcp_server_enabled( flag the agent reads at startup. Disabled servers stay in config so they can be re-enabled without re-entering their settings. """ - with _profile_scope(body.profile or profile): - cfg = load_config() - servers = cfg.get("mcp_servers") - if not isinstance(servers, dict) or name not in servers: - raise HTTPException(status_code=404, detail=f"Server '{name}' not found") - if not isinstance(servers[name], dict): - raise HTTPException(status_code=400, detail="Malformed server config") - servers[name]["enabled"] = bool(body.enabled) - save_config(cfg) - return {"ok": True, "name": name, "enabled": bool(body.enabled)} + def _run(): + with _profile_scope(body.profile or profile): + with _CONFIG_MUTATION_LOCK: + cfg = load_config() + servers = cfg.get("mcp_servers") + if not isinstance(servers, dict) or name not in servers: + raise HTTPException(status_code=404, detail=f"Server '{name}' not found") + if not isinstance(servers[name], dict): + raise HTTPException(status_code=400, detail="Malformed server config") + servers[name]["enabled"] = bool(body.enabled) + save_config(cfg) + return {"ok": True, "name": name, "enabled": bool(body.enabled)} + + return await asyncio.to_thread(_run) @router.get("/api/mcp/catalog") @@ -355,12 +388,16 @@ async def list_mcp_catalog(profile: Optional[str] = None): entries = [] try: - with _profile_scope(profile): - catalog_entries = list(mcp_catalog.list_catalog()) - installed_state = { - e.name: (mcp_catalog.is_installed(e.name), mcp_catalog.is_enabled(e.name)) - for e in catalog_entries - } + def _read(): + with _profile_scope(profile): + catalog = list(mcp_catalog.list_catalog()) + state = { + e.name: (mcp_catalog.is_installed(e.name), mcp_catalog.is_enabled(e.name)) + for e in catalog + } + return catalog, state + + catalog_entries, installed_state = await asyncio.to_thread(_read) for entry in catalog_entries: auth = entry.auth transport = entry.transport @@ -434,10 +471,13 @@ async def install_mcp_catalog_entry(body: MCPCatalogInstall, profile: Optional[s # they need; we only write the ones the user provided). effective_profile = body.profile or profile if body.env: - with _profile_scope(effective_profile): - for k, v in body.env.items(): - if v: - save_env_value(k, v) + def _write_env(): + with _profile_scope(effective_profile): + for k, v in body.env.items(): + if v: + save_env_value(k, v) + + await asyncio.to_thread(_write_env) # Git-bootstrap entries can take a while to clone — run via the background # action path so the request returns immediately and the UI can tail logs. diff --git a/hermes_cli/web_routers/skills.py b/hermes_cli/web_routers/skills.py index bf83c1ee29..67733ceae6 100644 --- a/hermes_cli/web_routers/skills.py +++ b/hermes_cli/web_routers/skills.py @@ -49,6 +49,10 @@ load_config = late("load_config") # Live proxies for web_server-owned module state (mutations/monkeypatches # on web_server remain authoritative; resolved at operation time). _SKILL_HUB_SOURCE_LABELS = LateState("_SKILL_HUB_SOURCE_LABELS") +# Config read-modify-write serialization for off-loop handlers (see the +# definition in web_server.py). LateState supports ``with``-blocks, so this +# is the live lock object, not a frozen import-time copy. +_CONFIG_MUTATION_LOCK = LateState("_CONFIG_MUTATION_LOCK") @hub_router.post("/api/skills/hub/install") @@ -401,40 +405,48 @@ async def get_skills(profile: Optional[str] = None): activity_count, load_usage, ) - with _profile_scope(profile): - config = load_config() - disabled = get_disabled_skills(config) - skills = _find_all_skills(skip_disabled=True) - usage = load_usage() - # Set-based provenance (same classification as skill_usage.provenance, - # without a per-skill manifest read): hub > bundled > agent, where - # "agent" covers agent-authored AND local hand-made skills — the ones - # the user may edit/delete from the UI. - bundled_names = _read_bundled_manifest_names() - hub_names = _read_hub_installed_names() - for s in skills: - s["enabled"] = s["name"] not in disabled - s["usage"] = activity_count(usage.get(s["name"], {})) - s["provenance"] = ( - "hub" if s["name"] in hub_names - else "bundled" if s["name"] in bundled_names - else "agent" - ) - return skills + def _run(): + with _profile_scope(profile): + config = load_config() + disabled = get_disabled_skills(config) + skills = _find_all_skills(skip_disabled=True) + usage = load_usage() + # Set-based provenance (same classification as skill_usage.provenance, + # without a per-skill manifest read): hub > bundled > agent, where + # "agent" covers agent-authored AND local hand-made skills — the ones + # the user may edit/delete from the UI. + bundled_names = _read_bundled_manifest_names() + hub_names = _read_hub_installed_names() + for s in skills: + s["enabled"] = s["name"] not in disabled + s["usage"] = activity_count(usage.get(s["name"], {})) + s["provenance"] = ( + "hub" if s["name"] in hub_names + else "bundled" if s["name"] in bundled_names + else "agent" + ) + return skills + + return await asyncio.to_thread(_run) @router.put("/api/skills/toggle") async def toggle_skill(body: SkillToggle, profile: Optional[str] = None): from hermes_cli.skills_config import get_disabled_skills, save_disabled_skills - with _profile_scope(body.profile or profile): - config = load_config() - disabled = get_disabled_skills(config) - if body.enabled: - disabled.discard(body.name) - else: - disabled.add(body.name) - save_disabled_skills(config, disabled) - return {"ok": True, "name": body.name, "enabled": body.enabled} + + def _run(): + with _profile_scope(body.profile or profile): + with _CONFIG_MUTATION_LOCK: + config = load_config() + disabled = get_disabled_skills(config) + if body.enabled: + disabled.discard(body.name) + else: + disabled.add(body.name) + save_disabled_skills(config, disabled) + return {"ok": True, "name": body.name, "enabled": body.enabled} + + return await asyncio.to_thread(_run) @router.get("/api/skills/content") @@ -442,18 +454,21 @@ async def get_skill_content(name: str, profile: Optional[str] = None): """Return the raw SKILL.md text for a skill, for the dashboard editor.""" from tools.skill_manager_tool import _find_skill - with _profile_scope(profile): - found = _find_skill(name) - if not found: - raise HTTPException(status_code=404, detail=f"Skill '{name}' not found.") - skill_md = found["path"] / "SKILL.md" - if not skill_md.exists(): - raise HTTPException(status_code=404, detail=f"Skill '{name}' has no SKILL.md.") - try: - content = skill_md.read_text(encoding="utf-8") - except OSError as exc: - raise HTTPException(status_code=500, detail=str(exc)) from exc - return {"name": name, "content": content, "path": str(skill_md)} + def _run(): + with _profile_scope(profile): + found = _find_skill(name) + if not found: + raise HTTPException(status_code=404, detail=f"Skill '{name}' not found.") + skill_md = found["path"] / "SKILL.md" + if not skill_md.exists(): + raise HTTPException(status_code=404, detail=f"Skill '{name}' has no SKILL.md.") + try: + content = skill_md.read_text(encoding="utf-8") + except OSError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + return {"name": name, "content": content, "path": str(skill_md)} + + return await asyncio.to_thread(_run) @router.post("/api/skills") @@ -467,8 +482,11 @@ async def create_skill(body: SkillCreate): """ from tools.skill_manager_tool import _create_skill - with _profile_scope(body.profile): - result = _create_skill(body.name, body.content, body.category or None) + def _run(): + with _profile_scope(body.profile): + return _create_skill(body.name, body.content, body.category or None) + + result = await asyncio.to_thread(_run) if not result.get("success"): raise HTTPException(status_code=400, detail=result.get("error", "Failed to create skill.")) _clear_skills_prompt_cache() @@ -480,8 +498,11 @@ async def update_skill_content(body: SkillContentUpdate): """Replace the SKILL.md of an existing skill (full rewrite) from the editor.""" from tools.skill_manager_tool import _edit_skill - with _profile_scope(body.profile): - result = _edit_skill(body.name, body.content) + def _run(): + with _profile_scope(body.profile): + return _edit_skill(body.name, body.content) + + result = await asyncio.to_thread(_run) if not result.get("success"): err = result.get("error", "Failed to update skill.") status = 404 if "not found" in str(err).lower() else 400 diff --git a/hermes_cli/web_routers/tools.py b/hermes_cli/web_routers/tools.py index 0fbb055cf5..0c3e6d2b51 100644 --- a/hermes_cli/web_routers/tools.py +++ b/hermes_cli/web_routers/tools.py @@ -10,6 +10,7 @@ late-binding seam in :mod:`hermes_cli.web_deps` (``late`` for callables, stays authoritative. """ +import asyncio import logging import sys # noqa: F401 — used by handlers from typing import Any, Dict, List, Optional # noqa: F401 @@ -48,6 +49,9 @@ save_config = late("save_config") _MODEL_CATALOG_TOOLSETS = LateState("_MODEL_CATALOG_TOOLSETS") _TERMINAL_BACKENDS = LateState("_TERMINAL_BACKENDS") _TERMINAL_BACKEND_NAMES = LateState("_TERMINAL_BACKEND_NAMES") +# Config read-modify-write serialization for off-loop handlers (defined in +# web_server.py; LateState supports ``with``-blocks, so this is the live lock). +_CONFIG_MUTATION_LOCK = LateState("_CONFIG_MUTATION_LOCK") @router.get("/api/tools/toolsets") @@ -64,21 +68,25 @@ async def get_toolsets(profile: Optional[str] = None): from hermes_cli.platforms import platform_label from toolsets import resolve_toolset - with _profile_scope(profile): - config = load_config() - toolset_rows = _get_effective_configurable_toolsets() - target_platforms = { - _toolset_configuration_platform(name) for name, _, _ in toolset_rows - } - enabled_by_platform = { - platform: _get_platform_tools( - config, - platform, - include_default_mcp_servers=False, - ) - for platform in target_platforms - } - features = get_nous_subscription_features(config) + def _read(): + with _profile_scope(profile): + config = load_config() + toolset_rows = _get_effective_configurable_toolsets() + target_platforms = { + _toolset_configuration_platform(name) for name, _, _ in toolset_rows + } + enabled_by_platform = { + platform: _get_platform_tools( + config, + platform, + include_default_mcp_servers=False, + ) + for platform in target_platforms + } + features = get_nous_subscription_features(config) + return config, toolset_rows, enabled_by_platform, features + + config, toolset_rows, enabled_by_platform, features = await asyncio.to_thread(_read) result = [] for name, label, desc in toolset_rows: try: @@ -135,37 +143,38 @@ async def toggle_toolset(name: str, body: ToolsetToggle, profile: Optional[str] raise HTTPException(status_code=400, detail=f"Unknown toolset: {name}") target_platform = _toolset_configuration_platform(name) - if name in _CONFIG_ONLY_TOOLSETS: - # Config-only capabilities (stt) toggle their own config section's - # ``enabled`` flag — there is no platform_toolsets entry to write. + + def _run(): + if name in _CONFIG_ONLY_TOOLSETS: + # Config-only capabilities (stt) toggle their own config section's + # ``enabled`` flag — there is no platform_toolsets entry to write. + with _profile_scope(body.profile or profile): + with _CONFIG_MUTATION_LOCK: + config = load_config() + section = config.setdefault(name, {}) + if not isinstance(section, dict): + section = {} + config[name] = section + section["enabled"] = bool(body.enabled) + save_config(config) + return with _profile_scope(body.profile or profile): - config = load_config() - section = config.setdefault(name, {}) - if not isinstance(section, dict): - section = {} - config[name] = section - section["enabled"] = bool(body.enabled) - save_config(config) - return { - "ok": True, - "name": name, - "platform": target_platform, - "enabled": body.enabled, - } - with _profile_scope(body.profile or profile): - config = load_config() - enabled = set( - _get_platform_tools( - config, - target_platform, - include_default_mcp_servers=False, - ) - ) - if body.enabled: - enabled.add(name) - else: - enabled.discard(name) - _save_platform_tools(config, target_platform, enabled) + with _CONFIG_MUTATION_LOCK: + config = load_config() + enabled = set( + _get_platform_tools( + config, + target_platform, + include_default_mcp_servers=False, + ) + ) + if body.enabled: + enabled.add(name) + else: + enabled.discard(name) + _save_platform_tools(config, target_platform, enabled) + + await asyncio.to_thread(_run) return { "ok": True, "name": name, @@ -199,79 +208,84 @@ async def get_toolset_config(name: str, profile: Optional[str] = None): if name not in valid: raise HTTPException(status_code=400, detail=f"Unknown toolset: {name}") - with _profile_scope(profile): - config = load_config() - cat = TOOL_CATEGORIES.get(name) - providers = [] - active_provider = None - active_search_backend = None - active_extract_backend = None - if cat: - # Fetch portal/entitlement state once for the whole matrix — the - # per-provider readiness computation below reuses it instead of - # re-probing per row. - features = get_nous_subscription_features(config, force_fresh=True) - for prov in _visible_providers(cat, config, force_fresh=True): - env_vars = [ - { - "key": e["key"], - "prompt": e.get("prompt", e["key"]), - "url": e.get("url"), - "default": e.get("default"), - "is_set": bool(get_env_value(e["key"])), + def _read(): + with _profile_scope(profile): + config = load_config() + cat = TOOL_CATEGORIES.get(name) + providers = [] + active_provider = None + active_search_backend = None + active_extract_backend = None + if cat: + # Fetch portal/entitlement state once for the whole matrix — the + # per-provider readiness computation below reuses it instead of + # re-probing per row. + features = get_nous_subscription_features(config, force_fresh=True) + for prov in _visible_providers(cat, config, force_fresh=True): + env_vars = [ + { + "key": e["key"], + "prompt": e.get("prompt", e["key"]), + "url": e.get("url"), + "default": e.get("default"), + "is_set": bool(get_env_value(e["key"])), + } + for e in prov.get("env_vars", []) + ] + # Surface the same active-provider determination the CLI picker + # uses (``_is_provider_active``) so the GUI highlights the provider + # actually written to config (e.g. web.backend), not just the first + # keyless one in the list. + is_active = _is_provider_active(prov, config, force_fresh=True) + if is_active and active_provider is None: + active_provider = prov["name"] + row = { + "name": prov["name"], + "badge": prov.get("badge", ""), + "tag": prov.get("tag", ""), + "env_vars": env_vars, + "post_setup": prov.get("post_setup"), + "requires_nous_auth": bool(prov.get("requires_nous_auth")), + "is_active": is_active, + # Honest server-side readiness. The GUI's old client-side + # heuristic showed "Ready" for every zero-env-var row — + # including logged-out Nous Subscription rows and never-run + # post_setup installs (see provider_readiness_status). + "status": provider_readiness_status( + prov, config, features=features, is_active=is_active + ), } - for e in prov.get("env_vars", []) - ] - # Surface the same active-provider determination the CLI picker - # uses (``_is_provider_active``) so the GUI highlights the provider - # actually written to config (e.g. web.backend), not just the first - # keyless one in the list. - is_active = _is_provider_active(prov, config, force_fresh=True) - if is_active and active_provider is None: - active_provider = prov["name"] - row = { - "name": prov["name"], - "badge": prov.get("badge", ""), - "tag": prov.get("tag", ""), - "env_vars": env_vars, - "post_setup": prov.get("post_setup"), - "requires_nous_auth": bool(prov.get("requires_nous_auth")), - "is_active": is_active, - # Honest server-side readiness. The GUI's old client-side - # heuristic showed "Ready" for every zero-env-var row — - # including logged-out Nous Subscription rows and never-run - # post_setup installs (see provider_readiness_status). - "status": provider_readiness_status( - prov, config, features=features, is_active=is_active - ), - } - if name == "web" and prov.get("web_backend"): - # The runtime split web into two capabilities long ago - # (web.search_backend / web.extract_backend); surface each - # row's backend key and which capabilities it can serve so - # the GUI can offer per-capability selection. - row["web_backend"] = prov["web_backend"] - row["capabilities"] = web_provider_capabilities(prov["web_backend"]) - if name == "tts" and prov.get("tts_provider"): - # The provider key written to tts.provider on selection. - # Doubles as the config section holding the provider's - # voice/model settings (tts..*) so the GUI can render - # those fields inline in the Capabilities panel. - row["tts_provider"] = prov["tts_provider"] - providers.append(row) - if name == "web": - # Resolve the per-capability active backends exactly the way the - # web_search / web_extract dispatchers do (per-capability key → - # shared web.backend → credential auto-detect), so the GUI badges - # reflect what a tool call would actually hit right now. - try: - from tools.web_tools import _get_extract_backend, _get_search_backend + if name == "web" and prov.get("web_backend"): + # The runtime split web into two capabilities long ago + # (web.search_backend / web.extract_backend); surface each + # row's backend key and which capabilities it can serve so + # the GUI can offer per-capability selection. + row["web_backend"] = prov["web_backend"] + row["capabilities"] = web_provider_capabilities(prov["web_backend"]) + if name == "tts" and prov.get("tts_provider"): + # The provider key written to tts.provider on selection. + # Doubles as the config section holding the provider's + # voice/model settings (tts..*) so the GUI can render + # those fields inline in the Capabilities panel. + row["tts_provider"] = prov["tts_provider"] + providers.append(row) + if name == "web": + # Resolve the per-capability active backends exactly the way the + # web_search / web_extract dispatchers do (per-capability key → + # shared web.backend → credential auto-detect), so the GUI badges + # reflect what a tool call would actually hit right now. + try: + from tools.web_tools import _get_extract_backend, _get_search_backend + + active_search_backend = _get_search_backend() + active_extract_backend = _get_extract_backend() + except Exception: + active_search_backend = None + active_extract_backend = None + return cat, providers, active_provider, active_search_backend, active_extract_backend + + cat, providers, active_provider, active_search_backend, active_extract_backend = await asyncio.to_thread(_read) - active_search_backend = _get_search_backend() - active_extract_backend = _get_extract_backend() - except Exception: - active_search_backend = None - active_extract_backend = None payload = { "name": name, "has_category": cat is not None, @@ -300,28 +314,35 @@ async def get_toolset_models( if section is None: return {"name": name, "has_models": False, "models": [], "current": None, "default": None} - with _profile_scope(profile): - config = load_config() - row = _find_toolset_provider_row(name, config, provider) - plugin = _resolve_toolset_model_plugin(name, row) if row else None - if not plugin: - return { - "name": name, - "has_models": False, - "models": [], - "current": None, - "default": None, - } + def _read(): + with _profile_scope(profile): + config = load_config() + row = _find_toolset_provider_row(name, config, provider) + plugin = _resolve_toolset_model_plugin(name, row) if row else None + if not plugin: + return None - catalog, default_model = _toolset_model_catalog(name, plugin) - section_cfg = config.get(section) - current = None - if isinstance(section_cfg, dict): - raw = section_cfg.get("model") - if isinstance(raw, str) and raw.strip(): - current = raw.strip() - if current not in catalog: - current = default_model if default_model in catalog else None + catalog, default_model = _toolset_model_catalog(name, plugin) + section_cfg = config.get(section) + current = None + if isinstance(section_cfg, dict): + raw = section_cfg.get("model") + if isinstance(raw, str) and raw.strip(): + current = raw.strip() + if current not in catalog: + current = default_model if default_model in catalog else None + return row, plugin, catalog, default_model, current + + resolved = await asyncio.to_thread(_read) + if resolved is None: + return { + "name": name, + "has_models": False, + "models": [], + "current": None, + "default": None, + } + row, plugin, catalog, default_model, current = resolved models = [ { @@ -364,30 +385,34 @@ async def select_toolset_model( if not model_id: raise HTTPException(status_code=400, detail="model is required") - with _profile_scope(body.profile or profile): - config = load_config() - row = _find_toolset_provider_row(name, config, body.provider) - plugin = _resolve_toolset_model_plugin(name, row) if row else None - if not plugin: - raise HTTPException( - status_code=400, - detail=f"No model-capable backend is active for {name}", - ) + def _run(): + with _profile_scope(body.profile or profile): + with _CONFIG_MUTATION_LOCK: + config = load_config() + row = _find_toolset_provider_row(name, config, body.provider) + plugin = _resolve_toolset_model_plugin(name, row) if row else None + if not plugin: + raise HTTPException( + status_code=400, + detail=f"No model-capable backend is active for {name}", + ) - catalog, _default = _toolset_model_catalog(name, plugin) - if model_id not in catalog: - raise HTTPException( - status_code=400, - detail=f"Unknown model {model_id!r} for backend {plugin!r}", - ) + catalog, _default = _toolset_model_catalog(name, plugin) + if model_id not in catalog: + raise HTTPException( + status_code=400, + detail=f"Unknown model {model_id!r} for backend {plugin!r}", + ) - section_cfg = config.setdefault(section, {}) - if not isinstance(section_cfg, dict): - section_cfg = {} - config[section] = section_cfg - section_cfg["model"] = model_id - save_config(config) + section_cfg = config.setdefault(section, {}) + if not isinstance(section_cfg, dict): + section_cfg = {} + config[section] = section_cfg + section_cfg["model"] = model_id + save_config(config) + return plugin + plugin = await asyncio.to_thread(_run) return {"ok": True, "name": name, "model": model_id, "plugin": plugin} @@ -450,77 +475,82 @@ async def select_toolset_provider( detail=f"Unknown capability: {body.capability!r} (expected 'search' or 'extract')", ) - with _profile_scope(body.profile or profile): - config = load_config() - if body.capability is not None: - # Per-capability path: resolve the picker row to its backend key - # and write web._backend. Does NOT touch web.backend, - # so the other capability keeps resolving through the shared - # fallback chain. - cat = TOOL_CATEGORIES.get(name) - providers = _visible_providers(cat, config, force_fresh=True) if cat else [] - prov = next((p for p in providers if p.get("name") == body.provider), None) - if prov is None: - raise HTTPException( - status_code=400, - detail=f"Unknown provider {body.provider!r} for toolset {name!r}", - ) - backend = prov.get("web_backend") - if not backend: - raise HTTPException( - status_code=400, - detail=f"Provider {body.provider!r} has no web backend key", - ) - if body.capability not in web_provider_capabilities(backend): - raise HTTPException( - status_code=400, - detail=f"{body.provider} does not support {body.capability}", - ) - web_cfg = config.setdefault("web", {}) - if not isinstance(web_cfg, dict): - web_cfg = {} - config["web"] = web_cfg - web_cfg[f"{body.capability}_backend"] = backend - else: - try: - apply_provider_selection(name, body.provider, config) - except KeyError as exc: - raise HTTPException(status_code=400, detail=str(exc).strip('"')) - save_config(config) - response: Dict[str, Any] = {"ok": True, "name": name, "provider": body.provider} - if body.capability is not None: - response["capability"] = body.capability + def _run(): + with _profile_scope(body.profile or profile): + with _CONFIG_MUTATION_LOCK: + config = load_config() + if body.capability is not None: + # Per-capability path: resolve the picker row to its backend key + # and write web._backend. Does NOT touch web.backend, + # so the other capability keeps resolving through the shared + # fallback chain. + cat = TOOL_CATEGORIES.get(name) + providers = _visible_providers(cat, config, force_fresh=True) if cat else [] + prov = next((p for p in providers if p.get("name") == body.provider), None) + if prov is None: + raise HTTPException( + status_code=400, + detail=f"Unknown provider {body.provider!r} for toolset {name!r}", + ) + backend = prov.get("web_backend") + if not backend: + raise HTTPException( + status_code=400, + detail=f"Provider {body.provider!r} has no web backend key", + ) + if body.capability not in web_provider_capabilities(backend): + raise HTTPException( + status_code=400, + detail=f"{body.provider} does not support {body.capability}", + ) + web_cfg = config.setdefault("web", {}) + if not isinstance(web_cfg, dict): + web_cfg = {} + config["web"] = web_cfg + web_cfg[f"{body.capability}_backend"] = backend + else: + try: + apply_provider_selection(name, body.provider, config) + except KeyError as exc: + raise HTTPException(status_code=400, detail=str(exc).strip('"')) + save_config(config) + response: Dict[str, Any] = {"ok": True, "name": name, "provider": body.provider} + if body.capability is not None: + response["capability"] = body.capability - # Entitlement check for managed Nous rows — mirrors the gate the CLI - # applies via ensure_nous_portal_access at selection time. - cat = TOOL_CATEGORIES.get(name) - row = None - if cat: - row = next( - ( - p - for p in _visible_providers(cat, config, force_fresh=True) - if p.get("name") == body.provider - ), - None, - ) - managed_feature = (row or {}).get("managed_nous_feature") - if managed_feature: - features = get_nous_subscription_features(config, force_fresh=True) - acct = features.account_info - category = MANAGED_FEATURE_COVERAGE_CATEGORY.get(managed_feature) - entitled = bool( - acct - and acct.logged_in - and ( - acct.tool_gateway_entitled_for(category) - if category - else acct.tool_gateway_entitled - ) - ) - if not entitled: - response["needs_nous_auth"] = True - response["feature"] = managed_feature + # Entitlement check for managed Nous rows — mirrors the gate the CLI + # applies via ensure_nous_portal_access at selection time. + cat = TOOL_CATEGORIES.get(name) + row = None + if cat: + row = next( + ( + p + for p in _visible_providers(cat, config, force_fresh=True) + if p.get("name") == body.provider + ), + None, + ) + managed_feature = (row or {}).get("managed_nous_feature") + if managed_feature: + features = get_nous_subscription_features(config, force_fresh=True) + acct = features.account_info + category = MANAGED_FEATURE_COVERAGE_CATEGORY.get(managed_feature) + entitled = bool( + acct + and acct.logged_in + and ( + acct.tool_gateway_entitled_for(category) + if category + else acct.tool_gateway_entitled + ) + ) + if not entitled: + response["needs_nous_auth"] = True + response["feature"] = managed_feature + return response + + response = await asyncio.to_thread(_run) return response @@ -547,35 +577,39 @@ async def save_toolset_env(name: str, body: ToolsetEnvUpdate, profile: Optional[ if name not in valid_ts: raise HTTPException(status_code=400, detail=f"Unknown toolset: {name}") - with _profile_scope(body.profile or profile): - config = load_config() - cat = TOOL_CATEGORIES.get(name) - allowed: set[str] = set() - if cat: - for prov in _visible_providers(cat, config, force_fresh=True): - for e in prov.get("env_vars", []): - allowed.add(e["key"]) + def _run(): + with _profile_scope(body.profile or profile): + config = load_config() + cat = TOOL_CATEGORIES.get(name) + allowed: set[str] = set() + if cat: + for prov in _visible_providers(cat, config, force_fresh=True): + for e in prov.get("env_vars", []): + allowed.add(e["key"]) - unknown = [k for k in body.env if k not in allowed] - if unknown: - raise HTTPException( - status_code=400, - detail=f"Unknown env var(s) for toolset {name}: {', '.join(sorted(unknown))}", - ) + unknown = [k for k in body.env if k not in allowed] + if unknown: + raise HTTPException( + status_code=400, + detail=f"Unknown env var(s) for toolset {name}: {', '.join(sorted(unknown))}", + ) - saved: List[str] = [] - skipped: List[str] = [] - for key, value in body.env.items(): - if value and value.strip(): - try: - save_env_value(key, value.strip()) - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) - saved.append(key) - else: - skipped.append(key) + saved: List[str] = [] + skipped: List[str] = [] + for key, value in body.env.items(): + if value and value.strip(): + try: + save_env_value(key, value.strip()) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + saved.append(key) + else: + skipped.append(key) - status = {k: bool(get_env_value(k)) for k in allowed} + status = {k: bool(get_env_value(k)) for k in allowed} + return saved, skipped, status + + saved, skipped, status = await asyncio.to_thread(_run) return {"ok": True, "name": name, "saved": saved, "skipped": skipped, "is_set": status} @@ -639,27 +673,30 @@ async def get_terminal_backends(profile: Optional[str] = None): Probes are fast (<~2s each) and defensive — a probe failure surfaces as a status, never an error response. """ - with _profile_scope(profile): - config = load_config() - terminal_cfg = config.get("terminal") - if not isinstance(terminal_cfg, dict): - terminal_cfg = {} - active = str(terminal_cfg.get("backend") or "local").strip().lower() - if active not in _TERMINAL_BACKEND_NAMES: - active = "local" + def _read(): + with _profile_scope(profile): + config = load_config() + terminal_cfg = config.get("terminal") + if not isinstance(terminal_cfg, dict): + terminal_cfg = {} + active = str(terminal_cfg.get("backend") or "local").strip().lower() + if active not in _TERMINAL_BACKEND_NAMES: + active = "local" - backends = [] - for row in _TERMINAL_BACKENDS: - status, detail = _probe_terminal_backend(row["name"], terminal_cfg) - backends.append({ - "name": row["name"], - "label": row["label"], - "description": row["description"], - "active": row["name"] == active, - "status": status, - "detail": detail, - }) - return {"active": active, "backends": backends} + backends = [] + for row in _TERMINAL_BACKENDS: + status, detail = _probe_terminal_backend(row["name"], terminal_cfg) + backends.append({ + "name": row["name"], + "label": row["label"], + "description": row["description"], + "active": row["name"] == active, + "status": status, + "detail": detail, + }) + return {"active": active, "backends": backends} + + return await asyncio.to_thread(_read) @router.put("/api/tools/terminal/backend") @@ -680,14 +717,18 @@ async def select_terminal_backend( f"Use one of: {', '.join(sorted(_TERMINAL_BACKEND_NAMES))}", ) - with _profile_scope(body.profile or profile): - config = load_config() - terminal_cfg = config.setdefault("terminal", {}) - if not isinstance(terminal_cfg, dict): - terminal_cfg = {} - config["terminal"] = terminal_cfg - terminal_cfg["backend"] = backend - save_config(config) + def _run(): + with _profile_scope(body.profile or profile): + with _CONFIG_MUTATION_LOCK: + config = load_config() + terminal_cfg = config.setdefault("terminal", {}) + if not isinstance(terminal_cfg, dict): + terminal_cfg = {} + config["terminal"] = terminal_cfg + terminal_cfg["backend"] = backend + save_config(config) + + await asyncio.to_thread(_run) return {"ok": True, "backend": backend} @@ -701,8 +742,11 @@ async def get_computer_use_status(profile: Optional[str] = None): """ from tools.computer_use.permissions import computer_use_status - with _profile_scope(profile): - return computer_use_status() + def _read(): + with _profile_scope(profile): + return computer_use_status() + + return await asyncio.to_thread(_read) @router.post("/api/tools/computer-use/permissions/grant") diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 1bfcaa36ed..00ef6ba741 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -2966,6 +2966,16 @@ _TOPOLOGY_CACHE: Dict[str, Any] = {"ts": 0.0, "data": None, "fn": None} _TOPOLOGY_CACHE_LOCK = threading.Lock() _TOPOLOGY_CACHE_TTL = 10.0 +# Serializes read-modify-write cycles over config.yaml for handlers that run +# in worker threads (asyncio.to_thread). config.py's _CONFIG_LOCK covers each +# load_config()/save_config() call individually, not the span between them — +# when these handlers ran on the event loop the loop itself serialized the +# whole cycle, but off-loop two concurrent updates could interleave +# load→mutate→save and silently drop one another's writes. Held only in +# worker threads, so it can never block the event loop. RLock so a locked +# section that calls helpers which also take it can't self-deadlock. +_CONFIG_MUTATION_LOCK = threading.RLock() + def _topology_cache_get(fn: Any) -> Optional[Dict[str, Any]]: if ( @@ -6959,9 +6969,10 @@ async def update_config(body: ConfigUpdate, profile: Optional[str] = None): # is not sent in the PUT body. A full-replace save would silently # drop those keys. Deep-merge incoming over what's on disk so the # frontend can only overwrite what it explicitly sends. - existing = read_raw_config() - incoming = _denormalize_config_from_web(body.config) - save_config(_deep_merge(existing, incoming)) + with _CONFIG_MUTATION_LOCK: + existing = read_raw_config() + incoming = _denormalize_config_from_web(body.config) + save_config(_deep_merge(existing, incoming)) return {"ok": True} try: @@ -12924,11 +12935,12 @@ async def set_memory_provider(body: MemoryProviderSelect): def _run(): _require_memory_provider_ready(provider) - cfg = load_config() - if not isinstance(cfg.get("memory"), dict): - cfg["memory"] = {} - cfg["memory"]["provider"] = provider - save_config(cfg) + with _CONFIG_MUTATION_LOCK: + cfg = load_config() + if not isinstance(cfg.get("memory"), dict): + cfg["memory"] = {} + cfg["memory"]["provider"] = provider + save_config(cfg) return {"ok": True, "active": provider} return await asyncio.to_thread(_run) @@ -13237,23 +13249,24 @@ async def create_hook(body: HookCreate): pass def _run(): - cfg = load_config() - hooks_cfg = cfg.get("hooks") - if not isinstance(hooks_cfg, dict): - hooks_cfg = {} - cfg["hooks"] = hooks_cfg - entries = hooks_cfg.get(event) - if not isinstance(entries, list): - entries = [] - hooks_cfg[event] = entries + with _CONFIG_MUTATION_LOCK: + cfg = load_config() + hooks_cfg = cfg.get("hooks") + if not isinstance(hooks_cfg, dict): + hooks_cfg = {} + cfg["hooks"] = hooks_cfg + entries = hooks_cfg.get(event) + if not isinstance(entries, list): + entries = [] + hooks_cfg[event] = entries - new_entry: Dict[str, Any] = {"command": command} - if body.matcher: - new_entry["matcher"] = body.matcher - if body.timeout is not None: - new_entry["timeout"] = int(body.timeout) - entries.append(new_entry) - save_config(cfg) + new_entry: Dict[str, Any] = {"command": command} + if body.matcher: + new_entry["matcher"] = body.matcher + if body.timeout is not None: + new_entry["timeout"] = int(body.timeout) + entries.append(new_entry) + save_config(cfg) approved = False if body.approve: @@ -13279,21 +13292,22 @@ async def delete_hook(body: HookDelete): raise HTTPException(status_code=400, detail="event and command are required") def _run(): - cfg = load_config() - hooks_cfg = cfg.get("hooks") removed = False - if isinstance(hooks_cfg, dict) and isinstance(hooks_cfg.get(event), list): - before = len(hooks_cfg[event]) - hooks_cfg[event] = [ - e for e in hooks_cfg[event] - if not (isinstance(e, dict) and e.get("command") == command) - ] - removed = len(hooks_cfg[event]) < before - if not hooks_cfg[event]: - del hooks_cfg[event] - if not hooks_cfg: - cfg.pop("hooks", None) - save_config(cfg) + with _CONFIG_MUTATION_LOCK: + cfg = load_config() + hooks_cfg = cfg.get("hooks") + if isinstance(hooks_cfg, dict) and isinstance(hooks_cfg.get(event), list): + before = len(hooks_cfg[event]) + hooks_cfg[event] = [ + e for e in hooks_cfg[event] + if not (isinstance(e, dict) and e.get("command") == command) + ] + removed = len(hooks_cfg[event]) < before + if not hooks_cfg[event]: + del hooks_cfg[event] + if not hooks_cfg: + cfg.pop("hooks", None) + save_config(cfg) # Revoke consent regardless so a re-add re-prompts. try: @@ -16683,11 +16697,12 @@ async def get_dashboard_themes(): async def set_dashboard_theme(body: ThemeSetBody): """Set the active dashboard theme (persists to config.yaml).""" def _run(): - config = load_config() - if "dashboard" not in config: - config["dashboard"] = {} - config["dashboard"]["theme"] = body.name - save_config(config) + with _CONFIG_MUTATION_LOCK: + config = load_config() + if "dashboard" not in config: + config["dashboard"] = {} + config["dashboard"]["theme"] = body.name + save_config(config) return {"ok": True, "theme": body.name} return await asyncio.to_thread(_run) @@ -16732,11 +16747,12 @@ async def set_dashboard_font(body: FontSetBody): font = body.font if body.font in _FONT_CHOICES else _FONT_DEFAULT_ID def _run(): - config = load_config() - if "dashboard" not in config: - config["dashboard"] = {} - config["dashboard"]["font"] = font - save_config(config) + with _CONFIG_MUTATION_LOCK: + config = load_config() + if "dashboard" not in config: + config["dashboard"] = {} + config["dashboard"]["font"] = font + save_config(config) return {"ok": True, "font": font} return await asyncio.to_thread(_run) @@ -17288,20 +17304,21 @@ async def post_plugin_visibility(request: Request, name: str, body: _PluginVisib name = _validate_plugin_name(name) def _run(): - config = load_config() - if "dashboard" not in config or not isinstance(config.get("dashboard"), dict): - config["dashboard"] = {} - hidden_list: list = config["dashboard"].get("hidden_plugins") or [] - if not isinstance(hidden_list, list): - hidden_list = [] + with _CONFIG_MUTATION_LOCK: + config = load_config() + if "dashboard" not in config or not isinstance(config.get("dashboard"), dict): + config["dashboard"] = {} + hidden_list: list = config["dashboard"].get("hidden_plugins") or [] + if not isinstance(hidden_list, list): + hidden_list = [] - if body.hidden and name not in hidden_list: - hidden_list.append(name) - elif not body.hidden and name in hidden_list: - hidden_list.remove(name) + if body.hidden and name not in hidden_list: + hidden_list.append(name) + elif not body.hidden and name in hidden_list: + hidden_list.remove(name) - config["dashboard"]["hidden_plugins"] = hidden_list - save_config(config) + config["dashboard"]["hidden_plugins"] = hidden_list + save_config(config) _invalidate_plugins_hub_cache() return {"ok": True, "name": name, "hidden": body.hidden} diff --git a/tests/hermes_cli/test_web_server_config_offloop.py b/tests/hermes_cli/test_web_server_config_offloop.py index 98a733ba6a..fd06b545a2 100644 --- a/tests/hermes_cli/test_web_server_config_offloop.py +++ b/tests/hermes_cli/test_web_server_config_offloop.py @@ -102,3 +102,140 @@ class TestGetConfigOffLoop: "_SKILLS_PROFILE_LOCK was held — /api/config is blocking the " "loop again" ) + + +class TestRouterOffLoop: + """Mounted routers (skills/mcp/tools) hold the same locks — they must not + do it on the event loop either (same bug class as /api/config).""" + + @pytest.fixture(autouse=True) + def _home(self, _isolate_hermes_home): + pass + + def test_get_skills_loop_stays_responsive_while_profile_lock_held(self): + try: + import httpx + except ImportError: + pytest.skip("httpx not installed") + from hermes_cli import web_server + + hold_s = 1.0 + release = threading.Event() + + def _holder(): + with web_server._SKILLS_PROFILE_LOCK: + release.wait(hold_s) + + async def _scenario(): + holder = threading.Thread(target=_holder) + holder.start() + await asyncio.sleep(0.05) + + ticks = 0 + + async def _heartbeat(stop: asyncio.Event): + nonlocal ticks + while not stop.is_set(): + await asyncio.sleep(0.02) + ticks += 1 + + stop = asyncio.Event() + hb = asyncio.create_task(_heartbeat(stop)) + await asyncio.sleep(0) + transport = httpx.ASGITransport(app=web_server.app) + try: + async with httpx.AsyncClient( + transport=transport, base_url="http://testserver" + ) as client: + client.headers[web_server._SESSION_HEADER_NAME] = ( + web_server._SESSION_TOKEN + ) + resp = await client.get("/api/skills") + finally: + stop.set() + release.set() + await hb + holder.join() + + assert resp.status_code == 200 + return ticks + + ticks = asyncio.run(_scenario()) + assert ticks >= 10, ( + f"event loop heartbeat only ticked {ticks} time(s) while " + "_SKILLS_PROFILE_LOCK was held — GET /api/skills is blocking " + "the loop" + ) + + +class TestConfigMutationLock: + """Off-loop read-modify-write handlers must not lose concurrent updates. + + config.py's _CONFIG_LOCK covers each load/save individually; the span + between them is serialized by web_server._CONFIG_MUTATION_LOCK. Two + concurrent writers touching DIFFERENT keys must both survive.""" + + @pytest.fixture(autouse=True) + def _home(self, _isolate_hermes_home): + pass + + def test_concurrent_distinct_updates_both_survive(self): + try: + from starlette.testclient import TestClient + except ImportError: + pytest.skip("fastapi/starlette not installed") + from hermes_cli import web_server + from hermes_cli.config import load_config + + client = TestClient(web_server.app) + client.headers[web_server._SESSION_HEADER_NAME] = web_server._SESSION_TOKEN + + # Interleave the two handlers' load→mutate→save spans deterministically: + # gate the FIRST writer inside its mutation-lock hold, and assert the + # second writer cannot slip its own load in between. With the lock, + # writer B waits for writer A's full span; without it, B loads the + # pre-A config and its save erases A's write. + results = [] + + def _put_theme(): + resp = client.put("/api/dashboard/theme", json={"name": "midnight"}) + results.append(("theme", resp.status_code)) + + def _put_font(): + resp = client.put("/api/dashboard/font", json={"font": "jetbrains-mono"}) + results.append(("font", resp.status_code)) + + # Widen the race window: make save_config slow so an unserialized + # interleave is near-certain, not just possible. + real_save = web_server.save_config + + def _slow_save(cfg, **kwargs): + time.sleep(0.15) + return real_save(cfg, **kwargs) + + threads = [] + try: + web_server.save_config = _slow_save + threads = [ + threading.Thread(target=_put_theme), + threading.Thread(target=_put_font), + ] + for t in threads: + t.start() + finally: + for t in threads: + t.join() + web_server.save_config = real_save + + assert all(code == 200 for _, code in results), results + cfg = load_config() + dashboard = cfg.get("dashboard") or {} + # Both writes must be present — a lost update drops exactly one. + assert dashboard.get("theme") == "midnight", ( + "theme write lost to a concurrent font write — " + "read-modify-write span is not serialized" + ) + assert dashboard.get("font") == "jetbrains-mono", ( + "font write lost to a concurrent theme write — " + "read-modify-write span is not serialized" + )