fix(status): aggregate independent per-profile gateway failures; harden key filter (OOF-3)

- /api/status now folds LIVE independent per-profile gateways' platform
  failures (gateway_mode == 'multiple', the OOF-3 deployment mode) into
  gateway_platforms under the validated <profile>:<platform> grammar, so
  NAS fleet health sees them without a schema change. ?profile= requests
  stay unmerged (single-profile view).
- Namespaced-key validation no longer fails open: colon-containing keys
  are grammar-checked even when configured-platform loading throws.
- Platform key segment now accepts hyphens, matching plugin platform IDs
  (plugins/platforms/<dir> names, e.g. foo-bar).
This commit is contained in:
Shannon Sands
2026-08-06 09:20:13 +10:00
committed by Teknium
parent f755ed5e90
commit 1d46a9fe0f
2 changed files with 317 additions and 20 deletions
+111 -19
View File
@@ -3158,6 +3158,11 @@ def _collect_profile_gateway_topology() -> Dict[str, Any]:
multiple profiles (gateway.multiplex_profiles), ``"single"`` for one
live gateway, ``"multiple"`` for independent per-profile gateways,
``"none"`` when nothing is running.
* ``profile_platforms`` — ``{profile: platforms}`` raw runtime platform
maps for each LIVE gateway. Internal aggregation input for
``/api/status`` (independent per-profile gateways write failures to
their own ``gateway_state.json``, which the unparameterized endpoint
would otherwise never see). Never exposed directly.
"""
try:
from hermes_cli.profiles import _check_gateway_running, profiles_to_serve
@@ -3165,10 +3170,16 @@ def _collect_profile_gateway_topology() -> Dict[str, Any]:
homes = profiles_to_serve(True)
except Exception:
_log.debug("profile/gateway topology enumeration failed", exc_info=True)
return {"profiles": [], "gateway_mode": "unknown", "gateways": []}
return {
"profiles": [],
"gateway_mode": "unknown",
"gateways": [],
"profile_platforms": {},
}
profile_names = [name for name, _home in homes]
gateways: List[Dict[str, Any]] = []
profile_platforms: Dict[str, dict] = {}
multiplex = False
for name, home in homes:
try:
@@ -3183,6 +3194,9 @@ def _collect_profile_gateway_topology() -> Dict[str, Any]:
served = [str(p) for p in ((runtime or {}).get("served_profiles") or [])]
if name == "default" and len(served) > 1:
multiplex = True
plats = (runtime or {}).get("platforms")
if isinstance(plats, dict) and plats:
profile_platforms[name] = plats
entry: Dict[str, Any] = {
"profile": name,
"ports": _profile_platform_ports(home, runtime),
@@ -3200,7 +3214,12 @@ def _collect_profile_gateway_topology() -> Dict[str, Any]:
else:
mode = "none"
return {"profiles": profile_names, "gateway_mode": mode, "gateways": gateways}
return {
"profiles": profile_names,
"gateway_mode": mode,
"gateways": gateways,
"profile_platforms": profile_platforms,
}
# /api/status is polled ~1/s by the desktop app while it waits for the backend
@@ -3288,7 +3307,11 @@ async def get_health():
_PROFILE_PLATFORM_STATUS_KEY_RE = re.compile(
r"^[a-z0-9][a-z0-9_-]{0,63}:[a-z][a-z0-9_]{0,63}$"
# Profile segment mirrors hermes_cli.profiles._PROFILE_ID_RE. Platform
# segment mirrors the Platform enum's normalized values: built-in members
# plus plugin directory names (lowercased), which allow hyphens as well
# as underscores (e.g. ``reviewer:foo-bar``).
r"^[a-z0-9][a-z0-9_-]{0,63}:[a-z0-9][a-z0-9_-]{0,63}$"
)
@@ -3297,6 +3320,59 @@ def _is_profile_platform_status_key(key: object) -> bool:
return isinstance(key, str) and bool(_PROFILE_PLATFORM_STATUS_KEY_RE.fullmatch(key))
def _status_platform_key_allowed(
key: object, configured: "set[str] | None"
) -> bool:
"""Decide whether a runtime-status platform key may appear publicly.
Namespaced ``<profile>:<platform>`` keys are validated against the key
grammar *unconditionally* — the config-set load failing must not fail
open into projecting arbitrary colon-containing keys from a process-local
JSON file onto the public endpoint. Plain platform keys keep the
long-standing behavior: checked against the configured set when it
loaded, passed through when it did not.
"""
if not isinstance(key, str):
return False
if ":" in key:
return _is_profile_platform_status_key(key)
return configured is None or key in configured
def _merge_profile_gateway_platforms(
gateway_platforms: dict, profile_platforms: dict
) -> dict:
"""Merge independent per-profile gateway platform states (OOF-3).
Hosts that run separate gateway services per profile (``gateway_mode ==
"multiple"``) persist each profile's platform failures in that profile's
own ``gateway_state.json``. The unparameterized ``/api/status`` — the
machine-level probe NAS health monitoring reads — only read the active
profile's file, so those failures were invisible to fleet health. Fold
them in under the same validated ``<profile>:<platform>`` grammar the
multiplex path uses. The active profile's own map is skipped (its
entries are already present, including any multiplex-namespaced ones),
and existing keys are never overwritten.
"""
try:
from hermes_cli.profiles import get_active_profile_name
active = get_active_profile_name()
except Exception:
active = "default"
merged = dict(gateway_platforms)
for prof, plats in (profile_platforms or {}).items():
if prof == active or not isinstance(plats, dict):
continue
for key, value in plats.items():
if not isinstance(key, str) or ":" in key or not isinstance(value, dict):
continue
namespaced = f"{prof}:{key}"
if not _is_profile_platform_status_key(namespaced):
continue
merged.setdefault(namespaced, value)
return merged
@app.get("/api/status")
async def get_status(profile: Optional[str] = None):
status_scope = None
@@ -3395,19 +3471,18 @@ async def get_status(profile: Optional[str] = None):
if runtime:
gateway_state = runtime.get("gateway_state")
gateway_platforms = runtime.get("platforms") or {}
if configured_gateway_platforms is not None:
gateway_platforms = {
key: value
for key, value in gateway_platforms.items()
# Namespaced entries are emitted by configured secondary-
# profile adapters. The config set here belongs to the
# active/default profile, so suffix-checking against it
# would incorrectly hide secondary-only platforms. Keep
# the accepted grammar narrow because this public endpoint
# is projecting a process-local JSON file.
if key in configured_gateway_platforms
or _is_profile_platform_status_key(key)
}
# Namespaced entries are emitted by configured secondary-profile
# adapters. The config set here belongs to the active/default
# profile, so suffix-checking against it would incorrectly hide
# secondary-only platforms. Colon-containing keys are validated
# against the narrow key grammar UNCONDITIONALLY — a failed config
# load must not fail open into projecting arbitrary keys from a
# process-local JSON file onto this public endpoint.
gateway_platforms = {
key: value
for key, value in gateway_platforms.items()
if _status_platform_key_allowed(key, configured_gateway_platforms)
}
gateway_exit_reason = runtime.get("exit_reason")
# Contract: gateway_updated_at is RFC3339 string | null, never a
# number. ``runtime`` here may be the local gateway_state.json
@@ -3429,6 +3504,23 @@ async def get_status(profile: Optional[str] = None):
if gateway_running and gateway_state is None and remote_health_body is not None:
gateway_state = "running"
# Profile + gateway topology (cached, TTL 10s): fetched here — before
# the platform rollup — because plain ``/api/status`` is the
# machine-level probe NAS reads, and hosts running independent
# per-profile gateway services (gateway_mode == "multiple") persist
# each profile's platform failures in that profile's own
# gateway_state.json. Fold those in under the validated
# ``<profile>:<platform>`` grammar so fleet health sees them (OOF-3).
# A ``?profile=`` request targets one profile's view and is left
# unmerged.
topology = await asyncio.get_running_loop().run_in_executor(
None, _collect_profile_gateway_topology_cached
)
if not requested_profile:
gateway_platforms = _merge_profile_gateway_platforms(
gateway_platforms, topology.get("profile_platforms") or {}
)
active_sessions = await _status_active_sessions()
# Busy/drainable readout (NAS lifecycle-safety gate). active_agents is
@@ -3647,9 +3739,9 @@ async def get_status(profile: Optional[str] = None):
# the network (a gated bind), so they must survive the auth gate. The
# per-gateway ``gateways[]`` detail carries host ports (deployment
# recon), so it stays gated with the host paths / PID below.
topology = await asyncio.get_running_loop().run_in_executor(
None, _collect_profile_gateway_topology_cached
)
# (``topology`` was already fetched above, before the platform rollup,
# so the per-profile platform merge could use it — the TTL cache makes
# the earlier fetch the only real scan either way.)
status["profiles"] = topology["profiles"]
status["gateway_mode"] = topology["gateway_mode"]
@@ -77,7 +77,35 @@ class TestCollectProfileGatewayTopology:
assert topo["profiles"] == ["default", "coder"]
assert topo["gateway_mode"] == "none"
assert topo["gateways"] == []
assert topo["profile_platforms"] == {}
def test_collects_per_profile_platform_maps(self, tmp_path, monkeypatch):
# Independent per-profile gateways (gateway_mode == "multiple") each
# write their own gateway_state.json; the collector surfaces every
# LIVE profile's raw platform map for the /api/status merge (OOF-3).
homes = [("default", tmp_path / "d"), ("coder", tmp_path / "c")]
runtimes = {
"default": {"platforms": {"telegram": {"state": "connected"}}},
"coder": {
"platforms": {
"discord": {
"state": "fatal",
"error_code": "duplicate_credential",
}
}
},
}
_patch_topology(
monkeypatch, homes, running={"default", "coder"}, runtimes=runtimes
)
topo = _collect_profile_gateway_topology()
assert topo["gateway_mode"] == "multiple"
assert topo["profile_platforms"] == {
"default": {"telegram": {"state": "connected"}},
"coder": {
"discord": {"state": "fatal", "error_code": "duplicate_credential"}
},
}
@@ -89,7 +117,12 @@ class TestCollectProfileGatewayTopology:
monkeypatch.setattr(profiles_mod, "profiles_to_serve", _boom)
topo = _collect_profile_gateway_topology()
assert topo == {"profiles": [], "gateway_mode": "unknown", "gateways": []}
assert topo == {
"profiles": [],
"gateway_mode": "unknown",
"gateways": [],
"profile_platforms": {},
}
# ---------------------------------------------------------------------------
@@ -184,6 +217,178 @@ class TestStatusEndpointTopology:
assert resp.status_code == 200
assert "reviewer:discord:../../secret" not in resp.json()["gateway_platforms"]
def test_namespaced_key_validation_does_not_fail_open(self, monkeypatch):
# If loading the configured-platform set throws, plain keys keep the
# historical pass-through — but colon-containing keys must STILL be
# validated against the key grammar. A config-load failure must not
# let malformed keys from a process-local JSON file reach the public
# endpoint.
monkeypatch.setattr(web_server, "get_running_pid_cached", lambda: 123)
monkeypatch.setattr(
web_server,
"read_runtime_status",
lambda path=None: {
"gateway_state": "running",
"platforms": {
"telegram": {"state": "connected"},
"reviewer:discord": {"state": "fatal"},
"reviewer:discord:../../secret": {"state": "fatal"},
"REVIEWER:DISCORD": {"state": "fatal"},
},
},
)
def _boom():
raise RuntimeError("config unreadable")
monkeypatch.setattr(
web_server, "_load_configured_gateway_platforms", _boom
)
resp = self.client.get("/api/status")
assert resp.status_code == 200
platforms = resp.json()["gateway_platforms"]
assert "telegram" in platforms
assert "reviewer:discord" in platforms
assert "reviewer:discord:../../secret" not in platforms
assert "REVIEWER:DISCORD" not in platforms
def test_status_preserves_hyphenated_plugin_platform_key(self, monkeypatch):
# Plugin platform IDs may contain hyphens (plugins/platforms/<dir>
# names are lowercased directory names; the Platform enum accepts
# them). A valid ``reviewer:foo-bar`` fatal entry must survive the
# public filter.
monkeypatch.setattr(web_server, "get_running_pid_cached", lambda: 123)
monkeypatch.setattr(
web_server,
"read_runtime_status",
lambda path=None: {
"gateway_state": "running",
"platforms": {
"reviewer:foo-bar": {
"state": "fatal",
"error_code": "duplicate_credential",
},
},
},
)
monkeypatch.setattr(
web_server,
"_load_configured_gateway_platforms",
lambda: {"telegram"},
)
resp = self.client.get("/api/status")
assert resp.status_code == 200
platforms = resp.json()["gateway_platforms"]
assert platforms["reviewer:foo-bar"]["error_code"] == "duplicate_credential"
def test_status_merges_independent_profile_gateway_failures(self, monkeypatch):
# OOF-3 deployment mode: separate gateway services per profile
# (gateway_mode == "multiple"). Each profile's failures live in its
# own gateway_state.json; the machine-level /api/status must fold
# them in as <profile>:<platform> so NAS health monitoring sees them.
import hermes_cli.profiles as profiles_mod
monkeypatch.setattr(web_server, "get_running_pid_cached", lambda: 123)
monkeypatch.setattr(
web_server,
"read_runtime_status",
lambda path=None: {
"gateway_state": "running",
"platforms": {"telegram": {"state": "connected"}},
},
)
monkeypatch.setattr(
web_server,
"_load_configured_gateway_platforms",
lambda: {"telegram"},
)
monkeypatch.setattr(
profiles_mod, "get_active_profile_name", lambda: "default"
)
monkeypatch.setattr(
web_server, "_collect_profile_gateway_topology",
lambda: {
"profiles": ["default", "lead-gen-outreach"],
"gateway_mode": "multiple",
"gateways": [
{"profile": "default", "ports": {}},
{"profile": "lead-gen-outreach", "ports": {}},
],
"profile_platforms": {
"default": {"telegram": {"state": "connected"}},
"lead-gen-outreach": {
"telegram": {
"state": "fatal",
"error_code": "duplicate_credential",
},
"bad key": {"state": "fatal"},
},
},
},
)
resp = self.client.get("/api/status")
assert resp.status_code == 200
data = resp.json()
platforms = data["gateway_platforms"]
# Active profile's own entry untouched, no self-namespacing.
assert platforms["telegram"]["state"] == "connected"
assert "default:telegram" not in platforms
# Independent profile's failure folded in under the namespaced key.
assert (
platforms["lead-gen-outreach:telegram"]["error_code"]
== "duplicate_credential"
)
# Keys that don't survive the grammar are dropped, not projected.
assert not any("bad key" in key for key in platforms)
# The platforms component rollup counts the merged failure.
assert data["components"]["platforms"]["status"] == "degraded"
def test_profile_scoped_status_does_not_merge_other_profiles(self, monkeypatch):
# ?profile=<name> targets one profile's view — merging every other
# profile's failures into it would misattribute state.
import hermes_cli.profiles as profiles_mod
monkeypatch.setattr(web_server, "get_running_pid_cached", lambda: 123)
monkeypatch.setattr(
web_server,
"read_runtime_status",
lambda path=None: {
"gateway_state": "running",
"platforms": {"telegram": {"state": "connected"}},
},
)
monkeypatch.setattr(
web_server,
"_load_configured_gateway_platforms",
lambda: {"telegram"},
)
monkeypatch.setattr(
profiles_mod, "get_active_profile_name", lambda: "default"
)
monkeypatch.setattr(
web_server, "_collect_profile_gateway_topology",
lambda: {
"profiles": ["default", "coder"],
"gateway_mode": "multiple",
"gateways": [],
"profile_platforms": {
"coder": {"telegram": {"state": "fatal"}},
},
},
)
resp = self.client.get("/api/status", params={"profile": "current"})
assert resp.status_code == 200
platforms = resp.json()["gateway_platforms"]
assert "coder:telegram" not in platforms
def test_profile_names_and_mode_public_when_auth_gated(self, monkeypatch):
# Profile NAMES + gateway_mode are low-sensitivity product surface: the
# Hermes Cloud Portal reads /api/status over the network (a gated bind)