fix(status): aggregate independent per-profile gateway failures; harden key filter (OOF-3)
- /api/status now folds LIVE independent per-profile gateways' platform failures (gateway_mode == 'multiple', the OOF-3 deployment mode) into gateway_platforms under the validated <profile>:<platform> grammar, so NAS fleet health sees them without a schema change. ?profile= requests stay unmerged (single-profile view). - Namespaced-key validation no longer fails open: colon-containing keys are grammar-checked even when configured-platform loading throws. - Platform key segment now accepts hyphens, matching plugin platform IDs (plugins/platforms/<dir> names, e.g. foo-bar).
This commit is contained in:
+111
-19
@@ -3158,6 +3158,11 @@ def _collect_profile_gateway_topology() -> Dict[str, Any]:
|
||||
multiple profiles (gateway.multiplex_profiles), ``"single"`` for one
|
||||
live gateway, ``"multiple"`` for independent per-profile gateways,
|
||||
``"none"`` when nothing is running.
|
||||
* ``profile_platforms`` — ``{profile: platforms}`` raw runtime platform
|
||||
maps for each LIVE gateway. Internal aggregation input for
|
||||
``/api/status`` (independent per-profile gateways write failures to
|
||||
their own ``gateway_state.json``, which the unparameterized endpoint
|
||||
would otherwise never see). Never exposed directly.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.profiles import _check_gateway_running, profiles_to_serve
|
||||
@@ -3165,10 +3170,16 @@ def _collect_profile_gateway_topology() -> Dict[str, Any]:
|
||||
homes = profiles_to_serve(True)
|
||||
except Exception:
|
||||
_log.debug("profile/gateway topology enumeration failed", exc_info=True)
|
||||
return {"profiles": [], "gateway_mode": "unknown", "gateways": []}
|
||||
return {
|
||||
"profiles": [],
|
||||
"gateway_mode": "unknown",
|
||||
"gateways": [],
|
||||
"profile_platforms": {},
|
||||
}
|
||||
|
||||
profile_names = [name for name, _home in homes]
|
||||
gateways: List[Dict[str, Any]] = []
|
||||
profile_platforms: Dict[str, dict] = {}
|
||||
multiplex = False
|
||||
for name, home in homes:
|
||||
try:
|
||||
@@ -3183,6 +3194,9 @@ def _collect_profile_gateway_topology() -> Dict[str, Any]:
|
||||
served = [str(p) for p in ((runtime or {}).get("served_profiles") or [])]
|
||||
if name == "default" and len(served) > 1:
|
||||
multiplex = True
|
||||
plats = (runtime or {}).get("platforms")
|
||||
if isinstance(plats, dict) and plats:
|
||||
profile_platforms[name] = plats
|
||||
entry: Dict[str, Any] = {
|
||||
"profile": name,
|
||||
"ports": _profile_platform_ports(home, runtime),
|
||||
@@ -3200,7 +3214,12 @@ def _collect_profile_gateway_topology() -> Dict[str, Any]:
|
||||
else:
|
||||
mode = "none"
|
||||
|
||||
return {"profiles": profile_names, "gateway_mode": mode, "gateways": gateways}
|
||||
return {
|
||||
"profiles": profile_names,
|
||||
"gateway_mode": mode,
|
||||
"gateways": gateways,
|
||||
"profile_platforms": profile_platforms,
|
||||
}
|
||||
|
||||
|
||||
# /api/status is polled ~1/s by the desktop app while it waits for the backend
|
||||
@@ -3288,7 +3307,11 @@ async def get_health():
|
||||
|
||||
|
||||
_PROFILE_PLATFORM_STATUS_KEY_RE = re.compile(
|
||||
r"^[a-z0-9][a-z0-9_-]{0,63}:[a-z][a-z0-9_]{0,63}$"
|
||||
# Profile segment mirrors hermes_cli.profiles._PROFILE_ID_RE. Platform
|
||||
# segment mirrors the Platform enum's normalized values: built-in members
|
||||
# plus plugin directory names (lowercased), which allow hyphens as well
|
||||
# as underscores (e.g. ``reviewer:foo-bar``).
|
||||
r"^[a-z0-9][a-z0-9_-]{0,63}:[a-z0-9][a-z0-9_-]{0,63}$"
|
||||
)
|
||||
|
||||
|
||||
@@ -3297,6 +3320,59 @@ def _is_profile_platform_status_key(key: object) -> bool:
|
||||
return isinstance(key, str) and bool(_PROFILE_PLATFORM_STATUS_KEY_RE.fullmatch(key))
|
||||
|
||||
|
||||
def _status_platform_key_allowed(
|
||||
key: object, configured: "set[str] | None"
|
||||
) -> bool:
|
||||
"""Decide whether a runtime-status platform key may appear publicly.
|
||||
|
||||
Namespaced ``<profile>:<platform>`` keys are validated against the key
|
||||
grammar *unconditionally* — the config-set load failing must not fail
|
||||
open into projecting arbitrary colon-containing keys from a process-local
|
||||
JSON file onto the public endpoint. Plain platform keys keep the
|
||||
long-standing behavior: checked against the configured set when it
|
||||
loaded, passed through when it did not.
|
||||
"""
|
||||
if not isinstance(key, str):
|
||||
return False
|
||||
if ":" in key:
|
||||
return _is_profile_platform_status_key(key)
|
||||
return configured is None or key in configured
|
||||
|
||||
|
||||
def _merge_profile_gateway_platforms(
|
||||
gateway_platforms: dict, profile_platforms: dict
|
||||
) -> dict:
|
||||
"""Merge independent per-profile gateway platform states (OOF-3).
|
||||
|
||||
Hosts that run separate gateway services per profile (``gateway_mode ==
|
||||
"multiple"``) persist each profile's platform failures in that profile's
|
||||
own ``gateway_state.json``. The unparameterized ``/api/status`` — the
|
||||
machine-level probe NAS health monitoring reads — only read the active
|
||||
profile's file, so those failures were invisible to fleet health. Fold
|
||||
them in under the same validated ``<profile>:<platform>`` grammar the
|
||||
multiplex path uses. The active profile's own map is skipped (its
|
||||
entries are already present, including any multiplex-namespaced ones),
|
||||
and existing keys are never overwritten.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.profiles import get_active_profile_name
|
||||
active = get_active_profile_name()
|
||||
except Exception:
|
||||
active = "default"
|
||||
merged = dict(gateway_platforms)
|
||||
for prof, plats in (profile_platforms or {}).items():
|
||||
if prof == active or not isinstance(plats, dict):
|
||||
continue
|
||||
for key, value in plats.items():
|
||||
if not isinstance(key, str) or ":" in key or not isinstance(value, dict):
|
||||
continue
|
||||
namespaced = f"{prof}:{key}"
|
||||
if not _is_profile_platform_status_key(namespaced):
|
||||
continue
|
||||
merged.setdefault(namespaced, value)
|
||||
return merged
|
||||
|
||||
|
||||
@app.get("/api/status")
|
||||
async def get_status(profile: Optional[str] = None):
|
||||
status_scope = None
|
||||
@@ -3395,19 +3471,18 @@ async def get_status(profile: Optional[str] = None):
|
||||
if runtime:
|
||||
gateway_state = runtime.get("gateway_state")
|
||||
gateway_platforms = runtime.get("platforms") or {}
|
||||
if configured_gateway_platforms is not None:
|
||||
gateway_platforms = {
|
||||
key: value
|
||||
for key, value in gateway_platforms.items()
|
||||
# Namespaced entries are emitted by configured secondary-
|
||||
# profile adapters. The config set here belongs to the
|
||||
# active/default profile, so suffix-checking against it
|
||||
# would incorrectly hide secondary-only platforms. Keep
|
||||
# the accepted grammar narrow because this public endpoint
|
||||
# is projecting a process-local JSON file.
|
||||
if key in configured_gateway_platforms
|
||||
or _is_profile_platform_status_key(key)
|
||||
}
|
||||
# Namespaced entries are emitted by configured secondary-profile
|
||||
# adapters. The config set here belongs to the active/default
|
||||
# profile, so suffix-checking against it would incorrectly hide
|
||||
# secondary-only platforms. Colon-containing keys are validated
|
||||
# against the narrow key grammar UNCONDITIONALLY — a failed config
|
||||
# load must not fail open into projecting arbitrary keys from a
|
||||
# process-local JSON file onto this public endpoint.
|
||||
gateway_platforms = {
|
||||
key: value
|
||||
for key, value in gateway_platforms.items()
|
||||
if _status_platform_key_allowed(key, configured_gateway_platforms)
|
||||
}
|
||||
gateway_exit_reason = runtime.get("exit_reason")
|
||||
# Contract: gateway_updated_at is RFC3339 string | null, never a
|
||||
# number. ``runtime`` here may be the local gateway_state.json
|
||||
@@ -3429,6 +3504,23 @@ async def get_status(profile: Optional[str] = None):
|
||||
if gateway_running and gateway_state is None and remote_health_body is not None:
|
||||
gateway_state = "running"
|
||||
|
||||
# Profile + gateway topology (cached, TTL 10s): fetched here — before
|
||||
# the platform rollup — because plain ``/api/status`` is the
|
||||
# machine-level probe NAS reads, and hosts running independent
|
||||
# per-profile gateway services (gateway_mode == "multiple") persist
|
||||
# each profile's platform failures in that profile's own
|
||||
# gateway_state.json. Fold those in under the validated
|
||||
# ``<profile>:<platform>`` grammar so fleet health sees them (OOF-3).
|
||||
# A ``?profile=`` request targets one profile's view and is left
|
||||
# unmerged.
|
||||
topology = await asyncio.get_running_loop().run_in_executor(
|
||||
None, _collect_profile_gateway_topology_cached
|
||||
)
|
||||
if not requested_profile:
|
||||
gateway_platforms = _merge_profile_gateway_platforms(
|
||||
gateway_platforms, topology.get("profile_platforms") or {}
|
||||
)
|
||||
|
||||
active_sessions = await _status_active_sessions()
|
||||
|
||||
# Busy/drainable readout (NAS lifecycle-safety gate). active_agents is
|
||||
@@ -3647,9 +3739,9 @@ async def get_status(profile: Optional[str] = None):
|
||||
# the network (a gated bind), so they must survive the auth gate. The
|
||||
# per-gateway ``gateways[]`` detail carries host ports (deployment
|
||||
# recon), so it stays gated with the host paths / PID below.
|
||||
topology = await asyncio.get_running_loop().run_in_executor(
|
||||
None, _collect_profile_gateway_topology_cached
|
||||
)
|
||||
# (``topology`` was already fetched above, before the platform rollup,
|
||||
# so the per-profile platform merge could use it — the TTL cache makes
|
||||
# the earlier fetch the only real scan either way.)
|
||||
status["profiles"] = topology["profiles"]
|
||||
status["gateway_mode"] = topology["gateway_mode"]
|
||||
|
||||
|
||||
@@ -77,7 +77,35 @@ class TestCollectProfileGatewayTopology:
|
||||
assert topo["profiles"] == ["default", "coder"]
|
||||
assert topo["gateway_mode"] == "none"
|
||||
assert topo["gateways"] == []
|
||||
assert topo["profile_platforms"] == {}
|
||||
|
||||
def test_collects_per_profile_platform_maps(self, tmp_path, monkeypatch):
|
||||
# Independent per-profile gateways (gateway_mode == "multiple") each
|
||||
# write their own gateway_state.json; the collector surfaces every
|
||||
# LIVE profile's raw platform map for the /api/status merge (OOF-3).
|
||||
homes = [("default", tmp_path / "d"), ("coder", tmp_path / "c")]
|
||||
runtimes = {
|
||||
"default": {"platforms": {"telegram": {"state": "connected"}}},
|
||||
"coder": {
|
||||
"platforms": {
|
||||
"discord": {
|
||||
"state": "fatal",
|
||||
"error_code": "duplicate_credential",
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
_patch_topology(
|
||||
monkeypatch, homes, running={"default", "coder"}, runtimes=runtimes
|
||||
)
|
||||
topo = _collect_profile_gateway_topology()
|
||||
assert topo["gateway_mode"] == "multiple"
|
||||
assert topo["profile_platforms"] == {
|
||||
"default": {"telegram": {"state": "connected"}},
|
||||
"coder": {
|
||||
"discord": {"state": "fatal", "error_code": "duplicate_credential"}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -89,7 +117,12 @@ class TestCollectProfileGatewayTopology:
|
||||
|
||||
monkeypatch.setattr(profiles_mod, "profiles_to_serve", _boom)
|
||||
topo = _collect_profile_gateway_topology()
|
||||
assert topo == {"profiles": [], "gateway_mode": "unknown", "gateways": []}
|
||||
assert topo == {
|
||||
"profiles": [],
|
||||
"gateway_mode": "unknown",
|
||||
"gateways": [],
|
||||
"profile_platforms": {},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -184,6 +217,178 @@ class TestStatusEndpointTopology:
|
||||
assert resp.status_code == 200
|
||||
assert "reviewer:discord:../../secret" not in resp.json()["gateway_platforms"]
|
||||
|
||||
def test_namespaced_key_validation_does_not_fail_open(self, monkeypatch):
|
||||
# If loading the configured-platform set throws, plain keys keep the
|
||||
# historical pass-through — but colon-containing keys must STILL be
|
||||
# validated against the key grammar. A config-load failure must not
|
||||
# let malformed keys from a process-local JSON file reach the public
|
||||
# endpoint.
|
||||
monkeypatch.setattr(web_server, "get_running_pid_cached", lambda: 123)
|
||||
monkeypatch.setattr(
|
||||
web_server,
|
||||
"read_runtime_status",
|
||||
lambda path=None: {
|
||||
"gateway_state": "running",
|
||||
"platforms": {
|
||||
"telegram": {"state": "connected"},
|
||||
"reviewer:discord": {"state": "fatal"},
|
||||
"reviewer:discord:../../secret": {"state": "fatal"},
|
||||
"REVIEWER:DISCORD": {"state": "fatal"},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
def _boom():
|
||||
raise RuntimeError("config unreadable")
|
||||
|
||||
monkeypatch.setattr(
|
||||
web_server, "_load_configured_gateway_platforms", _boom
|
||||
)
|
||||
|
||||
resp = self.client.get("/api/status")
|
||||
|
||||
assert resp.status_code == 200
|
||||
platforms = resp.json()["gateway_platforms"]
|
||||
assert "telegram" in platforms
|
||||
assert "reviewer:discord" in platforms
|
||||
assert "reviewer:discord:../../secret" not in platforms
|
||||
assert "REVIEWER:DISCORD" not in platforms
|
||||
|
||||
def test_status_preserves_hyphenated_plugin_platform_key(self, monkeypatch):
|
||||
# Plugin platform IDs may contain hyphens (plugins/platforms/<dir>
|
||||
# names are lowercased directory names; the Platform enum accepts
|
||||
# them). A valid ``reviewer:foo-bar`` fatal entry must survive the
|
||||
# public filter.
|
||||
monkeypatch.setattr(web_server, "get_running_pid_cached", lambda: 123)
|
||||
monkeypatch.setattr(
|
||||
web_server,
|
||||
"read_runtime_status",
|
||||
lambda path=None: {
|
||||
"gateway_state": "running",
|
||||
"platforms": {
|
||||
"reviewer:foo-bar": {
|
||||
"state": "fatal",
|
||||
"error_code": "duplicate_credential",
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
web_server,
|
||||
"_load_configured_gateway_platforms",
|
||||
lambda: {"telegram"},
|
||||
)
|
||||
|
||||
resp = self.client.get("/api/status")
|
||||
|
||||
assert resp.status_code == 200
|
||||
platforms = resp.json()["gateway_platforms"]
|
||||
assert platforms["reviewer:foo-bar"]["error_code"] == "duplicate_credential"
|
||||
|
||||
def test_status_merges_independent_profile_gateway_failures(self, monkeypatch):
|
||||
# OOF-3 deployment mode: separate gateway services per profile
|
||||
# (gateway_mode == "multiple"). Each profile's failures live in its
|
||||
# own gateway_state.json; the machine-level /api/status must fold
|
||||
# them in as <profile>:<platform> so NAS health monitoring sees them.
|
||||
import hermes_cli.profiles as profiles_mod
|
||||
|
||||
monkeypatch.setattr(web_server, "get_running_pid_cached", lambda: 123)
|
||||
monkeypatch.setattr(
|
||||
web_server,
|
||||
"read_runtime_status",
|
||||
lambda path=None: {
|
||||
"gateway_state": "running",
|
||||
"platforms": {"telegram": {"state": "connected"}},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
web_server,
|
||||
"_load_configured_gateway_platforms",
|
||||
lambda: {"telegram"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
profiles_mod, "get_active_profile_name", lambda: "default"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
web_server, "_collect_profile_gateway_topology",
|
||||
lambda: {
|
||||
"profiles": ["default", "lead-gen-outreach"],
|
||||
"gateway_mode": "multiple",
|
||||
"gateways": [
|
||||
{"profile": "default", "ports": {}},
|
||||
{"profile": "lead-gen-outreach", "ports": {}},
|
||||
],
|
||||
"profile_platforms": {
|
||||
"default": {"telegram": {"state": "connected"}},
|
||||
"lead-gen-outreach": {
|
||||
"telegram": {
|
||||
"state": "fatal",
|
||||
"error_code": "duplicate_credential",
|
||||
},
|
||||
"bad key": {"state": "fatal"},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
resp = self.client.get("/api/status")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
platforms = data["gateway_platforms"]
|
||||
# Active profile's own entry untouched, no self-namespacing.
|
||||
assert platforms["telegram"]["state"] == "connected"
|
||||
assert "default:telegram" not in platforms
|
||||
# Independent profile's failure folded in under the namespaced key.
|
||||
assert (
|
||||
platforms["lead-gen-outreach:telegram"]["error_code"]
|
||||
== "duplicate_credential"
|
||||
)
|
||||
# Keys that don't survive the grammar are dropped, not projected.
|
||||
assert not any("bad key" in key for key in platforms)
|
||||
# The platforms component rollup counts the merged failure.
|
||||
assert data["components"]["platforms"]["status"] == "degraded"
|
||||
|
||||
def test_profile_scoped_status_does_not_merge_other_profiles(self, monkeypatch):
|
||||
# ?profile=<name> targets one profile's view — merging every other
|
||||
# profile's failures into it would misattribute state.
|
||||
import hermes_cli.profiles as profiles_mod
|
||||
|
||||
monkeypatch.setattr(web_server, "get_running_pid_cached", lambda: 123)
|
||||
monkeypatch.setattr(
|
||||
web_server,
|
||||
"read_runtime_status",
|
||||
lambda path=None: {
|
||||
"gateway_state": "running",
|
||||
"platforms": {"telegram": {"state": "connected"}},
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
web_server,
|
||||
"_load_configured_gateway_platforms",
|
||||
lambda: {"telegram"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
profiles_mod, "get_active_profile_name", lambda: "default"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
web_server, "_collect_profile_gateway_topology",
|
||||
lambda: {
|
||||
"profiles": ["default", "coder"],
|
||||
"gateway_mode": "multiple",
|
||||
"gateways": [],
|
||||
"profile_platforms": {
|
||||
"coder": {"telegram": {"state": "fatal"}},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
resp = self.client.get("/api/status", params={"profile": "current"})
|
||||
|
||||
assert resp.status_code == 200
|
||||
platforms = resp.json()["gateway_platforms"]
|
||||
assert "coder:telegram" not in platforms
|
||||
|
||||
def test_profile_names_and_mode_public_when_auth_gated(self, monkeypatch):
|
||||
# Profile NAMES + gateway_mode are low-sensitivity product surface: the
|
||||
# Hermes Cloud Portal reads /api/status over the network (a gated bind)
|
||||
|
||||
Reference in New Issue
Block a user