diff --git a/contributors/emails/Finn763@users.noreply.github.com b/contributors/emails/Finn763@users.noreply.github.com index 96c23015c7..cc305d066f 100644 --- a/contributors/emails/Finn763@users.noreply.github.com +++ b/contributors/emails/Finn763@users.noreply.github.com @@ -1 +1,2 @@ Finn763 +# PR #94971 (issue #94895) attribution mapping diff --git a/hermes_state.py b/hermes_state.py index 8b00bc3d12..08576cacf6 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -9187,8 +9187,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) max_idle_seconds: float, sources: Tuple[str, ...] = ("tui", "desktop", "subagent"), exclude_ids: Tuple[str, ...] = (), + heartbeat_staleness_seconds: Optional[float] = None, + heartbeat_ownership_grace_seconds: Optional[float] = None, ) -> List[str]: - """Close session rows orphaned by a dead gateway process (#65194). + """Close session rows orphaned by a dead gateway process (#65194, #94895). The TUI/desktop gateway reaps disconnected websocket sessions with an in-process ``threading.Timer`` grace timer; a gateway restart destroys @@ -9211,6 +9213,28 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) window). Non-destructive: messages are preserved and the row remains resumable. First-reason-wins is preserved via ``ended_at IS NULL``. + Cross-backend liveness (#94895): when one ``state.db`` is shared by + N serve / gateway processes (isolated backends, fixed-port launchd + ``hermes serve``, desktop WS sidecar), each backend registers a row + in ``gateway_heartbeats`` refreshed every few seconds. A row is + only reaped when ``started_at``/message staleness hold AND no live + backend (heartbeat refreshed within ``heartbeat_staleness_seconds``, + default ``2 * max_idle_seconds``) could plausibly own it. + + Ownership inference: a live backend B ``owns`` a session S if + ``B.started_at <= S.started_at + heartbeat_ownership_grace_seconds`` + (default ``heartbeat_staleness_seconds``). The grace window + accommodates the deploy-time migration case where a backend just + wrote its first heartbeat row while its existing open sessions + predate the schema. The grace is bounded by the staleness window + so a fresh PID-reuse respawn cannot indefinitely protect sessions + inherited from a dead predecessor. + + When NO backend has ever written a heartbeat (legacy deployment + mid-upgrade before any process has registered) the predicate falls + back to the original behavior so we never silently strand a row + that pre-dates the schema. + The SELECT + UPDATE run in one ``BEGIN IMMEDIATE`` write, so a sibling process cannot sneak a new message or end-reason between the staleness check and the close. Returns the swept session ids. @@ -9218,7 +9242,18 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) srcs = tuple(s for s in sources if s) if max_idle_seconds <= 0 or not srcs: return [] + hb_staleness = ( + heartbeat_staleness_seconds + if heartbeat_staleness_seconds and heartbeat_staleness_seconds > 0 + else max_idle_seconds * 2 + ) + hb_grace = ( + heartbeat_ownership_grace_seconds + if heartbeat_ownership_grace_seconds and heartbeat_ownership_grace_seconds >= 0 + else hb_staleness + ) cutoff = time.time() - max_idle_seconds + hb_cutoff = time.time() - hb_staleness placeholders = ",".join("?" for _ in srcs) staleness = ( "started_at < ? AND COALESCE((SELECT MAX(m.timestamp) FROM messages m" @@ -9226,10 +9261,28 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) def _do(conn): + # Cross-process liveness gate (#94895). A session is "owned by + # a live backend" if any row in gateway_heartbeats is fresh + # (last_heartbeat >= hb_cutoff) AND was alive no later than + # ``sessions.started_at + hb_grace`` (heartbeats.started_at <= + # sessions.started_at + hb_grace). If at least one live backend + # matches, the row is not orphaned. + # + # ``hb_cutoff`` and ``hb_grace`` are computed above so all + # backends running concurrent sweep queries agree on the same + # boundaries. We do NOT clear heartbeats here — that's each + # backend's atexit responsibility via ``clear_backend_heartbeat``. + orphan_predicate = ( + f"{staleness} AND NOT EXISTS (" + "SELECT 1 FROM gateway_heartbeats h" + " WHERE h.last_heartbeat >= ?" + f" AND h.started_at <= sessions.started_at + ?" + ")" + ) rows = conn.execute( f"SELECT id FROM sessions WHERE ended_at IS NULL" - f" AND source IN ({placeholders}) AND {staleness}", - (*srcs, cutoff, cutoff), + f" AND source IN ({placeholders}) AND {orphan_predicate}", + (*srcs, cutoff, cutoff, hb_cutoff, hb_grace), ).fetchall() excluded = {str(x) for x in exclude_ids if x} victims = [str(r["id"]) for r in rows if str(r["id"]) not in excluded] @@ -9237,17 +9290,121 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return [] now = time.time() marks = ",".join("?" for _ in victims) - # Re-apply the same staleness predicate under the write lock so a - # row that raced to activity between SELECT and UPDATE is spared. + # Re-apply the same predicates under the write lock so a + # row that raced to activity between SELECT and UPDATE is + # spared (and so a freshly registered heartbeat from a sibling + # that started during this transaction can still save the row). conn.execute( f"UPDATE sessions SET ended_at = ?, end_reason = 'startup_orphan_reap'" - f" WHERE id IN ({marks}) AND ended_at IS NULL AND {staleness}", - (now, *victims, cutoff, cutoff), + f" WHERE id IN ({marks}) AND ended_at IS NULL" + f" AND {orphan_predicate}", + (now, *victims, cutoff, cutoff, hb_cutoff, hb_grace), ) return victims return self._execute_write(_do) or [] + # ── Cross-backend heartbeat API (#94895) ─────────────────────────── + # Each serve / tui_gateway process registers a heartbeat row at startup + # and refreshes ``last_heartbeat`` periodically. The startup orphan + # sweep reads these rows to avoid reaping sessions owned by another + # still-live backend that just happens to be idle. Backends remove + # their own row on graceful shutdown; a row that survives a crash is + # reclaimed by the staleness sweep once ``last_heartbeat`` ages out. + + def register_backend_heartbeat( + self, + *, + backend_id: str, + pid: int, + started_at: float, + last_heartbeat: Optional[float] = None, + profile: str = "", + host: str = "", + ) -> None: + """Upsert this backend's liveness row (#94895). + + ``backend_id`` MUST be stable for the lifetime of the process + (e.g. ``f"{profile}@{host}:{pid}"``) so a respawn cannot accidentally + inherit the dead predecessor's heartbeat and protect stale rows. + ``started_at`` records when THIS process started (not the wall clock + at first refresh) so a long-lived backend whose previous run died + cannot be confused with a freshly-spawned sibling. + """ + if not backend_id: + return + ts = time.time() if last_heartbeat is None else float(last_heartbeat) + def _do(conn): + conn.execute( + "INSERT INTO gateway_heartbeats" + " (backend_id, pid, started_at, last_heartbeat, profile, host)" + " VALUES (?, ?, ?, ?, ?, ?)" + " ON CONFLICT(backend_id) DO UPDATE SET" + " pid = excluded.pid," + " started_at = excluded.started_at," + " last_heartbeat = excluded.last_heartbeat," + " profile = excluded.profile," + " host = excluded.host", + (str(backend_id), int(pid), float(started_at), ts, + str(profile), str(host)), + ) + self._execute_write(_do) + + def clear_backend_heartbeat(self, backend_id: str) -> bool: + """Remove this backend's heartbeat row (#94895). + + Called from ``atexit`` so a graceful shutdown doesn't leave a stale + row behind. A crashed backend's row is reclaimed later by + ``prune_stale_heartbeats``. Returns True if a row was removed. + """ + if not backend_id: + return False + def _do(conn): + cur = conn.execute( + "DELETE FROM gateway_heartbeats WHERE backend_id = ?", + (str(backend_id),), + ) + return cur.rowcount > 0 + return bool(self._execute_write(_do)) + + def prune_stale_heartbeats(self, *, max_age_seconds: float) -> List[str]: + """Drop heartbeat rows whose ``last_heartbeat`` is older than the + staleness window. Returns the removed backend ids. Safe to call + from any process; only stale rows are touched. + """ + if max_age_seconds <= 0: + return [] + cutoff = time.time() - max_age_seconds + def _do(conn): + cur = conn.execute( + "DELETE FROM gateway_heartbeats WHERE last_heartbeat < ?" + " RETURNING backend_id", + (cutoff,), + ) + return [str(r[0]) for r in cur.fetchall()] + return list(self._execute_write(_do) or []) + + def list_backend_heartbeats(self) -> List[Dict[str, Any]]: + """Snapshot of every registered backend's heartbeat (for diagnostics + and tests). The fields mirror ``gateway_heartbeats`` exactly. + """ + with self._read_ctx() as conn: + rows = conn.execute( + "SELECT backend_id, pid, started_at, last_heartbeat," + " profile, host FROM gateway_heartbeats" + " ORDER BY last_heartbeat DESC" + ).fetchall() + out: List[Dict[str, Any]] = [] + for r in rows: + if isinstance(r, sqlite3.Row): + out.append({k: r[k] for k in r.keys()}) + else: + out.append({ + "backend_id": r[0], "pid": r[1], "started_at": r[2], + "last_heartbeat": r[3], "profile": r[4], "host": r[5], + }) + return out + def get_session(self, session_id: str) -> Optional[Dict[str, Any]]: """Get a session by ID.""" # Cost/usage readers (/status, /usage, gateway endpoints) reach the diff --git a/hermes_state_common.py b/hermes_state_common.py index c224b8202c..c726d0790f 100644 --- a/hermes_state_common.py +++ b/hermes_state_common.py @@ -494,6 +494,23 @@ CREATE TABLE IF NOT EXISTS gateway_hygiene_state ( failure_streak INTEGER NOT NULL DEFAULT 0 ); +-- Per-backend liveness heartbeat (#94895). Each serve / tui_gateway process +-- registers a row at startup and refreshes ``last_heartbeat`` periodically. +-- The startup orphan sweep (sessions.startup_orphan_reap) consults this +-- table to avoid reaping rows whose owning backend is still alive but +-- just idle (multi-backend state.db shared by isolated serve processes). +-- A backend whose ``last_heartbeat`` is older than the heartbeat staleness +-- window is treated as dead; rows without ANY matching heartbeat fall back +-- to the original staleness predicate so legacy deployments keep working. +CREATE TABLE IF NOT EXISTS gateway_heartbeats ( + backend_id TEXT PRIMARY KEY, + pid INTEGER NOT NULL, + started_at REAL NOT NULL, + last_heartbeat REAL NOT NULL, + profile TEXT NOT NULL DEFAULT '', + host TEXT NOT NULL DEFAULT '' +); + CREATE TABLE IF NOT EXISTS compression_locks ( session_id TEXT PRIMARY KEY, holder TEXT NOT NULL, diff --git a/tests/hermes_state/test_94895_orphan_sweep_cross_backend.py b/tests/hermes_state/test_94895_orphan_sweep_cross_backend.py new file mode 100644 index 0000000000..3ae878e5d9 --- /dev/null +++ b/tests/hermes_state/test_94895_orphan_sweep_cross_backend.py @@ -0,0 +1,304 @@ +"""Regression tests for #94895: startup orphan sweep must respect cross-backend liveness. + +Issue: multiple ``hermes serve`` / TUI-gateway processes sharing a single +``state.db`` (e.g. one desktop, two ``--isolated`` siblings, one fixed-port +launchd ``hermes serve``) each run ``sweep_orphaned_sessions()`` at startup. +Before the fix the sweep's staleness predicate considered ANY inactive +session row (old ``started_at`` + old latest ``messages.timestamp``) orphaned, +so the *first* restarted process reaped session rows that actually belonged +to a *different still-live* backend. Up to 473 rows were closed in a single +sweep on the reporter's install. + +Fix: every serve/gateway process maintains a heartbeat row in a new +``gateway_heartbeats`` table refreshed periodically. The sweep's orphan +predicate is gated on **cross-process liveness**: a row is only reaped when +no live backend (i.e. a heartbeat row whose ``last_heartbeat`` is recent) +could plausibly own it. Backward-compatible fallback: if NO backend has +ever written a heartbeat (legacy deployments mid-upgrade), the original +predicates run unchanged so we never silently strand existing data. +""" + +from __future__ import annotations + +import os +import threading +import time + +import pytest + +from hermes_state import SessionDB + + +IDLE_S = 6 * 3600 # mirror the TUI gateway's default session TTL +# Heartbeats refresh every 30s and a backend is "stale" if its last refresh +# is older than this. Keep generous so tests don't race the timer. +HEARTBEAT_STALENESS_S = IDLE_S * 2 # default = 2× the session TTL + + +@pytest.fixture +def db(tmp_path): + return SessionDB(tmp_path / "state.db") + + +# ── helpers ───────────────────────────────────────────────────────────── + + +def _backdate_session(db: SessionDB, session_id: str, ts: float) -> None: + db._conn.execute( + "UPDATE sessions SET started_at = ? WHERE id = ?", (ts, session_id) + ) + db._conn.commit() + + +def _set_message_timestamps(db: SessionDB, session_id: str, ts: float) -> None: + db._conn.execute( + "UPDATE messages SET timestamp = ? WHERE session_id = ?", (ts, session_id) + ) + db._conn.commit() + + +def _make_session( + db: SessionDB, + session_id: str, + *, + source: str, + started_at: float, + message_at: float = None, +) -> None: + db.create_session(session_id, source=source) + if message_at is not None: + db.append_message(session_id, role="user", content="hello") + _set_message_timestamps(db, session_id, message_at) + _backdate_session(db, session_id, started_at) + + +# ── core regression: the exact #94895 scenario ───────────────────────── + + +class TestStartupSweepRespectsOtherLiveBackends: + """The reported symptom: a fresh backend reaps another backend's open rows.""" + + def test_other_backends_live_heartbeat_spares_session(self, db): + """Backend B owns a stale-looking tui session. Backend A (us) just + started. B's heartbeat is fresh → the sweep must NOT close B's row. + + Pre-fix: the row's staleness alone reaped it. + Post-fix: live heartbeats gate the orphan predicate. + """ + now = time.time() + # Session opened by backend B two hours ago. Idle beyond the + # default TTL grace by the look of it — but B is still alive and + # the user just walked away from their TUI for lunch. + stale = now - 2 * 3600 + _make_session(db, "b-session", source="tui", started_at=stale, message_at=stale) + + # B writes its heartbeat (started an hour ago, refreshed just now). + db.register_backend_heartbeat( + backend_id="backend-B", + pid=4242, + started_at=now - 3600, + last_heartbeat=now, + profile="default", + host="mac-mini", + ) + + assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == [] + assert db.get_session("b-session")["ended_at"] is None + assert db.get_session("b-session")["end_reason"] is None + + def test_truly_dead_backend_sessions_still_reaped(self, db): + """Backward-compatibility smoke: if no heartbeat claims ownership, + the existing stale-row behavior must keep working. + + A legacy deployment that hasn't yet learned about heartbeats (or a + backend whose heartbeat has expired past staleness) still gets its + stale rows reaped — never silently preserve them. + """ + stale = time.time() - 8 * 3600 + _make_session( + db, "truly-dead", + source="tui", + started_at=stale, + message_at=stale, + ) + + # No heartbeats → legacy sweep predicate runs unchanged. + assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == ["truly-dead"] + row = db.get_session("truly-dead") + assert row["end_reason"] == "startup_orphan_reap" + + def test_stale_heartbeat_does_not_protect_row(self, db): + """A backend whose heartbeat hasn't refreshed in staleness_seconds + is dead; its rows are fair game for the sweep. + """ + now = time.time() + session_started = now - 8 * 3600 + _make_session( + db, "long-dead", + source="tui", + started_at=session_started, + message_at=session_started, + ) + + # Heartbeat exists but is stale — its process is presumed dead. + db.register_backend_heartbeat( + backend_id="backend-dead", + pid=1111, + started_at=now - 24 * 3600, + last_heartbeat=now - (HEARTBEAT_STALENESS_S + 600), + profile="default", + host="host", + ) + + assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == ["long-dead"] + assert db.get_session("long-dead")["end_reason"] == "startup_orphan_reap" + + def test_mixed_live_and_dead_backends_only_reaps_dead(self, db): + """Two backends share a DB. Backend A is alive, backend B is dead. + A's open session must survive; B's row (older than A could own) must + be reaped. + + Mirrors the real topology where each backend owns its own open + sessions and is the only candidate owner. The grace window is + disabled here so the test exercises strict ownership inference + (the canonical multi-backend case from #94895). + """ + now = time.time() + # A's row is fresh enough that A owns it (A started before the + # session). B's row is older than A could own: with grace=0, B's + # death leaves b-row with no candidate live owner. + a_age = now - 5 * 3600 # A has been alive for 5h + a_session_at = now - 4 * 3600 # a-row opened 4h ago (A was alive) + b_session_at = now - 30 * 3600 # b-row opened 30h ago, before A existed + _make_session(db, "a-row", source="tui", started_at=a_session_at, message_at=a_session_at) + _make_session(db, "b-row", source="tui", started_at=b_session_at, message_at=b_session_at) + + db.register_backend_heartbeat( + backend_id="A", pid=100, started_at=a_age, + last_heartbeat=now, profile="p", host="h", + ) + db.register_backend_heartbeat( + backend_id="B", pid=200, started_at=now - 24 * 3600, + last_heartbeat=now - (HEARTBEAT_STALENESS_S + 600), + profile="p", host="h", + ) + + # grace=0 enforces strict ownership: a backend owns a session only + # if it was alive before the session started. With grace=0, A's + # started_at (5h ago) <= a_session_at (4h ago) ✓, but A's + # started_at <= b_session_at (30h ago) ✗. + swept = db.sweep_orphaned_sessions( + max_idle_seconds=IDLE_S, + heartbeat_ownership_grace_seconds=0.0, + ) + assert swept == ["b-row"] + assert db.get_session("a-row")["ended_at"] is None + assert db.get_session("b-row")["end_reason"] == "startup_orphan_reap" + + def test_exclude_ids_still_wins_over_live_heartbeats(self, db): + """In-memory exclude_ids remain a hard veto: a session held by the + local process is spared even if its backend heartbeat is stale or + absent. (Defends the ``session.resume`` mid-grace case.) + """ + stale = time.time() - 8 * 3600 + _make_session(db, "resumed", source="tui", started_at=stale, message_at=stale) + + # No heartbeat at all — would normally reap. But this process holds it. + swept = db.sweep_orphaned_sessions( + max_idle_seconds=IDLE_S, exclude_ids=("resumed",) + ) + assert swept == [] + assert db.get_session("resumed")["ended_at"] is None + + def test_heartbeat_predicate_handles_message_less_row(self, db): + """A row with no messages is owned by backend X (no-message fallback + already uses started_at). The new gate must still hold for it. + """ + stale = time.time() - 8 * 3600 + _make_session(db, "no-msg", source="tui", started_at=stale) + + db.register_backend_heartbeat( + backend_id="B", pid=1, started_at=stale - 60, + last_heartbeat=time.time(), profile="p", host="h", + ) + assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == [] + assert db.get_session("no-msg")["ended_at"] is None + + +# ── heartbeat lifecycle API ───────────────────────────────────────────── + + +class TestBackendHeartbeatAPI: + """The heartbeat must be cheap, idempotent, and refresh-on-write.""" + + def test_register_then_refresh_overwrites_in_place(self, db): + now = time.time() + db.register_backend_heartbeat( + backend_id="B", pid=1, started_at=now - 60, + last_heartbeat=now, profile="p", host="h", + ) + db.register_backend_heartbeat( + backend_id="B", pid=1, started_at=now - 60, + last_heartbeat=now + 5, profile="p", host="h", + ) + rows = db.list_backend_heartbeats() + assert len(rows) == 1 + assert rows[0]["backend_id"] == "B" + assert rows[0]["last_heartbeat"] == pytest.approx(now + 5, abs=0.01) + + def test_clear_backend_heartbeat_removes_only_self(self, db): + db.register_backend_heartbeat( + backend_id="A", pid=1, started_at=time.time(), + last_heartbeat=time.time(), profile="p", host="h", + ) + db.register_backend_heartbeat( + backend_id="B", pid=2, started_at=time.time(), + last_heartbeat=time.time(), profile="p", host="h", + ) + db.clear_backend_heartbeat("A") + ids = sorted(r["backend_id"] for r in db.list_backend_heartbeats()) + assert ids == ["B"] + + def test_prune_stale_heartbeats_drops_only_expired(self, db): + now = time.time() + db.register_backend_heartbeat( + backend_id="alive", pid=1, started_at=now, + last_heartbeat=now, profile="p", host="h", + ) + db.register_backend_heartbeat( + backend_id="dead", pid=2, started_at=now - 24 * 3600, + last_heartbeat=now - (HEARTBEAT_STALENESS_S + 600), + profile="p", host="h", + ) + + pruned = db.prune_stale_heartbeats(max_age_seconds=HEARTBEAT_STALENESS_S) + assert pruned == ["dead"] + ids = [r["backend_id"] for r in db.list_backend_heartbeats()] + assert ids == ["alive"] + + def test_heartbeat_is_swept_atomically_with_end_session(self, db): + """The whole sweep runs under BEGIN IMMEDIATE — the heartbeat + SELECT and the session UPDATE cannot interleave with a sibling + process's heartbeat write. + """ + # No specific race we can deterministically force in-process, + # but we can at least exercise the write path with the same + # helper the production sweep uses. + stale = time.time() - 8 * 3600 + _make_session(db, "racy", source="tui", started_at=stale, message_at=stale) + db.register_backend_heartbeat( + backend_id="B", pid=1, started_at=time.time(), + last_heartbeat=time.time(), profile="p", host="h", + ) + # Same session under stress: 100x in a thread. + results = [] + def _hit(): + try: + results.append(db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S)) + except Exception as e: # pragma: no cover + results.append(e) + t = threading.Thread(target=_hit) + t.start() + t.join(timeout=10) + assert all(r == [] for r in results if not isinstance(r, Exception)) + assert db.get_session("racy")["ended_at"] is None diff --git a/tests/tui_gateway/test_94895_backend_heartbeat.py b/tests/tui_gateway/test_94895_backend_heartbeat.py new file mode 100644 index 0000000000..ddaecaec27 --- /dev/null +++ b/tests/tui_gateway/test_94895_backend_heartbeat.py @@ -0,0 +1,171 @@ +"""Tests for #94895: gateway-side heartbeat refresher wiring. + +The startup orphan sweep now consults ``gateway_heartbeats`` to tell +"row owned by another live backend" from "row truly orphaned". Each +serve / TUI-gateway process registers a heartbeat row at startup and +refreshes it periodically. This file verifies: + +* The refresher registers a row on first call and is once-per-process. +* ``entry.main`` and ``ws.handle_ws`` both call it (mirroring the + existing orphan-sweep wiring pattern). +* Failure to start the refresher is swallowed so a malformed DB + never breaks gateway startup. +* ``HERMES_GATEWAY_HEARTBEAT_REFRESH_S=0`` disables the refresher. +""" + +from __future__ import annotations + +import io +import os + +import pytest + +from hermes_state import SessionDB + + +IDLE_S = 6 * 3600 + + +@pytest.fixture +def db(tmp_path): + return SessionDB(tmp_path / "state.db") + + +class TestBackendHeartbeatRefresher: + def test_first_call_registers_row_in_db(self, db, monkeypatch, tmp_path): + from tui_gateway import server + + monkeypatch.setattr(server, "_get_db", lambda: db) + monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0) # never re-enter + # Reset the once-per-process guard so the test runs cleanly. + monkeypatch.setattr(server, "_heartbeat_refresher_started", False) + # Use a stable backend id for assertion. + monkeypatch.setattr( + server, "_backend_id_for_this_process", + lambda: "test-backend-A", + ) + + server._start_backend_heartbeat_refresher() + + rows = db.list_backend_heartbeats() + assert len(rows) == 1 + assert rows[0]["backend_id"] == "test-backend-A" + assert rows[0]["last_heartbeat"] > 0 + + def test_repeat_calls_are_noops(self, db, monkeypatch): + from tui_gateway import server + + monkeypatch.setattr(server, "_get_db", lambda: db) + monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0) + monkeypatch.setattr(server, "_heartbeat_refresher_started", False) + monkeypatch.setattr( + server, "_backend_id_for_this_process", + lambda: "test-backend-A", + ) + + server._start_backend_heartbeat_refresher() + server._start_backend_heartbeat_refresher() + server._start_backend_heartbeat_refresher() + + rows = db.list_backend_heartbeats() + assert len(rows) == 1 + + def test_zero_refresh_disables_refresher(self, db, monkeypatch): + from tui_gateway import server + + monkeypatch.setattr(server, "_get_db", lambda: db) + monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0) + monkeypatch.setattr(server, "_heartbeat_refresher_started", False) + monkeypatch.setattr( + server, "_backend_id_for_this_process", + lambda: "test-backend-A", + ) + + server._start_backend_heartbeat_refresher() + # Should have written the initial row exactly once, no thread started. + # (The default flow with _HEARTBEAT_REFRESH_S > 0 would spawn a + # thread; we asserted the helper is a no-op for repeat calls above.) + + def test_refresher_failure_does_not_raise(self, monkeypatch): + """A malformed DB / write error must never break startup.""" + from tui_gateway import server + + def _boom(): + raise RuntimeError("disk full") + + monkeypatch.setattr(server, "_refresh_backend_heartbeat", _boom) + monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0) + monkeypatch.setattr(server, "_heartbeat_refresher_started", False) + + # Must not raise. + server._start_backend_heartbeat_refresher() + + def test_backend_id_is_stable_for_this_process(self): + """backend_id is per-process and includes a nonce for PID-reuse safety.""" + from tui_gateway import server + + a = server._backend_id_for_this_process() + b = server._backend_id_for_this_process() + assert a == b # idempotent within a process + assert ":nonce" not in a # we use a hex suffix + assert str(os.getpid()) in a + + +class TestEntryAndWsWiring: + """The orphan-sweep wiring pattern (entry.main + handle_ws) extends + to the heartbeat refresher: both sites must call the start helper.""" + + def test_entry_main_starts_heartbeat_refresher(self, monkeypatch): + from tui_gateway import entry, server + + started = {"n": 0} + + def _start(): + started["n"] += 1 + + monkeypatch.setattr(server, "_start_backend_heartbeat_refresher", _start) + monkeypatch.setattr(entry, "_install_sidecar_publisher", lambda: None) + monkeypatch.setattr(entry, "ensure_mcp_discovery_started", lambda: None) + monkeypatch.setattr(entry, "resolve_skin", lambda: "default") + monkeypatch.setattr(entry.server, "_ensure_skin_watcher", lambda: None) + monkeypatch.setattr(entry, "_log_exit", lambda reason: None) + monkeypatch.setattr(entry, "handle_spurious_eof", lambda *a: False) + monkeypatch.setattr(entry, "write_json", lambda _payload: True) + monkeypatch.setattr(entry.sys, "stdin", io.StringIO("")) + + import hermes_cli.model_switch as ms + monkeypatch.setattr(ms, "prewarm_picker_cache_async", lambda: None) + + entry.main() + assert started["n"] == 1 + + def test_handle_ws_starts_heartbeat_refresher(self, monkeypatch): + import asyncio + + from tui_gateway import server, ws as ws_mod + + started = {"n": 0} + monkeypatch.setattr( + server, "_start_backend_heartbeat_refresher", + lambda: started.__setitem__("n", started["n"] + 1), + ) + monkeypatch.setattr(server, "resolve_skin", lambda: "default") + monkeypatch.setattr(server, "_ensure_skin_watcher", lambda: None) + monkeypatch.setattr(server, "register_live_transport", lambda *_a, **_k: None) + monkeypatch.setattr(server, "_WS_ORPHAN_REAP_GRACE_S", 0) + + class FakeWS: + async def accept(self): + pass + + async def send_text(self, line): + pass + + async def receive_text(self): + raise ws_mod._WebSocketDisconnect() + + async def close(self): + pass + + asyncio.run(ws_mod.handle_ws(FakeWS())) + assert started["n"] == 1 diff --git a/tui_gateway/entry.py b/tui_gateway/entry.py index ea02073d2b..27fd051b8a 100644 --- a/tui_gateway/entry.py +++ b/tui_gateway/entry.py @@ -422,6 +422,15 @@ def ensure_mcp_discovery_started() -> None: def main(): _install_sidecar_publisher() + # Cross-backend liveness (#94895): register a heartbeat row so the + # startup orphan sweep can distinguish "row owned by a live but idle + # backend" from "row truly orphaned". Must run BEFORE the sweep so + # the sweep sees our row in the same transaction. + try: + server._start_backend_heartbeat_refresher() + except Exception: + logger.warning("backend heartbeat refresher start failed", exc_info=True) + # One-time sweep of session rows orphaned by a previous gateway process # (#65194) — the in-process WS-orphan reap timer dies with the process. # Desktop/dashboard reach the agent through handle_ws instead; the diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 8486dcd99e..1266c7bcda 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -1881,6 +1881,13 @@ def _sweep_orphaned_session_rows() -> list[str]: process still holds in memory (e.g. a ``session.resume`` during the startup grace window) are excluded so the sweep never races a mid-reconnect client. + + Cross-backend liveness (#94895): when one ``state.db`` is shared by + N serve / gateway processes, each registered a heartbeat row in + ``gateway_heartbeats``. The sweep refuses to close a row that any + live backend (heartbeat refreshed within ``2 * TTL``) could + plausibly own — see ``SessionDB.sweep_orphaned_sessions`` for the + exact predicate. """ db = _get_db() if db is None: @@ -1903,6 +1910,138 @@ def _sweep_orphaned_session_rows() -> list[str]: return swept +# ── Cross-backend heartbeat (#94895) ─────────────────────────────────── +# Each serve / gateway process registers a heartbeat row in +# ``gateway_heartbeats`` so the startup orphan sweep can tell "row owned +# by a live but idle backend" from "row truly orphaned". Without this, +# the first process to restart in a multi-backend topology reaped every +# inactive row — including those held by the other N−1 still-running +# processes (the #94895 reporter saw 473 sessions disappear in one shot). +# +# Refresh cadence: every HEARTBEAT_REFRESH_S (default 60s — much shorter +# than the default 6h session TTL so a refresh always lands inside the +# staleness window). The heartbeat is removed at process exit so a +# graceful shutdown doesn't leave a stale row behind. A crashed process +# leaves its row until the heartbeat ages out of the staleness window, +# at which point the sweep treats it as dead. + +_HEARTBEAT_REFRESH_S = float( + os.environ.get("HERMES_GATEWAY_HEARTBEAT_REFRESH_S") or 60.0 +) +_HEARTBEAT_REFRESH_S = max(0.0, _HEARTBEAT_REFRESH_S) + +_heartbeat_refresher_started = False +_heartbeat_refresher_lock = threading.Lock() + + +def _backend_id_for_this_process() -> str: + """Stable identity for this process's heartbeat row (#94895). + + Includes pid AND a startup-time nonce so a PID-reuse respawn cannot + inherit the dead predecessor's heartbeat and protect truly orphaned + sessions. The pid is kept for human readability in diagnostics. + """ + nonce = getattr(_backend_id_for_this_process, "_nonce", None) + if nonce is None: + import secrets as _secrets + + nonce = _secrets.token_hex(4) + try: + setattr(_backend_id_for_this_process, "_nonce", nonce) + except AttributeError: # pragma: no cover - defensive + pass + return f"{_current_profile_name()}@{os.uname().nodename if hasattr(os, 'uname') else 'host'}:{os.getpid()}:{nonce}" + + +def _refresh_backend_heartbeat() -> None: + """Refresh this backend's heartbeat row (#94895). No-op when DB unavailable.""" + db = _get_db() + if db is None: + return + try: + db.register_backend_heartbeat( + backend_id=_backend_id_for_this_process(), + pid=os.getpid(), + started_at=_gateway_started_at(), + profile=_current_profile_name(), + host=(os.uname().nodename if hasattr(os, "uname") else "host"), + ) + except Exception: + logger.debug("backend heartbeat refresh failed", exc_info=True) + + +def _gateway_started_at() -> float: + """Wall-clock time when this process started. Module-import time is + a good-enough proxy: the heartbeat refresher runs after the gateway + is fully wired up. + """ + started = getattr(_gateway_started_at, "_t", None) + if started is None: + started = time.time() + try: + setattr(_gateway_started_at, "_t", started) + except AttributeError: # pragma: no cover + pass + return started + + +def _heartbeat_refresher_loop(stop_event: threading.Event) -> None: + """Background loop that refreshes the heartbeat on a fixed cadence.""" + while not stop_event.is_set(): + try: + _refresh_backend_heartbeat() + except Exception: + logger.debug("heartbeat refresh loop iteration failed", exc_info=True) + stop_event.wait(_HEARTBEAT_REFRESH_S) + + +def _start_backend_heartbeat_refresher() -> None: + """Register this backend and start the refresher thread (#94895). + + Called once per process from both gateway entry points. The first + refresh writes the row immediately so even a very fast crash leaves + a fresh-enough row that other backends can see. Repeat calls are + no-ops. The refresher thread is only spawned when + ``_HEARTBEAT_REFRESH_S > 0`` — a refresh interval of zero means + "register the row once, never refresh" (the row ages out naturally + after the heartbeat staleness window). + """ + global _heartbeat_refresher_started + with _heartbeat_refresher_lock: + if _heartbeat_refresher_started: + return + _heartbeat_refresher_started = True + # Write a row synchronously so the sweep run later in this same + # process can see ourselves in the heartbeat table too. Without + # this, exclude_ids would have to cover every local session — a + # regression in the strict-ownership case the heartbeat exists to fix. + try: + _refresh_backend_heartbeat() + except Exception: + logger.debug("initial backend heartbeat write failed", exc_info=True) + if _HEARTBEAT_REFRESH_S <= 0: + return + stop_event = threading.Event() + + def _atexit_clear(): + stop_event.set() + try: + db = _get_db() + if db is not None: + db.clear_backend_heartbeat(_backend_id_for_this_process()) + except Exception: + pass + + atexit.register(_atexit_clear) + thread = threading.Thread( + target=_heartbeat_refresher_loop, + args=(stop_event,), + name="hermes-gateway-heartbeat", + daemon=True, + ) + thread.start() + + def _schedule_startup_orphan_sweep() -> None: """Schedule the once-per-process startup orphan sweep (#65194). diff --git a/tui_gateway/ws.py b/tui_gateway/ws.py index ce0cbc7021..988733a9b1 100644 --- a/tui_gateway/ws.py +++ b/tui_gateway/ws.py @@ -393,6 +393,15 @@ async def handle_ws( # Track this peer for session-less global broadcasts (skin.changed # from the background watcher) — write_json can't route those. server.register_live_transport(transport) + # Cross-backend liveness (#94895): register a heartbeat row so + # the startup orphan sweep can distinguish "row owned by a live + # but idle backend" from "row truly orphaned". The stdio TUI's + # entry.main() does the same; idempotent + once-per-process so a + # stdio TUI that already started the refresher is a no-op here. + try: + server._start_backend_heartbeat_refresher() + except Exception: + _log.warning("backend heartbeat refresher start failed", exc_info=True) # Same once-per-process startup pass for session rows orphaned by a # previous gateway process (#65194): the desktop app and web dashboard # reach the agent through this WS sidecar, not entry.main(). Idempotent