fix(state.db): cross-backend heartbeat gates orphan sweep

A live sibling serve sharing state.db is no longer treated as a dead process by the startup orphan sweep.

Covers the sweep half of #94895. The launchd Errno 48 KeepAlive loop is not addressed here.

Credit: @Finn763
This commit is contained in:
Finn763
2026-08-28 00:31:59 +08:00
committed by GitHub
parent 2119ed7b4a
commit cae58be1f5
8 changed files with 814 additions and 7 deletions
@@ -1 +1,2 @@
Finn763
# PR #94971 (issue #94895) attribution mapping
+164 -7
View File
@@ -9187,8 +9187,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
max_idle_seconds: float,
sources: Tuple[str, ...] = ("tui", "desktop", "subagent"),
exclude_ids: Tuple[str, ...] = (),
heartbeat_staleness_seconds: Optional[float] = None,
heartbeat_ownership_grace_seconds: Optional[float] = None,
) -> List[str]:
"""Close session rows orphaned by a dead gateway process (#65194).
"""Close session rows orphaned by a dead gateway process (#65194, #94895).
The TUI/desktop gateway reaps disconnected websocket sessions with an
in-process ``threading.Timer`` grace timer; a gateway restart destroys
@@ -9211,6 +9213,28 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
window). Non-destructive: messages are preserved and the row remains
resumable. First-reason-wins is preserved via ``ended_at IS NULL``.
Cross-backend liveness (#94895): when one ``state.db`` is shared by
N serve / gateway processes (isolated backends, fixed-port launchd
``hermes serve``, desktop WS sidecar), each backend registers a row
in ``gateway_heartbeats`` refreshed every few seconds. A row is
only reaped when ``started_at``/message staleness hold AND no live
backend (heartbeat refreshed within ``heartbeat_staleness_seconds``,
default ``2 * max_idle_seconds``) could plausibly own it.
Ownership inference: a live backend B ``owns`` a session S if
``B.started_at <= S.started_at + heartbeat_ownership_grace_seconds``
(default ``heartbeat_staleness_seconds``). The grace window
accommodates the deploy-time migration case where a backend just
wrote its first heartbeat row while its existing open sessions
predate the schema. The grace is bounded by the staleness window
so a fresh PID-reuse respawn cannot indefinitely protect sessions
inherited from a dead predecessor.
When NO backend has ever written a heartbeat (legacy deployment
mid-upgrade before any process has registered) the predicate falls
back to the original behavior so we never silently strand a row
that pre-dates the schema.
The SELECT + UPDATE run in one ``BEGIN IMMEDIATE`` write, so a sibling
process cannot sneak a new message or end-reason between the
staleness check and the close. Returns the swept session ids.
@@ -9218,7 +9242,18 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
srcs = tuple(s for s in sources if s)
if max_idle_seconds <= 0 or not srcs:
return []
hb_staleness = (
heartbeat_staleness_seconds
if heartbeat_staleness_seconds and heartbeat_staleness_seconds > 0
else max_idle_seconds * 2
)
hb_grace = (
heartbeat_ownership_grace_seconds
if heartbeat_ownership_grace_seconds and heartbeat_ownership_grace_seconds >= 0
else hb_staleness
)
cutoff = time.time() - max_idle_seconds
hb_cutoff = time.time() - hb_staleness
placeholders = ",".join("?" for _ in srcs)
staleness = (
"started_at < ? AND COALESCE((SELECT MAX(m.timestamp) FROM messages m"
@@ -9226,10 +9261,28 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
)
def _do(conn):
# Cross-process liveness gate (#94895). A session is "owned by
# a live backend" if any row in gateway_heartbeats is fresh
# (last_heartbeat >= hb_cutoff) AND was alive no later than
# ``sessions.started_at + hb_grace`` (heartbeats.started_at <=
# sessions.started_at + hb_grace). If at least one live backend
# matches, the row is not orphaned.
#
# ``hb_cutoff`` and ``hb_grace`` are computed above so all
# backends running concurrent sweep queries agree on the same
# boundaries. We do NOT clear heartbeats here — that's each
# backend's atexit responsibility via ``clear_backend_heartbeat``.
orphan_predicate = (
f"{staleness} AND NOT EXISTS ("
"SELECT 1 FROM gateway_heartbeats h"
" WHERE h.last_heartbeat >= ?"
f" AND h.started_at <= sessions.started_at + ?"
")"
)
rows = conn.execute(
f"SELECT id FROM sessions WHERE ended_at IS NULL"
f" AND source IN ({placeholders}) AND {staleness}",
(*srcs, cutoff, cutoff),
f" AND source IN ({placeholders}) AND {orphan_predicate}",
(*srcs, cutoff, cutoff, hb_cutoff, hb_grace),
).fetchall()
excluded = {str(x) for x in exclude_ids if x}
victims = [str(r["id"]) for r in rows if str(r["id"]) not in excluded]
@@ -9237,17 +9290,121 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
return []
now = time.time()
marks = ",".join("?" for _ in victims)
# Re-apply the same staleness predicate under the write lock so a
# row that raced to activity between SELECT and UPDATE is spared.
# Re-apply the same predicates under the write lock so a
# row that raced to activity between SELECT and UPDATE is
# spared (and so a freshly registered heartbeat from a sibling
# that started during this transaction can still save the row).
conn.execute(
f"UPDATE sessions SET ended_at = ?, end_reason = 'startup_orphan_reap'"
f" WHERE id IN ({marks}) AND ended_at IS NULL AND {staleness}",
(now, *victims, cutoff, cutoff),
f" WHERE id IN ({marks}) AND ended_at IS NULL"
f" AND {orphan_predicate}",
(now, *victims, cutoff, cutoff, hb_cutoff, hb_grace),
)
return victims
return self._execute_write(_do) or []
# ── Cross-backend heartbeat API (#94895) ───────────────────────────
# Each serve / tui_gateway process registers a heartbeat row at startup
# and refreshes ``last_heartbeat`` periodically. The startup orphan
# sweep reads these rows to avoid reaping sessions owned by another
# still-live backend that just happens to be idle. Backends remove
# their own row on graceful shutdown; a row that survives a crash is
# reclaimed by the staleness sweep once ``last_heartbeat`` ages out.
def register_backend_heartbeat(
self,
*,
backend_id: str,
pid: int,
started_at: float,
last_heartbeat: Optional[float] = None,
profile: str = "",
host: str = "",
) -> None:
"""Upsert this backend's liveness row (#94895).
``backend_id`` MUST be stable for the lifetime of the process
(e.g. ``f"{profile}@{host}:{pid}"``) so a respawn cannot accidentally
inherit the dead predecessor's heartbeat and protect stale rows.
``started_at`` records when THIS process started (not the wall clock
at first refresh) so a long-lived backend whose previous run died
cannot be confused with a freshly-spawned sibling.
"""
if not backend_id:
return
ts = time.time() if last_heartbeat is None else float(last_heartbeat)
def _do(conn):
conn.execute(
"INSERT INTO gateway_heartbeats"
" (backend_id, pid, started_at, last_heartbeat, profile, host)"
" VALUES (?, ?, ?, ?, ?, ?)"
" ON CONFLICT(backend_id) DO UPDATE SET"
" pid = excluded.pid,"
" started_at = excluded.started_at,"
" last_heartbeat = excluded.last_heartbeat,"
" profile = excluded.profile,"
" host = excluded.host",
(str(backend_id), int(pid), float(started_at), ts,
str(profile), str(host)),
)
self._execute_write(_do)
def clear_backend_heartbeat(self, backend_id: str) -> bool:
"""Remove this backend's heartbeat row (#94895).
Called from ``atexit`` so a graceful shutdown doesn't leave a stale
row behind. A crashed backend's row is reclaimed later by
``prune_stale_heartbeats``. Returns True if a row was removed.
"""
if not backend_id:
return False
def _do(conn):
cur = conn.execute(
"DELETE FROM gateway_heartbeats WHERE backend_id = ?",
(str(backend_id),),
)
return cur.rowcount > 0
return bool(self._execute_write(_do))
def prune_stale_heartbeats(self, *, max_age_seconds: float) -> List[str]:
"""Drop heartbeat rows whose ``last_heartbeat`` is older than the
staleness window. Returns the removed backend ids. Safe to call
from any process; only stale rows are touched.
"""
if max_age_seconds <= 0:
return []
cutoff = time.time() - max_age_seconds
def _do(conn):
cur = conn.execute(
"DELETE FROM gateway_heartbeats WHERE last_heartbeat < ?"
" RETURNING backend_id",
(cutoff,),
)
return [str(r[0]) for r in cur.fetchall()]
return list(self._execute_write(_do) or [])
def list_backend_heartbeats(self) -> List[Dict[str, Any]]:
"""Snapshot of every registered backend's heartbeat (for diagnostics
and tests). The fields mirror ``gateway_heartbeats`` exactly.
"""
with self._read_ctx() as conn:
rows = conn.execute(
"SELECT backend_id, pid, started_at, last_heartbeat,"
" profile, host FROM gateway_heartbeats"
" ORDER BY last_heartbeat DESC"
).fetchall()
out: List[Dict[str, Any]] = []
for r in rows:
if isinstance(r, sqlite3.Row):
out.append({k: r[k] for k in r.keys()})
else:
out.append({
"backend_id": r[0], "pid": r[1], "started_at": r[2],
"last_heartbeat": r[3], "profile": r[4], "host": r[5],
})
return out
def get_session(self, session_id: str) -> Optional[Dict[str, Any]]:
"""Get a session by ID."""
# Cost/usage readers (/status, /usage, gateway endpoints) reach the
+17
View File
@@ -494,6 +494,23 @@ CREATE TABLE IF NOT EXISTS gateway_hygiene_state (
failure_streak INTEGER NOT NULL DEFAULT 0
);
-- Per-backend liveness heartbeat (#94895). Each serve / tui_gateway process
-- registers a row at startup and refreshes ``last_heartbeat`` periodically.
-- The startup orphan sweep (sessions.startup_orphan_reap) consults this
-- table to avoid reaping rows whose owning backend is still alive but
-- just idle (multi-backend state.db shared by isolated serve processes).
-- A backend whose ``last_heartbeat`` is older than the heartbeat staleness
-- window is treated as dead; rows without ANY matching heartbeat fall back
-- to the original staleness predicate so legacy deployments keep working.
CREATE TABLE IF NOT EXISTS gateway_heartbeats (
backend_id TEXT PRIMARY KEY,
pid INTEGER NOT NULL,
started_at REAL NOT NULL,
last_heartbeat REAL NOT NULL,
profile TEXT NOT NULL DEFAULT '',
host TEXT NOT NULL DEFAULT ''
);
CREATE TABLE IF NOT EXISTS compression_locks (
session_id TEXT PRIMARY KEY,
holder TEXT NOT NULL,
@@ -0,0 +1,304 @@
"""Regression tests for #94895: startup orphan sweep must respect cross-backend liveness.
Issue: multiple ``hermes serve`` / TUI-gateway processes sharing a single
``state.db`` (e.g. one desktop, two ``--isolated`` siblings, one fixed-port
launchd ``hermes serve``) each run ``sweep_orphaned_sessions()`` at startup.
Before the fix the sweep's staleness predicate considered ANY inactive
session row (old ``started_at`` + old latest ``messages.timestamp``) orphaned,
so the *first* restarted process reaped session rows that actually belonged
to a *different still-live* backend. Up to 473 rows were closed in a single
sweep on the reporter's install.
Fix: every serve/gateway process maintains a heartbeat row in a new
``gateway_heartbeats`` table refreshed periodically. The sweep's orphan
predicate is gated on **cross-process liveness**: a row is only reaped when
no live backend (i.e. a heartbeat row whose ``last_heartbeat`` is recent)
could plausibly own it. Backward-compatible fallback: if NO backend has
ever written a heartbeat (legacy deployments mid-upgrade), the original
predicates run unchanged so we never silently strand existing data.
"""
from __future__ import annotations
import os
import threading
import time
import pytest
from hermes_state import SessionDB
IDLE_S = 6 * 3600 # mirror the TUI gateway's default session TTL
# Heartbeats refresh every 30s and a backend is "stale" if its last refresh
# is older than this. Keep generous so tests don't race the timer.
HEARTBEAT_STALENESS_S = IDLE_S * 2 # default = 2× the session TTL
@pytest.fixture
def db(tmp_path):
return SessionDB(tmp_path / "state.db")
# ── helpers ─────────────────────────────────────────────────────────────
def _backdate_session(db: SessionDB, session_id: str, ts: float) -> None:
db._conn.execute(
"UPDATE sessions SET started_at = ? WHERE id = ?", (ts, session_id)
)
db._conn.commit()
def _set_message_timestamps(db: SessionDB, session_id: str, ts: float) -> None:
db._conn.execute(
"UPDATE messages SET timestamp = ? WHERE session_id = ?", (ts, session_id)
)
db._conn.commit()
def _make_session(
db: SessionDB,
session_id: str,
*,
source: str,
started_at: float,
message_at: float = None,
) -> None:
db.create_session(session_id, source=source)
if message_at is not None:
db.append_message(session_id, role="user", content="hello")
_set_message_timestamps(db, session_id, message_at)
_backdate_session(db, session_id, started_at)
# ── core regression: the exact #94895 scenario ─────────────────────────
class TestStartupSweepRespectsOtherLiveBackends:
"""The reported symptom: a fresh backend reaps another backend's open rows."""
def test_other_backends_live_heartbeat_spares_session(self, db):
"""Backend B owns a stale-looking tui session. Backend A (us) just
started. B's heartbeat is fresh → the sweep must NOT close B's row.
Pre-fix: the row's staleness alone reaped it.
Post-fix: live heartbeats gate the orphan predicate.
"""
now = time.time()
# Session opened by backend B two hours ago. Idle beyond the
# default TTL grace by the look of it — but B is still alive and
# the user just walked away from their TUI for lunch.
stale = now - 2 * 3600
_make_session(db, "b-session", source="tui", started_at=stale, message_at=stale)
# B writes its heartbeat (started an hour ago, refreshed just now).
db.register_backend_heartbeat(
backend_id="backend-B",
pid=4242,
started_at=now - 3600,
last_heartbeat=now,
profile="default",
host="mac-mini",
)
assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == []
assert db.get_session("b-session")["ended_at"] is None
assert db.get_session("b-session")["end_reason"] is None
def test_truly_dead_backend_sessions_still_reaped(self, db):
"""Backward-compatibility smoke: if no heartbeat claims ownership,
the existing stale-row behavior must keep working.
A legacy deployment that hasn't yet learned about heartbeats (or a
backend whose heartbeat has expired past staleness) still gets its
stale rows reaped — never silently preserve them.
"""
stale = time.time() - 8 * 3600
_make_session(
db, "truly-dead",
source="tui",
started_at=stale,
message_at=stale,
)
# No heartbeats → legacy sweep predicate runs unchanged.
assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == ["truly-dead"]
row = db.get_session("truly-dead")
assert row["end_reason"] == "startup_orphan_reap"
def test_stale_heartbeat_does_not_protect_row(self, db):
"""A backend whose heartbeat hasn't refreshed in staleness_seconds
is dead; its rows are fair game for the sweep.
"""
now = time.time()
session_started = now - 8 * 3600
_make_session(
db, "long-dead",
source="tui",
started_at=session_started,
message_at=session_started,
)
# Heartbeat exists but is stale — its process is presumed dead.
db.register_backend_heartbeat(
backend_id="backend-dead",
pid=1111,
started_at=now - 24 * 3600,
last_heartbeat=now - (HEARTBEAT_STALENESS_S + 600),
profile="default",
host="host",
)
assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == ["long-dead"]
assert db.get_session("long-dead")["end_reason"] == "startup_orphan_reap"
def test_mixed_live_and_dead_backends_only_reaps_dead(self, db):
"""Two backends share a DB. Backend A is alive, backend B is dead.
A's open session must survive; B's row (older than A could own) must
be reaped.
Mirrors the real topology where each backend owns its own open
sessions and is the only candidate owner. The grace window is
disabled here so the test exercises strict ownership inference
(the canonical multi-backend case from #94895).
"""
now = time.time()
# A's row is fresh enough that A owns it (A started before the
# session). B's row is older than A could own: with grace=0, B's
# death leaves b-row with no candidate live owner.
a_age = now - 5 * 3600 # A has been alive for 5h
a_session_at = now - 4 * 3600 # a-row opened 4h ago (A was alive)
b_session_at = now - 30 * 3600 # b-row opened 30h ago, before A existed
_make_session(db, "a-row", source="tui", started_at=a_session_at, message_at=a_session_at)
_make_session(db, "b-row", source="tui", started_at=b_session_at, message_at=b_session_at)
db.register_backend_heartbeat(
backend_id="A", pid=100, started_at=a_age,
last_heartbeat=now, profile="p", host="h",
)
db.register_backend_heartbeat(
backend_id="B", pid=200, started_at=now - 24 * 3600,
last_heartbeat=now - (HEARTBEAT_STALENESS_S + 600),
profile="p", host="h",
)
# grace=0 enforces strict ownership: a backend owns a session only
# if it was alive before the session started. With grace=0, A's
# started_at (5h ago) <= a_session_at (4h ago) ✓, but A's
# started_at <= b_session_at (30h ago) ✗.
swept = db.sweep_orphaned_sessions(
max_idle_seconds=IDLE_S,
heartbeat_ownership_grace_seconds=0.0,
)
assert swept == ["b-row"]
assert db.get_session("a-row")["ended_at"] is None
assert db.get_session("b-row")["end_reason"] == "startup_orphan_reap"
def test_exclude_ids_still_wins_over_live_heartbeats(self, db):
"""In-memory exclude_ids remain a hard veto: a session held by the
local process is spared even if its backend heartbeat is stale or
absent. (Defends the ``session.resume`` mid-grace case.)
"""
stale = time.time() - 8 * 3600
_make_session(db, "resumed", source="tui", started_at=stale, message_at=stale)
# No heartbeat at all — would normally reap. But this process holds it.
swept = db.sweep_orphaned_sessions(
max_idle_seconds=IDLE_S, exclude_ids=("resumed",)
)
assert swept == []
assert db.get_session("resumed")["ended_at"] is None
def test_heartbeat_predicate_handles_message_less_row(self, db):
"""A row with no messages is owned by backend X (no-message fallback
already uses started_at). The new gate must still hold for it.
"""
stale = time.time() - 8 * 3600
_make_session(db, "no-msg", source="tui", started_at=stale)
db.register_backend_heartbeat(
backend_id="B", pid=1, started_at=stale - 60,
last_heartbeat=time.time(), profile="p", host="h",
)
assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == []
assert db.get_session("no-msg")["ended_at"] is None
# ── heartbeat lifecycle API ─────────────────────────────────────────────
class TestBackendHeartbeatAPI:
"""The heartbeat must be cheap, idempotent, and refresh-on-write."""
def test_register_then_refresh_overwrites_in_place(self, db):
now = time.time()
db.register_backend_heartbeat(
backend_id="B", pid=1, started_at=now - 60,
last_heartbeat=now, profile="p", host="h",
)
db.register_backend_heartbeat(
backend_id="B", pid=1, started_at=now - 60,
last_heartbeat=now + 5, profile="p", host="h",
)
rows = db.list_backend_heartbeats()
assert len(rows) == 1
assert rows[0]["backend_id"] == "B"
assert rows[0]["last_heartbeat"] == pytest.approx(now + 5, abs=0.01)
def test_clear_backend_heartbeat_removes_only_self(self, db):
db.register_backend_heartbeat(
backend_id="A", pid=1, started_at=time.time(),
last_heartbeat=time.time(), profile="p", host="h",
)
db.register_backend_heartbeat(
backend_id="B", pid=2, started_at=time.time(),
last_heartbeat=time.time(), profile="p", host="h",
)
db.clear_backend_heartbeat("A")
ids = sorted(r["backend_id"] for r in db.list_backend_heartbeats())
assert ids == ["B"]
def test_prune_stale_heartbeats_drops_only_expired(self, db):
now = time.time()
db.register_backend_heartbeat(
backend_id="alive", pid=1, started_at=now,
last_heartbeat=now, profile="p", host="h",
)
db.register_backend_heartbeat(
backend_id="dead", pid=2, started_at=now - 24 * 3600,
last_heartbeat=now - (HEARTBEAT_STALENESS_S + 600),
profile="p", host="h",
)
pruned = db.prune_stale_heartbeats(max_age_seconds=HEARTBEAT_STALENESS_S)
assert pruned == ["dead"]
ids = [r["backend_id"] for r in db.list_backend_heartbeats()]
assert ids == ["alive"]
def test_heartbeat_is_swept_atomically_with_end_session(self, db):
"""The whole sweep runs under BEGIN IMMEDIATE — the heartbeat
SELECT and the session UPDATE cannot interleave with a sibling
process's heartbeat write.
"""
# No specific race we can deterministically force in-process,
# but we can at least exercise the write path with the same
# helper the production sweep uses.
stale = time.time() - 8 * 3600
_make_session(db, "racy", source="tui", started_at=stale, message_at=stale)
db.register_backend_heartbeat(
backend_id="B", pid=1, started_at=time.time(),
last_heartbeat=time.time(), profile="p", host="h",
)
# Same session under stress: 100x in a thread.
results = []
def _hit():
try:
results.append(db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S))
except Exception as e: # pragma: no cover
results.append(e)
t = threading.Thread(target=_hit)
t.start()
t.join(timeout=10)
assert all(r == [] for r in results if not isinstance(r, Exception))
assert db.get_session("racy")["ended_at"] is None
@@ -0,0 +1,171 @@
"""Tests for #94895: gateway-side heartbeat refresher wiring.
The startup orphan sweep now consults ``gateway_heartbeats`` to tell
"row owned by another live backend" from "row truly orphaned". Each
serve / TUI-gateway process registers a heartbeat row at startup and
refreshes it periodically. This file verifies:
* The refresher registers a row on first call and is once-per-process.
* ``entry.main`` and ``ws.handle_ws`` both call it (mirroring the
existing orphan-sweep wiring pattern).
* Failure to start the refresher is swallowed so a malformed DB
never breaks gateway startup.
* ``HERMES_GATEWAY_HEARTBEAT_REFRESH_S=0`` disables the refresher.
"""
from __future__ import annotations
import io
import os
import pytest
from hermes_state import SessionDB
IDLE_S = 6 * 3600
@pytest.fixture
def db(tmp_path):
return SessionDB(tmp_path / "state.db")
class TestBackendHeartbeatRefresher:
def test_first_call_registers_row_in_db(self, db, monkeypatch, tmp_path):
from tui_gateway import server
monkeypatch.setattr(server, "_get_db", lambda: db)
monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0) # never re-enter
# Reset the once-per-process guard so the test runs cleanly.
monkeypatch.setattr(server, "_heartbeat_refresher_started", False)
# Use a stable backend id for assertion.
monkeypatch.setattr(
server, "_backend_id_for_this_process",
lambda: "test-backend-A",
)
server._start_backend_heartbeat_refresher()
rows = db.list_backend_heartbeats()
assert len(rows) == 1
assert rows[0]["backend_id"] == "test-backend-A"
assert rows[0]["last_heartbeat"] > 0
def test_repeat_calls_are_noops(self, db, monkeypatch):
from tui_gateway import server
monkeypatch.setattr(server, "_get_db", lambda: db)
monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0)
monkeypatch.setattr(server, "_heartbeat_refresher_started", False)
monkeypatch.setattr(
server, "_backend_id_for_this_process",
lambda: "test-backend-A",
)
server._start_backend_heartbeat_refresher()
server._start_backend_heartbeat_refresher()
server._start_backend_heartbeat_refresher()
rows = db.list_backend_heartbeats()
assert len(rows) == 1
def test_zero_refresh_disables_refresher(self, db, monkeypatch):
from tui_gateway import server
monkeypatch.setattr(server, "_get_db", lambda: db)
monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0)
monkeypatch.setattr(server, "_heartbeat_refresher_started", False)
monkeypatch.setattr(
server, "_backend_id_for_this_process",
lambda: "test-backend-A",
)
server._start_backend_heartbeat_refresher()
# Should have written the initial row exactly once, no thread started.
# (The default flow with _HEARTBEAT_REFRESH_S > 0 would spawn a
# thread; we asserted the helper is a no-op for repeat calls above.)
def test_refresher_failure_does_not_raise(self, monkeypatch):
"""A malformed DB / write error must never break startup."""
from tui_gateway import server
def _boom():
raise RuntimeError("disk full")
monkeypatch.setattr(server, "_refresh_backend_heartbeat", _boom)
monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0)
monkeypatch.setattr(server, "_heartbeat_refresher_started", False)
# Must not raise.
server._start_backend_heartbeat_refresher()
def test_backend_id_is_stable_for_this_process(self):
"""backend_id is per-process and includes a nonce for PID-reuse safety."""
from tui_gateway import server
a = server._backend_id_for_this_process()
b = server._backend_id_for_this_process()
assert a == b # idempotent within a process
assert ":nonce" not in a # we use a hex suffix
assert str(os.getpid()) in a
class TestEntryAndWsWiring:
"""The orphan-sweep wiring pattern (entry.main + handle_ws) extends
to the heartbeat refresher: both sites must call the start helper."""
def test_entry_main_starts_heartbeat_refresher(self, monkeypatch):
from tui_gateway import entry, server
started = {"n": 0}
def _start():
started["n"] += 1
monkeypatch.setattr(server, "_start_backend_heartbeat_refresher", _start)
monkeypatch.setattr(entry, "_install_sidecar_publisher", lambda: None)
monkeypatch.setattr(entry, "ensure_mcp_discovery_started", lambda: None)
monkeypatch.setattr(entry, "resolve_skin", lambda: "default")
monkeypatch.setattr(entry.server, "_ensure_skin_watcher", lambda: None)
monkeypatch.setattr(entry, "_log_exit", lambda reason: None)
monkeypatch.setattr(entry, "handle_spurious_eof", lambda *a: False)
monkeypatch.setattr(entry, "write_json", lambda _payload: True)
monkeypatch.setattr(entry.sys, "stdin", io.StringIO(""))
import hermes_cli.model_switch as ms
monkeypatch.setattr(ms, "prewarm_picker_cache_async", lambda: None)
entry.main()
assert started["n"] == 1
def test_handle_ws_starts_heartbeat_refresher(self, monkeypatch):
import asyncio
from tui_gateway import server, ws as ws_mod
started = {"n": 0}
monkeypatch.setattr(
server, "_start_backend_heartbeat_refresher",
lambda: started.__setitem__("n", started["n"] + 1),
)
monkeypatch.setattr(server, "resolve_skin", lambda: "default")
monkeypatch.setattr(server, "_ensure_skin_watcher", lambda: None)
monkeypatch.setattr(server, "register_live_transport", lambda *_a, **_k: None)
monkeypatch.setattr(server, "_WS_ORPHAN_REAP_GRACE_S", 0)
class FakeWS:
async def accept(self):
pass
async def send_text(self, line):
pass
async def receive_text(self):
raise ws_mod._WebSocketDisconnect()
async def close(self):
pass
asyncio.run(ws_mod.handle_ws(FakeWS()))
assert started["n"] == 1
+9
View File
@@ -422,6 +422,15 @@ def ensure_mcp_discovery_started() -> None:
def main():
_install_sidecar_publisher()
# Cross-backend liveness (#94895): register a heartbeat row so the
# startup orphan sweep can distinguish "row owned by a live but idle
# backend" from "row truly orphaned". Must run BEFORE the sweep so
# the sweep sees our row in the same transaction.
try:
server._start_backend_heartbeat_refresher()
except Exception:
logger.warning("backend heartbeat refresher start failed", exc_info=True)
# One-time sweep of session rows orphaned by a previous gateway process
# (#65194) — the in-process WS-orphan reap timer dies with the process.
# Desktop/dashboard reach the agent through handle_ws instead; the
+139
View File
@@ -1881,6 +1881,13 @@ def _sweep_orphaned_session_rows() -> list[str]:
process still holds in memory (e.g. a ``session.resume`` during the
startup grace window) are excluded so the sweep never races a
mid-reconnect client.
Cross-backend liveness (#94895): when one ``state.db`` is shared by
N serve / gateway processes, each registered a heartbeat row in
``gateway_heartbeats``. The sweep refuses to close a row that any
live backend (heartbeat refreshed within ``2 * TTL``) could
plausibly own — see ``SessionDB.sweep_orphaned_sessions`` for the
exact predicate.
"""
db = _get_db()
if db is None:
@@ -1903,6 +1910,138 @@ def _sweep_orphaned_session_rows() -> list[str]:
return swept
# ── Cross-backend heartbeat (#94895) ───────────────────────────────────
# Each serve / gateway process registers a heartbeat row in
# ``gateway_heartbeats`` so the startup orphan sweep can tell "row owned
# by a live but idle backend" from "row truly orphaned". Without this,
# the first process to restart in a multi-backend topology reaped every
# inactive row — including those held by the other N−1 still-running
# processes (the #94895 reporter saw 473 sessions disappear in one shot).
#
# Refresh cadence: every HEARTBEAT_REFRESH_S (default 60s — much shorter
# than the default 6h session TTL so a refresh always lands inside the
# staleness window). The heartbeat is removed at process exit so a
# graceful shutdown doesn't leave a stale row behind. A crashed process
# leaves its row until the heartbeat ages out of the staleness window,
# at which point the sweep treats it as dead.
_HEARTBEAT_REFRESH_S = float(
os.environ.get("HERMES_GATEWAY_HEARTBEAT_REFRESH_S") or 60.0
)
_HEARTBEAT_REFRESH_S = max(0.0, _HEARTBEAT_REFRESH_S)
_heartbeat_refresher_started = False
_heartbeat_refresher_lock = threading.Lock()
def _backend_id_for_this_process() -> str:
"""Stable identity for this process's heartbeat row (#94895).
Includes pid AND a startup-time nonce so a PID-reuse respawn cannot
inherit the dead predecessor's heartbeat and protect truly orphaned
sessions. The pid is kept for human readability in diagnostics.
"""
nonce = getattr(_backend_id_for_this_process, "_nonce", None)
if nonce is None:
import secrets as _secrets
nonce = _secrets.token_hex(4)
try:
setattr(_backend_id_for_this_process, "_nonce", nonce)
except AttributeError: # pragma: no cover - defensive
pass
return f"{_current_profile_name()}@{os.uname().nodename if hasattr(os, 'uname') else 'host'}:{os.getpid()}:{nonce}"
def _refresh_backend_heartbeat() -> None:
"""Refresh this backend's heartbeat row (#94895). No-op when DB unavailable."""
db = _get_db()
if db is None:
return
try:
db.register_backend_heartbeat(
backend_id=_backend_id_for_this_process(),
pid=os.getpid(),
started_at=_gateway_started_at(),
profile=_current_profile_name(),
host=(os.uname().nodename if hasattr(os, "uname") else "host"),
)
except Exception:
logger.debug("backend heartbeat refresh failed", exc_info=True)
def _gateway_started_at() -> float:
"""Wall-clock time when this process started. Module-import time is
a good-enough proxy: the heartbeat refresher runs after the gateway
is fully wired up.
"""
started = getattr(_gateway_started_at, "_t", None)
if started is None:
started = time.time()
try:
setattr(_gateway_started_at, "_t", started)
except AttributeError: # pragma: no cover
pass
return started
def _heartbeat_refresher_loop(stop_event: threading.Event) -> None:
"""Background loop that refreshes the heartbeat on a fixed cadence."""
while not stop_event.is_set():
try:
_refresh_backend_heartbeat()
except Exception:
logger.debug("heartbeat refresh loop iteration failed", exc_info=True)
stop_event.wait(_HEARTBEAT_REFRESH_S)
def _start_backend_heartbeat_refresher() -> None:
"""Register this backend and start the refresher thread (#94895).
Called once per process from both gateway entry points. The first
refresh writes the row immediately so even a very fast crash leaves
a fresh-enough row that other backends can see. Repeat calls are
no-ops. The refresher thread is only spawned when
``_HEARTBEAT_REFRESH_S > 0`` — a refresh interval of zero means
"register the row once, never refresh" (the row ages out naturally
after the heartbeat staleness window).
"""
global _heartbeat_refresher_started
with _heartbeat_refresher_lock:
if _heartbeat_refresher_started:
return
_heartbeat_refresher_started = True
# Write a row synchronously so the sweep run later in this same
# process can see ourselves in the heartbeat table too. Without
# this, exclude_ids would have to cover every local session — a
# regression in the strict-ownership case the heartbeat exists to fix.
try:
_refresh_backend_heartbeat()
except Exception:
logger.debug("initial backend heartbeat write failed", exc_info=True)
if _HEARTBEAT_REFRESH_S <= 0:
return
stop_event = threading.Event()
def _atexit_clear():
stop_event.set()
try:
db = _get_db()
if db is not None:
db.clear_backend_heartbeat(_backend_id_for_this_process())
except Exception:
pass
atexit.register(_atexit_clear)
thread = threading.Thread(
target=_heartbeat_refresher_loop,
args=(stop_event,),
name="hermes-gateway-heartbeat",
daemon=True,
)
thread.start()
def _schedule_startup_orphan_sweep() -> None:
"""Schedule the once-per-process startup orphan sweep (#65194).
+9
View File
@@ -393,6 +393,15 @@ async def handle_ws(
# Track this peer for session-less global broadcasts (skin.changed
# from the background watcher) — write_json can't route those.
server.register_live_transport(transport)
# Cross-backend liveness (#94895): register a heartbeat row so
# the startup orphan sweep can distinguish "row owned by a live
# but idle backend" from "row truly orphaned". The stdio TUI's
# entry.main() does the same; idempotent + once-per-process so a
# stdio TUI that already started the refresher is a no-op here.
try:
server._start_backend_heartbeat_refresher()
except Exception:
_log.warning("backend heartbeat refresher start failed", exc_info=True)
# Same once-per-process startup pass for session rows orphaned by a
# previous gateway process (#65194): the desktop app and web dashboard
# reach the agent through this WS sidecar, not entry.main(). Idempotent