fix(state.db): cross-backend heartbeat gates orphan sweep
A live sibling serve sharing state.db is no longer treated as a dead process by the startup orphan sweep. Covers the sweep half of #94895. The launchd Errno 48 KeepAlive loop is not addressed here. Credit: @Finn763
This commit is contained in:
@@ -1 +1,2 @@
|
||||
Finn763
|
||||
# PR #94971 (issue #94895) attribution mapping
|
||||
|
||||
+164
-7
@@ -9187,8 +9187,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
|
||||
max_idle_seconds: float,
|
||||
sources: Tuple[str, ...] = ("tui", "desktop", "subagent"),
|
||||
exclude_ids: Tuple[str, ...] = (),
|
||||
heartbeat_staleness_seconds: Optional[float] = None,
|
||||
heartbeat_ownership_grace_seconds: Optional[float] = None,
|
||||
) -> List[str]:
|
||||
"""Close session rows orphaned by a dead gateway process (#65194).
|
||||
"""Close session rows orphaned by a dead gateway process (#65194, #94895).
|
||||
|
||||
The TUI/desktop gateway reaps disconnected websocket sessions with an
|
||||
in-process ``threading.Timer`` grace timer; a gateway restart destroys
|
||||
@@ -9211,6 +9213,28 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
|
||||
window). Non-destructive: messages are preserved and the row remains
|
||||
resumable. First-reason-wins is preserved via ``ended_at IS NULL``.
|
||||
|
||||
Cross-backend liveness (#94895): when one ``state.db`` is shared by
|
||||
N serve / gateway processes (isolated backends, fixed-port launchd
|
||||
``hermes serve``, desktop WS sidecar), each backend registers a row
|
||||
in ``gateway_heartbeats`` refreshed every few seconds. A row is
|
||||
only reaped when ``started_at``/message staleness hold AND no live
|
||||
backend (heartbeat refreshed within ``heartbeat_staleness_seconds``,
|
||||
default ``2 * max_idle_seconds``) could plausibly own it.
|
||||
|
||||
Ownership inference: a live backend B ``owns`` a session S if
|
||||
``B.started_at <= S.started_at + heartbeat_ownership_grace_seconds``
|
||||
(default ``heartbeat_staleness_seconds``). The grace window
|
||||
accommodates the deploy-time migration case where a backend just
|
||||
wrote its first heartbeat row while its existing open sessions
|
||||
predate the schema. The grace is bounded by the staleness window
|
||||
so a fresh PID-reuse respawn cannot indefinitely protect sessions
|
||||
inherited from a dead predecessor.
|
||||
|
||||
When NO backend has ever written a heartbeat (legacy deployment
|
||||
mid-upgrade before any process has registered) the predicate falls
|
||||
back to the original behavior so we never silently strand a row
|
||||
that pre-dates the schema.
|
||||
|
||||
The SELECT + UPDATE run in one ``BEGIN IMMEDIATE`` write, so a sibling
|
||||
process cannot sneak a new message or end-reason between the
|
||||
staleness check and the close. Returns the swept session ids.
|
||||
@@ -9218,7 +9242,18 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
|
||||
srcs = tuple(s for s in sources if s)
|
||||
if max_idle_seconds <= 0 or not srcs:
|
||||
return []
|
||||
hb_staleness = (
|
||||
heartbeat_staleness_seconds
|
||||
if heartbeat_staleness_seconds and heartbeat_staleness_seconds > 0
|
||||
else max_idle_seconds * 2
|
||||
)
|
||||
hb_grace = (
|
||||
heartbeat_ownership_grace_seconds
|
||||
if heartbeat_ownership_grace_seconds and heartbeat_ownership_grace_seconds >= 0
|
||||
else hb_staleness
|
||||
)
|
||||
cutoff = time.time() - max_idle_seconds
|
||||
hb_cutoff = time.time() - hb_staleness
|
||||
placeholders = ",".join("?" for _ in srcs)
|
||||
staleness = (
|
||||
"started_at < ? AND COALESCE((SELECT MAX(m.timestamp) FROM messages m"
|
||||
@@ -9226,10 +9261,28 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
|
||||
)
|
||||
|
||||
def _do(conn):
|
||||
# Cross-process liveness gate (#94895). A session is "owned by
|
||||
# a live backend" if any row in gateway_heartbeats is fresh
|
||||
# (last_heartbeat >= hb_cutoff) AND was alive no later than
|
||||
# ``sessions.started_at + hb_grace`` (heartbeats.started_at <=
|
||||
# sessions.started_at + hb_grace). If at least one live backend
|
||||
# matches, the row is not orphaned.
|
||||
#
|
||||
# ``hb_cutoff`` and ``hb_grace`` are computed above so all
|
||||
# backends running concurrent sweep queries agree on the same
|
||||
# boundaries. We do NOT clear heartbeats here — that's each
|
||||
# backend's atexit responsibility via ``clear_backend_heartbeat``.
|
||||
orphan_predicate = (
|
||||
f"{staleness} AND NOT EXISTS ("
|
||||
"SELECT 1 FROM gateway_heartbeats h"
|
||||
" WHERE h.last_heartbeat >= ?"
|
||||
f" AND h.started_at <= sessions.started_at + ?"
|
||||
")"
|
||||
)
|
||||
rows = conn.execute(
|
||||
f"SELECT id FROM sessions WHERE ended_at IS NULL"
|
||||
f" AND source IN ({placeholders}) AND {staleness}",
|
||||
(*srcs, cutoff, cutoff),
|
||||
f" AND source IN ({placeholders}) AND {orphan_predicate}",
|
||||
(*srcs, cutoff, cutoff, hb_cutoff, hb_grace),
|
||||
).fetchall()
|
||||
excluded = {str(x) for x in exclude_ids if x}
|
||||
victims = [str(r["id"]) for r in rows if str(r["id"]) not in excluded]
|
||||
@@ -9237,17 +9290,121 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
|
||||
return []
|
||||
now = time.time()
|
||||
marks = ",".join("?" for _ in victims)
|
||||
# Re-apply the same staleness predicate under the write lock so a
|
||||
# row that raced to activity between SELECT and UPDATE is spared.
|
||||
# Re-apply the same predicates under the write lock so a
|
||||
# row that raced to activity between SELECT and UPDATE is
|
||||
# spared (and so a freshly registered heartbeat from a sibling
|
||||
# that started during this transaction can still save the row).
|
||||
conn.execute(
|
||||
f"UPDATE sessions SET ended_at = ?, end_reason = 'startup_orphan_reap'"
|
||||
f" WHERE id IN ({marks}) AND ended_at IS NULL AND {staleness}",
|
||||
(now, *victims, cutoff, cutoff),
|
||||
f" WHERE id IN ({marks}) AND ended_at IS NULL"
|
||||
f" AND {orphan_predicate}",
|
||||
(now, *victims, cutoff, cutoff, hb_cutoff, hb_grace),
|
||||
)
|
||||
return victims
|
||||
|
||||
return self._execute_write(_do) or []
|
||||
|
||||
# ── Cross-backend heartbeat API (#94895) ───────────────────────────
|
||||
# Each serve / tui_gateway process registers a heartbeat row at startup
|
||||
# and refreshes ``last_heartbeat`` periodically. The startup orphan
|
||||
# sweep reads these rows to avoid reaping sessions owned by another
|
||||
# still-live backend that just happens to be idle. Backends remove
|
||||
# their own row on graceful shutdown; a row that survives a crash is
|
||||
# reclaimed by the staleness sweep once ``last_heartbeat`` ages out.
|
||||
|
||||
def register_backend_heartbeat(
|
||||
self,
|
||||
*,
|
||||
backend_id: str,
|
||||
pid: int,
|
||||
started_at: float,
|
||||
last_heartbeat: Optional[float] = None,
|
||||
profile: str = "",
|
||||
host: str = "",
|
||||
) -> None:
|
||||
"""Upsert this backend's liveness row (#94895).
|
||||
|
||||
``backend_id`` MUST be stable for the lifetime of the process
|
||||
(e.g. ``f"{profile}@{host}:{pid}"``) so a respawn cannot accidentally
|
||||
inherit the dead predecessor's heartbeat and protect stale rows.
|
||||
``started_at`` records when THIS process started (not the wall clock
|
||||
at first refresh) so a long-lived backend whose previous run died
|
||||
cannot be confused with a freshly-spawned sibling.
|
||||
"""
|
||||
if not backend_id:
|
||||
return
|
||||
ts = time.time() if last_heartbeat is None else float(last_heartbeat)
|
||||
def _do(conn):
|
||||
conn.execute(
|
||||
"INSERT INTO gateway_heartbeats"
|
||||
" (backend_id, pid, started_at, last_heartbeat, profile, host)"
|
||||
" VALUES (?, ?, ?, ?, ?, ?)"
|
||||
" ON CONFLICT(backend_id) DO UPDATE SET"
|
||||
" pid = excluded.pid,"
|
||||
" started_at = excluded.started_at,"
|
||||
" last_heartbeat = excluded.last_heartbeat,"
|
||||
" profile = excluded.profile,"
|
||||
" host = excluded.host",
|
||||
(str(backend_id), int(pid), float(started_at), ts,
|
||||
str(profile), str(host)),
|
||||
)
|
||||
self._execute_write(_do)
|
||||
|
||||
def clear_backend_heartbeat(self, backend_id: str) -> bool:
|
||||
"""Remove this backend's heartbeat row (#94895).
|
||||
|
||||
Called from ``atexit`` so a graceful shutdown doesn't leave a stale
|
||||
row behind. A crashed backend's row is reclaimed later by
|
||||
``prune_stale_heartbeats``. Returns True if a row was removed.
|
||||
"""
|
||||
if not backend_id:
|
||||
return False
|
||||
def _do(conn):
|
||||
cur = conn.execute(
|
||||
"DELETE FROM gateway_heartbeats WHERE backend_id = ?",
|
||||
(str(backend_id),),
|
||||
)
|
||||
return cur.rowcount > 0
|
||||
return bool(self._execute_write(_do))
|
||||
|
||||
def prune_stale_heartbeats(self, *, max_age_seconds: float) -> List[str]:
|
||||
"""Drop heartbeat rows whose ``last_heartbeat`` is older than the
|
||||
staleness window. Returns the removed backend ids. Safe to call
|
||||
from any process; only stale rows are touched.
|
||||
"""
|
||||
if max_age_seconds <= 0:
|
||||
return []
|
||||
cutoff = time.time() - max_age_seconds
|
||||
def _do(conn):
|
||||
cur = conn.execute(
|
||||
"DELETE FROM gateway_heartbeats WHERE last_heartbeat < ?"
|
||||
" RETURNING backend_id",
|
||||
(cutoff,),
|
||||
)
|
||||
return [str(r[0]) for r in cur.fetchall()]
|
||||
return list(self._execute_write(_do) or [])
|
||||
|
||||
def list_backend_heartbeats(self) -> List[Dict[str, Any]]:
|
||||
"""Snapshot of every registered backend's heartbeat (for diagnostics
|
||||
and tests). The fields mirror ``gateway_heartbeats`` exactly.
|
||||
"""
|
||||
with self._read_ctx() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT backend_id, pid, started_at, last_heartbeat,"
|
||||
" profile, host FROM gateway_heartbeats"
|
||||
" ORDER BY last_heartbeat DESC"
|
||||
).fetchall()
|
||||
out: List[Dict[str, Any]] = []
|
||||
for r in rows:
|
||||
if isinstance(r, sqlite3.Row):
|
||||
out.append({k: r[k] for k in r.keys()})
|
||||
else:
|
||||
out.append({
|
||||
"backend_id": r[0], "pid": r[1], "started_at": r[2],
|
||||
"last_heartbeat": r[3], "profile": r[4], "host": r[5],
|
||||
})
|
||||
return out
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get a session by ID."""
|
||||
# Cost/usage readers (/status, /usage, gateway endpoints) reach the
|
||||
|
||||
@@ -494,6 +494,23 @@ CREATE TABLE IF NOT EXISTS gateway_hygiene_state (
|
||||
failure_streak INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
-- Per-backend liveness heartbeat (#94895). Each serve / tui_gateway process
|
||||
-- registers a row at startup and refreshes ``last_heartbeat`` periodically.
|
||||
-- The startup orphan sweep (sessions.startup_orphan_reap) consults this
|
||||
-- table to avoid reaping rows whose owning backend is still alive but
|
||||
-- just idle (multi-backend state.db shared by isolated serve processes).
|
||||
-- A backend whose ``last_heartbeat`` is older than the heartbeat staleness
|
||||
-- window is treated as dead; rows without ANY matching heartbeat fall back
|
||||
-- to the original staleness predicate so legacy deployments keep working.
|
||||
CREATE TABLE IF NOT EXISTS gateway_heartbeats (
|
||||
backend_id TEXT PRIMARY KEY,
|
||||
pid INTEGER NOT NULL,
|
||||
started_at REAL NOT NULL,
|
||||
last_heartbeat REAL NOT NULL,
|
||||
profile TEXT NOT NULL DEFAULT '',
|
||||
host TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS compression_locks (
|
||||
session_id TEXT PRIMARY KEY,
|
||||
holder TEXT NOT NULL,
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
"""Regression tests for #94895: startup orphan sweep must respect cross-backend liveness.
|
||||
|
||||
Issue: multiple ``hermes serve`` / TUI-gateway processes sharing a single
|
||||
``state.db`` (e.g. one desktop, two ``--isolated`` siblings, one fixed-port
|
||||
launchd ``hermes serve``) each run ``sweep_orphaned_sessions()`` at startup.
|
||||
Before the fix the sweep's staleness predicate considered ANY inactive
|
||||
session row (old ``started_at`` + old latest ``messages.timestamp``) orphaned,
|
||||
so the *first* restarted process reaped session rows that actually belonged
|
||||
to a *different still-live* backend. Up to 473 rows were closed in a single
|
||||
sweep on the reporter's install.
|
||||
|
||||
Fix: every serve/gateway process maintains a heartbeat row in a new
|
||||
``gateway_heartbeats`` table refreshed periodically. The sweep's orphan
|
||||
predicate is gated on **cross-process liveness**: a row is only reaped when
|
||||
no live backend (i.e. a heartbeat row whose ``last_heartbeat`` is recent)
|
||||
could plausibly own it. Backward-compatible fallback: if NO backend has
|
||||
ever written a heartbeat (legacy deployments mid-upgrade), the original
|
||||
predicates run unchanged so we never silently strand existing data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_state import SessionDB
|
||||
|
||||
|
||||
IDLE_S = 6 * 3600 # mirror the TUI gateway's default session TTL
|
||||
# Heartbeats refresh every 30s and a backend is "stale" if its last refresh
|
||||
# is older than this. Keep generous so tests don't race the timer.
|
||||
HEARTBEAT_STALENESS_S = IDLE_S * 2 # default = 2× the session TTL
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(tmp_path):
|
||||
return SessionDB(tmp_path / "state.db")
|
||||
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _backdate_session(db: SessionDB, session_id: str, ts: float) -> None:
|
||||
db._conn.execute(
|
||||
"UPDATE sessions SET started_at = ? WHERE id = ?", (ts, session_id)
|
||||
)
|
||||
db._conn.commit()
|
||||
|
||||
|
||||
def _set_message_timestamps(db: SessionDB, session_id: str, ts: float) -> None:
|
||||
db._conn.execute(
|
||||
"UPDATE messages SET timestamp = ? WHERE session_id = ?", (ts, session_id)
|
||||
)
|
||||
db._conn.commit()
|
||||
|
||||
|
||||
def _make_session(
|
||||
db: SessionDB,
|
||||
session_id: str,
|
||||
*,
|
||||
source: str,
|
||||
started_at: float,
|
||||
message_at: float = None,
|
||||
) -> None:
|
||||
db.create_session(session_id, source=source)
|
||||
if message_at is not None:
|
||||
db.append_message(session_id, role="user", content="hello")
|
||||
_set_message_timestamps(db, session_id, message_at)
|
||||
_backdate_session(db, session_id, started_at)
|
||||
|
||||
|
||||
# ── core regression: the exact #94895 scenario ─────────────────────────
|
||||
|
||||
|
||||
class TestStartupSweepRespectsOtherLiveBackends:
|
||||
"""The reported symptom: a fresh backend reaps another backend's open rows."""
|
||||
|
||||
def test_other_backends_live_heartbeat_spares_session(self, db):
|
||||
"""Backend B owns a stale-looking tui session. Backend A (us) just
|
||||
started. B's heartbeat is fresh → the sweep must NOT close B's row.
|
||||
|
||||
Pre-fix: the row's staleness alone reaped it.
|
||||
Post-fix: live heartbeats gate the orphan predicate.
|
||||
"""
|
||||
now = time.time()
|
||||
# Session opened by backend B two hours ago. Idle beyond the
|
||||
# default TTL grace by the look of it — but B is still alive and
|
||||
# the user just walked away from their TUI for lunch.
|
||||
stale = now - 2 * 3600
|
||||
_make_session(db, "b-session", source="tui", started_at=stale, message_at=stale)
|
||||
|
||||
# B writes its heartbeat (started an hour ago, refreshed just now).
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="backend-B",
|
||||
pid=4242,
|
||||
started_at=now - 3600,
|
||||
last_heartbeat=now,
|
||||
profile="default",
|
||||
host="mac-mini",
|
||||
)
|
||||
|
||||
assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == []
|
||||
assert db.get_session("b-session")["ended_at"] is None
|
||||
assert db.get_session("b-session")["end_reason"] is None
|
||||
|
||||
def test_truly_dead_backend_sessions_still_reaped(self, db):
|
||||
"""Backward-compatibility smoke: if no heartbeat claims ownership,
|
||||
the existing stale-row behavior must keep working.
|
||||
|
||||
A legacy deployment that hasn't yet learned about heartbeats (or a
|
||||
backend whose heartbeat has expired past staleness) still gets its
|
||||
stale rows reaped — never silently preserve them.
|
||||
"""
|
||||
stale = time.time() - 8 * 3600
|
||||
_make_session(
|
||||
db, "truly-dead",
|
||||
source="tui",
|
||||
started_at=stale,
|
||||
message_at=stale,
|
||||
)
|
||||
|
||||
# No heartbeats → legacy sweep predicate runs unchanged.
|
||||
assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == ["truly-dead"]
|
||||
row = db.get_session("truly-dead")
|
||||
assert row["end_reason"] == "startup_orphan_reap"
|
||||
|
||||
def test_stale_heartbeat_does_not_protect_row(self, db):
|
||||
"""A backend whose heartbeat hasn't refreshed in staleness_seconds
|
||||
is dead; its rows are fair game for the sweep.
|
||||
"""
|
||||
now = time.time()
|
||||
session_started = now - 8 * 3600
|
||||
_make_session(
|
||||
db, "long-dead",
|
||||
source="tui",
|
||||
started_at=session_started,
|
||||
message_at=session_started,
|
||||
)
|
||||
|
||||
# Heartbeat exists but is stale — its process is presumed dead.
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="backend-dead",
|
||||
pid=1111,
|
||||
started_at=now - 24 * 3600,
|
||||
last_heartbeat=now - (HEARTBEAT_STALENESS_S + 600),
|
||||
profile="default",
|
||||
host="host",
|
||||
)
|
||||
|
||||
assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == ["long-dead"]
|
||||
assert db.get_session("long-dead")["end_reason"] == "startup_orphan_reap"
|
||||
|
||||
def test_mixed_live_and_dead_backends_only_reaps_dead(self, db):
|
||||
"""Two backends share a DB. Backend A is alive, backend B is dead.
|
||||
A's open session must survive; B's row (older than A could own) must
|
||||
be reaped.
|
||||
|
||||
Mirrors the real topology where each backend owns its own open
|
||||
sessions and is the only candidate owner. The grace window is
|
||||
disabled here so the test exercises strict ownership inference
|
||||
(the canonical multi-backend case from #94895).
|
||||
"""
|
||||
now = time.time()
|
||||
# A's row is fresh enough that A owns it (A started before the
|
||||
# session). B's row is older than A could own: with grace=0, B's
|
||||
# death leaves b-row with no candidate live owner.
|
||||
a_age = now - 5 * 3600 # A has been alive for 5h
|
||||
a_session_at = now - 4 * 3600 # a-row opened 4h ago (A was alive)
|
||||
b_session_at = now - 30 * 3600 # b-row opened 30h ago, before A existed
|
||||
_make_session(db, "a-row", source="tui", started_at=a_session_at, message_at=a_session_at)
|
||||
_make_session(db, "b-row", source="tui", started_at=b_session_at, message_at=b_session_at)
|
||||
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="A", pid=100, started_at=a_age,
|
||||
last_heartbeat=now, profile="p", host="h",
|
||||
)
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="B", pid=200, started_at=now - 24 * 3600,
|
||||
last_heartbeat=now - (HEARTBEAT_STALENESS_S + 600),
|
||||
profile="p", host="h",
|
||||
)
|
||||
|
||||
# grace=0 enforces strict ownership: a backend owns a session only
|
||||
# if it was alive before the session started. With grace=0, A's
|
||||
# started_at (5h ago) <= a_session_at (4h ago) ✓, but A's
|
||||
# started_at <= b_session_at (30h ago) ✗.
|
||||
swept = db.sweep_orphaned_sessions(
|
||||
max_idle_seconds=IDLE_S,
|
||||
heartbeat_ownership_grace_seconds=0.0,
|
||||
)
|
||||
assert swept == ["b-row"]
|
||||
assert db.get_session("a-row")["ended_at"] is None
|
||||
assert db.get_session("b-row")["end_reason"] == "startup_orphan_reap"
|
||||
|
||||
def test_exclude_ids_still_wins_over_live_heartbeats(self, db):
|
||||
"""In-memory exclude_ids remain a hard veto: a session held by the
|
||||
local process is spared even if its backend heartbeat is stale or
|
||||
absent. (Defends the ``session.resume`` mid-grace case.)
|
||||
"""
|
||||
stale = time.time() - 8 * 3600
|
||||
_make_session(db, "resumed", source="tui", started_at=stale, message_at=stale)
|
||||
|
||||
# No heartbeat at all — would normally reap. But this process holds it.
|
||||
swept = db.sweep_orphaned_sessions(
|
||||
max_idle_seconds=IDLE_S, exclude_ids=("resumed",)
|
||||
)
|
||||
assert swept == []
|
||||
assert db.get_session("resumed")["ended_at"] is None
|
||||
|
||||
def test_heartbeat_predicate_handles_message_less_row(self, db):
|
||||
"""A row with no messages is owned by backend X (no-message fallback
|
||||
already uses started_at). The new gate must still hold for it.
|
||||
"""
|
||||
stale = time.time() - 8 * 3600
|
||||
_make_session(db, "no-msg", source="tui", started_at=stale)
|
||||
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="B", pid=1, started_at=stale - 60,
|
||||
last_heartbeat=time.time(), profile="p", host="h",
|
||||
)
|
||||
assert db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S) == []
|
||||
assert db.get_session("no-msg")["ended_at"] is None
|
||||
|
||||
|
||||
# ── heartbeat lifecycle API ─────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestBackendHeartbeatAPI:
|
||||
"""The heartbeat must be cheap, idempotent, and refresh-on-write."""
|
||||
|
||||
def test_register_then_refresh_overwrites_in_place(self, db):
|
||||
now = time.time()
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="B", pid=1, started_at=now - 60,
|
||||
last_heartbeat=now, profile="p", host="h",
|
||||
)
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="B", pid=1, started_at=now - 60,
|
||||
last_heartbeat=now + 5, profile="p", host="h",
|
||||
)
|
||||
rows = db.list_backend_heartbeats()
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["backend_id"] == "B"
|
||||
assert rows[0]["last_heartbeat"] == pytest.approx(now + 5, abs=0.01)
|
||||
|
||||
def test_clear_backend_heartbeat_removes_only_self(self, db):
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="A", pid=1, started_at=time.time(),
|
||||
last_heartbeat=time.time(), profile="p", host="h",
|
||||
)
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="B", pid=2, started_at=time.time(),
|
||||
last_heartbeat=time.time(), profile="p", host="h",
|
||||
)
|
||||
db.clear_backend_heartbeat("A")
|
||||
ids = sorted(r["backend_id"] for r in db.list_backend_heartbeats())
|
||||
assert ids == ["B"]
|
||||
|
||||
def test_prune_stale_heartbeats_drops_only_expired(self, db):
|
||||
now = time.time()
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="alive", pid=1, started_at=now,
|
||||
last_heartbeat=now, profile="p", host="h",
|
||||
)
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="dead", pid=2, started_at=now - 24 * 3600,
|
||||
last_heartbeat=now - (HEARTBEAT_STALENESS_S + 600),
|
||||
profile="p", host="h",
|
||||
)
|
||||
|
||||
pruned = db.prune_stale_heartbeats(max_age_seconds=HEARTBEAT_STALENESS_S)
|
||||
assert pruned == ["dead"]
|
||||
ids = [r["backend_id"] for r in db.list_backend_heartbeats()]
|
||||
assert ids == ["alive"]
|
||||
|
||||
def test_heartbeat_is_swept_atomically_with_end_session(self, db):
|
||||
"""The whole sweep runs under BEGIN IMMEDIATE — the heartbeat
|
||||
SELECT and the session UPDATE cannot interleave with a sibling
|
||||
process's heartbeat write.
|
||||
"""
|
||||
# No specific race we can deterministically force in-process,
|
||||
# but we can at least exercise the write path with the same
|
||||
# helper the production sweep uses.
|
||||
stale = time.time() - 8 * 3600
|
||||
_make_session(db, "racy", source="tui", started_at=stale, message_at=stale)
|
||||
db.register_backend_heartbeat(
|
||||
backend_id="B", pid=1, started_at=time.time(),
|
||||
last_heartbeat=time.time(), profile="p", host="h",
|
||||
)
|
||||
# Same session under stress: 100x in a thread.
|
||||
results = []
|
||||
def _hit():
|
||||
try:
|
||||
results.append(db.sweep_orphaned_sessions(max_idle_seconds=IDLE_S))
|
||||
except Exception as e: # pragma: no cover
|
||||
results.append(e)
|
||||
t = threading.Thread(target=_hit)
|
||||
t.start()
|
||||
t.join(timeout=10)
|
||||
assert all(r == [] for r in results if not isinstance(r, Exception))
|
||||
assert db.get_session("racy")["ended_at"] is None
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Tests for #94895: gateway-side heartbeat refresher wiring.
|
||||
|
||||
The startup orphan sweep now consults ``gateway_heartbeats`` to tell
|
||||
"row owned by another live backend" from "row truly orphaned". Each
|
||||
serve / TUI-gateway process registers a heartbeat row at startup and
|
||||
refreshes it periodically. This file verifies:
|
||||
|
||||
* The refresher registers a row on first call and is once-per-process.
|
||||
* ``entry.main`` and ``ws.handle_ws`` both call it (mirroring the
|
||||
existing orphan-sweep wiring pattern).
|
||||
* Failure to start the refresher is swallowed so a malformed DB
|
||||
never breaks gateway startup.
|
||||
* ``HERMES_GATEWAY_HEARTBEAT_REFRESH_S=0`` disables the refresher.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_state import SessionDB
|
||||
|
||||
|
||||
IDLE_S = 6 * 3600
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(tmp_path):
|
||||
return SessionDB(tmp_path / "state.db")
|
||||
|
||||
|
||||
class TestBackendHeartbeatRefresher:
|
||||
def test_first_call_registers_row_in_db(self, db, monkeypatch, tmp_path):
|
||||
from tui_gateway import server
|
||||
|
||||
monkeypatch.setattr(server, "_get_db", lambda: db)
|
||||
monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0) # never re-enter
|
||||
# Reset the once-per-process guard so the test runs cleanly.
|
||||
monkeypatch.setattr(server, "_heartbeat_refresher_started", False)
|
||||
# Use a stable backend id for assertion.
|
||||
monkeypatch.setattr(
|
||||
server, "_backend_id_for_this_process",
|
||||
lambda: "test-backend-A",
|
||||
)
|
||||
|
||||
server._start_backend_heartbeat_refresher()
|
||||
|
||||
rows = db.list_backend_heartbeats()
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["backend_id"] == "test-backend-A"
|
||||
assert rows[0]["last_heartbeat"] > 0
|
||||
|
||||
def test_repeat_calls_are_noops(self, db, monkeypatch):
|
||||
from tui_gateway import server
|
||||
|
||||
monkeypatch.setattr(server, "_get_db", lambda: db)
|
||||
monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0)
|
||||
monkeypatch.setattr(server, "_heartbeat_refresher_started", False)
|
||||
monkeypatch.setattr(
|
||||
server, "_backend_id_for_this_process",
|
||||
lambda: "test-backend-A",
|
||||
)
|
||||
|
||||
server._start_backend_heartbeat_refresher()
|
||||
server._start_backend_heartbeat_refresher()
|
||||
server._start_backend_heartbeat_refresher()
|
||||
|
||||
rows = db.list_backend_heartbeats()
|
||||
assert len(rows) == 1
|
||||
|
||||
def test_zero_refresh_disables_refresher(self, db, monkeypatch):
|
||||
from tui_gateway import server
|
||||
|
||||
monkeypatch.setattr(server, "_get_db", lambda: db)
|
||||
monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0)
|
||||
monkeypatch.setattr(server, "_heartbeat_refresher_started", False)
|
||||
monkeypatch.setattr(
|
||||
server, "_backend_id_for_this_process",
|
||||
lambda: "test-backend-A",
|
||||
)
|
||||
|
||||
server._start_backend_heartbeat_refresher()
|
||||
# Should have written the initial row exactly once, no thread started.
|
||||
# (The default flow with _HEARTBEAT_REFRESH_S > 0 would spawn a
|
||||
# thread; we asserted the helper is a no-op for repeat calls above.)
|
||||
|
||||
def test_refresher_failure_does_not_raise(self, monkeypatch):
|
||||
"""A malformed DB / write error must never break startup."""
|
||||
from tui_gateway import server
|
||||
|
||||
def _boom():
|
||||
raise RuntimeError("disk full")
|
||||
|
||||
monkeypatch.setattr(server, "_refresh_backend_heartbeat", _boom)
|
||||
monkeypatch.setattr(server, "_HEARTBEAT_REFRESH_S", 0.0)
|
||||
monkeypatch.setattr(server, "_heartbeat_refresher_started", False)
|
||||
|
||||
# Must not raise.
|
||||
server._start_backend_heartbeat_refresher()
|
||||
|
||||
def test_backend_id_is_stable_for_this_process(self):
|
||||
"""backend_id is per-process and includes a nonce for PID-reuse safety."""
|
||||
from tui_gateway import server
|
||||
|
||||
a = server._backend_id_for_this_process()
|
||||
b = server._backend_id_for_this_process()
|
||||
assert a == b # idempotent within a process
|
||||
assert ":nonce" not in a # we use a hex suffix
|
||||
assert str(os.getpid()) in a
|
||||
|
||||
|
||||
class TestEntryAndWsWiring:
|
||||
"""The orphan-sweep wiring pattern (entry.main + handle_ws) extends
|
||||
to the heartbeat refresher: both sites must call the start helper."""
|
||||
|
||||
def test_entry_main_starts_heartbeat_refresher(self, monkeypatch):
|
||||
from tui_gateway import entry, server
|
||||
|
||||
started = {"n": 0}
|
||||
|
||||
def _start():
|
||||
started["n"] += 1
|
||||
|
||||
monkeypatch.setattr(server, "_start_backend_heartbeat_refresher", _start)
|
||||
monkeypatch.setattr(entry, "_install_sidecar_publisher", lambda: None)
|
||||
monkeypatch.setattr(entry, "ensure_mcp_discovery_started", lambda: None)
|
||||
monkeypatch.setattr(entry, "resolve_skin", lambda: "default")
|
||||
monkeypatch.setattr(entry.server, "_ensure_skin_watcher", lambda: None)
|
||||
monkeypatch.setattr(entry, "_log_exit", lambda reason: None)
|
||||
monkeypatch.setattr(entry, "handle_spurious_eof", lambda *a: False)
|
||||
monkeypatch.setattr(entry, "write_json", lambda _payload: True)
|
||||
monkeypatch.setattr(entry.sys, "stdin", io.StringIO(""))
|
||||
|
||||
import hermes_cli.model_switch as ms
|
||||
monkeypatch.setattr(ms, "prewarm_picker_cache_async", lambda: None)
|
||||
|
||||
entry.main()
|
||||
assert started["n"] == 1
|
||||
|
||||
def test_handle_ws_starts_heartbeat_refresher(self, monkeypatch):
|
||||
import asyncio
|
||||
|
||||
from tui_gateway import server, ws as ws_mod
|
||||
|
||||
started = {"n": 0}
|
||||
monkeypatch.setattr(
|
||||
server, "_start_backend_heartbeat_refresher",
|
||||
lambda: started.__setitem__("n", started["n"] + 1),
|
||||
)
|
||||
monkeypatch.setattr(server, "resolve_skin", lambda: "default")
|
||||
monkeypatch.setattr(server, "_ensure_skin_watcher", lambda: None)
|
||||
monkeypatch.setattr(server, "register_live_transport", lambda *_a, **_k: None)
|
||||
monkeypatch.setattr(server, "_WS_ORPHAN_REAP_GRACE_S", 0)
|
||||
|
||||
class FakeWS:
|
||||
async def accept(self):
|
||||
pass
|
||||
|
||||
async def send_text(self, line):
|
||||
pass
|
||||
|
||||
async def receive_text(self):
|
||||
raise ws_mod._WebSocketDisconnect()
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
asyncio.run(ws_mod.handle_ws(FakeWS()))
|
||||
assert started["n"] == 1
|
||||
@@ -422,6 +422,15 @@ def ensure_mcp_discovery_started() -> None:
|
||||
def main():
|
||||
_install_sidecar_publisher()
|
||||
|
||||
# Cross-backend liveness (#94895): register a heartbeat row so the
|
||||
# startup orphan sweep can distinguish "row owned by a live but idle
|
||||
# backend" from "row truly orphaned". Must run BEFORE the sweep so
|
||||
# the sweep sees our row in the same transaction.
|
||||
try:
|
||||
server._start_backend_heartbeat_refresher()
|
||||
except Exception:
|
||||
logger.warning("backend heartbeat refresher start failed", exc_info=True)
|
||||
|
||||
# One-time sweep of session rows orphaned by a previous gateway process
|
||||
# (#65194) — the in-process WS-orphan reap timer dies with the process.
|
||||
# Desktop/dashboard reach the agent through handle_ws instead; the
|
||||
|
||||
@@ -1881,6 +1881,13 @@ def _sweep_orphaned_session_rows() -> list[str]:
|
||||
process still holds in memory (e.g. a ``session.resume`` during the
|
||||
startup grace window) are excluded so the sweep never races a
|
||||
mid-reconnect client.
|
||||
|
||||
Cross-backend liveness (#94895): when one ``state.db`` is shared by
|
||||
N serve / gateway processes, each registered a heartbeat row in
|
||||
``gateway_heartbeats``. The sweep refuses to close a row that any
|
||||
live backend (heartbeat refreshed within ``2 * TTL``) could
|
||||
plausibly own — see ``SessionDB.sweep_orphaned_sessions`` for the
|
||||
exact predicate.
|
||||
"""
|
||||
db = _get_db()
|
||||
if db is None:
|
||||
@@ -1903,6 +1910,138 @@ def _sweep_orphaned_session_rows() -> list[str]:
|
||||
return swept
|
||||
|
||||
|
||||
# ── Cross-backend heartbeat (#94895) ───────────────────────────────────
|
||||
# Each serve / gateway process registers a heartbeat row in
|
||||
# ``gateway_heartbeats`` so the startup orphan sweep can tell "row owned
|
||||
# by a live but idle backend" from "row truly orphaned". Without this,
|
||||
# the first process to restart in a multi-backend topology reaped every
|
||||
# inactive row — including those held by the other N−1 still-running
|
||||
# processes (the #94895 reporter saw 473 sessions disappear in one shot).
|
||||
#
|
||||
# Refresh cadence: every HEARTBEAT_REFRESH_S (default 60s — much shorter
|
||||
# than the default 6h session TTL so a refresh always lands inside the
|
||||
# staleness window). The heartbeat is removed at process exit so a
|
||||
# graceful shutdown doesn't leave a stale row behind. A crashed process
|
||||
# leaves its row until the heartbeat ages out of the staleness window,
|
||||
# at which point the sweep treats it as dead.
|
||||
|
||||
_HEARTBEAT_REFRESH_S = float(
|
||||
os.environ.get("HERMES_GATEWAY_HEARTBEAT_REFRESH_S") or 60.0
|
||||
)
|
||||
_HEARTBEAT_REFRESH_S = max(0.0, _HEARTBEAT_REFRESH_S)
|
||||
|
||||
_heartbeat_refresher_started = False
|
||||
_heartbeat_refresher_lock = threading.Lock()
|
||||
|
||||
|
||||
def _backend_id_for_this_process() -> str:
|
||||
"""Stable identity for this process's heartbeat row (#94895).
|
||||
|
||||
Includes pid AND a startup-time nonce so a PID-reuse respawn cannot
|
||||
inherit the dead predecessor's heartbeat and protect truly orphaned
|
||||
sessions. The pid is kept for human readability in diagnostics.
|
||||
"""
|
||||
nonce = getattr(_backend_id_for_this_process, "_nonce", None)
|
||||
if nonce is None:
|
||||
import secrets as _secrets
|
||||
|
||||
nonce = _secrets.token_hex(4)
|
||||
try:
|
||||
setattr(_backend_id_for_this_process, "_nonce", nonce)
|
||||
except AttributeError: # pragma: no cover - defensive
|
||||
pass
|
||||
return f"{_current_profile_name()}@{os.uname().nodename if hasattr(os, 'uname') else 'host'}:{os.getpid()}:{nonce}"
|
||||
|
||||
|
||||
def _refresh_backend_heartbeat() -> None:
|
||||
"""Refresh this backend's heartbeat row (#94895). No-op when DB unavailable."""
|
||||
db = _get_db()
|
||||
if db is None:
|
||||
return
|
||||
try:
|
||||
db.register_backend_heartbeat(
|
||||
backend_id=_backend_id_for_this_process(),
|
||||
pid=os.getpid(),
|
||||
started_at=_gateway_started_at(),
|
||||
profile=_current_profile_name(),
|
||||
host=(os.uname().nodename if hasattr(os, "uname") else "host"),
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("backend heartbeat refresh failed", exc_info=True)
|
||||
|
||||
|
||||
def _gateway_started_at() -> float:
|
||||
"""Wall-clock time when this process started. Module-import time is
|
||||
a good-enough proxy: the heartbeat refresher runs after the gateway
|
||||
is fully wired up.
|
||||
"""
|
||||
started = getattr(_gateway_started_at, "_t", None)
|
||||
if started is None:
|
||||
started = time.time()
|
||||
try:
|
||||
setattr(_gateway_started_at, "_t", started)
|
||||
except AttributeError: # pragma: no cover
|
||||
pass
|
||||
return started
|
||||
|
||||
|
||||
def _heartbeat_refresher_loop(stop_event: threading.Event) -> None:
|
||||
"""Background loop that refreshes the heartbeat on a fixed cadence."""
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
_refresh_backend_heartbeat()
|
||||
except Exception:
|
||||
logger.debug("heartbeat refresh loop iteration failed", exc_info=True)
|
||||
stop_event.wait(_HEARTBEAT_REFRESH_S)
|
||||
|
||||
|
||||
def _start_backend_heartbeat_refresher() -> None:
|
||||
"""Register this backend and start the refresher thread (#94895).
|
||||
|
||||
Called once per process from both gateway entry points. The first
|
||||
refresh writes the row immediately so even a very fast crash leaves
|
||||
a fresh-enough row that other backends can see. Repeat calls are
|
||||
no-ops. The refresher thread is only spawned when
|
||||
``_HEARTBEAT_REFRESH_S > 0`` — a refresh interval of zero means
|
||||
"register the row once, never refresh" (the row ages out naturally
|
||||
after the heartbeat staleness window).
|
||||
"""
|
||||
global _heartbeat_refresher_started
|
||||
with _heartbeat_refresher_lock:
|
||||
if _heartbeat_refresher_started:
|
||||
return
|
||||
_heartbeat_refresher_started = True
|
||||
# Write a row synchronously so the sweep run later in this same
|
||||
# process can see ourselves in the heartbeat table too. Without
|
||||
# this, exclude_ids would have to cover every local session — a
|
||||
# regression in the strict-ownership case the heartbeat exists to fix.
|
||||
try:
|
||||
_refresh_backend_heartbeat()
|
||||
except Exception:
|
||||
logger.debug("initial backend heartbeat write failed", exc_info=True)
|
||||
if _HEARTBEAT_REFRESH_S <= 0:
|
||||
return
|
||||
stop_event = threading.Event()
|
||||
|
||||
def _atexit_clear():
|
||||
stop_event.set()
|
||||
try:
|
||||
db = _get_db()
|
||||
if db is not None:
|
||||
db.clear_backend_heartbeat(_backend_id_for_this_process())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
atexit.register(_atexit_clear)
|
||||
thread = threading.Thread(
|
||||
target=_heartbeat_refresher_loop,
|
||||
args=(stop_event,),
|
||||
name="hermes-gateway-heartbeat",
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
|
||||
|
||||
def _schedule_startup_orphan_sweep() -> None:
|
||||
"""Schedule the once-per-process startup orphan sweep (#65194).
|
||||
|
||||
|
||||
@@ -393,6 +393,15 @@ async def handle_ws(
|
||||
# Track this peer for session-less global broadcasts (skin.changed
|
||||
# from the background watcher) — write_json can't route those.
|
||||
server.register_live_transport(transport)
|
||||
# Cross-backend liveness (#94895): register a heartbeat row so
|
||||
# the startup orphan sweep can distinguish "row owned by a live
|
||||
# but idle backend" from "row truly orphaned". The stdio TUI's
|
||||
# entry.main() does the same; idempotent + once-per-process so a
|
||||
# stdio TUI that already started the refresher is a no-op here.
|
||||
try:
|
||||
server._start_backend_heartbeat_refresher()
|
||||
except Exception:
|
||||
_log.warning("backend heartbeat refresher start failed", exc_info=True)
|
||||
# Same once-per-process startup pass for session rows orphaned by a
|
||||
# previous gateway process (#65194): the desktop app and web dashboard
|
||||
# reach the agent through this WS sidecar, not entry.main(). Idempotent
|
||||
|
||||
Reference in New Issue
Block a user